개념 설명 전체 · v6.6 / kernel/sched/fair.c
1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Completely Fair Scheduling (CFS) Class (SCHED_NORMAL/SCHED_BATCH) 4 * 5 * Copyright (C) 2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> 6 * 7 * Interactivity improvements by Mike Galbraith 8 * (C) 2007 Mike Galbraith <efault@gmx.de> 9 * 10 * Various enhancements by Dmitry Adamushko. 11 * (C) 2007 Dmitry Adamushko <dmitry.adamushko@gmail.com> 12 * 13 * Group scheduling enhancements by Srivatsa Vaddagiri 14 * Copyright IBM Corporation, 2007 15 * Author: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com> 16 * 17 * Scaled math optimizations by Thomas Gleixner 18 * Copyright (C) 2007, Thomas Gleixner <tglx@linutronix.de> 19 * 20 * Adaptive scheduling granularity, math enhancements by Peter Zijlstra 21 * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra 22 */ 23 #include <linux/energy_model.h> 24 #include <linux/mmap_lock.h> 25 #include <linux/hugetlb_inline.h> 26 #include <linux/jiffies.h> 27 #include <linux/mm_api.h> 28 #include <linux/highmem.h> 29 #include <linux/spinlock_api.h> 30 #include <linux/cpumask_api.h> 31 #include <linux/lockdep_api.h> 32 #include <linux/softirq.h> 33 #include <linux/refcount_api.h> 34 #include <linux/topology.h> 35 #include <linux/sched/clock.h> 36 #include <linux/sched/cond_resched.h> 37 #include <linux/sched/cputime.h> 38 #include <linux/sched/isolation.h> 39 #include <linux/sched/nohz.h> 40 41 #include <linux/cpuidle.h> 42 #include <linux/interrupt.h> 43 #include <linux/memory-tiers.h> 44 #include <linux/mempolicy.h> 45 #include <linux/mutex_api.h> 46 #include <linux/profile.h> 47 #include <linux/psi.h> 48 #include <linux/ratelimit.h> 49 #include <linux/task_work.h> 50 #include <linux/rbtree_augmented.h> 51 52 #include <asm/switch_to.h> 53 54 #include <linux/sched/cond_resched.h> 55 56 #include "sched.h" 57 #include "stats.h" 58 #include "autogroup.h" 59 60 /* 61 * The initial- and re-scaling of tunables is configurable 62 * 63 * Options are: 64 * 65 * SCHED_TUNABLESCALING_NONE - unscaled, always *1 66 * SCHED_TUNABLESCALING_LOG - scaled logarithmical, *1+ilog(ncpus) 67 * SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus 68 * 69 * (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus)) 70 */ 71 unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG; 72 73 /* 74 * Minimal preemption granularity for CPU-bound tasks: 75 * 76 * (default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds) 77 */ 78 unsigned int sysctl_sched_base_slice = 750000ULL; 79 static unsigned int normalized_sysctl_sched_base_slice = 750000ULL; 80 81 /* 82 * After fork, child runs first. If set to 0 (default) then 83 * parent will (try to) run first. 84 */ 85 unsigned int sysctl_sched_child_runs_first __read_mostly; 86 87 const_debug unsigned int sysctl_sched_migration_cost = 500000UL; 88 89 int sched_thermal_decay_shift; 90 static int __init setup_sched_thermal_decay_shift(char *str) 91 { 92 int _shift = 0; 93 94 if (kstrtoint(str, 0, &_shift)) 95 pr_warn("Unable to set scheduler thermal pressure decay shift parameter\n"); 96 97 sched_thermal_decay_shift = clamp(_shift, 0, 10); 98 return 1; 99 } 100 __setup("sched_thermal_decay_shift=", setup_sched_thermal_decay_shift); 101 102 #ifdef CONFIG_SMP 103 /* 104 * For asym packing, by default the lower numbered CPU has higher priority. 105 */ 106 int __weak arch_asym_cpu_priority(int cpu) 107 { 108 return -cpu; 109 } 110 111 /* 112 * The margin used when comparing utilization with CPU capacity. 113 * 114 * (default: ~20%) 115 */ 116 #define fits_capacity(cap, max) ((cap) * 1280 < (max) * 1024) 117 118 /* 119 * The margin used when comparing CPU capacities. 120 * is 'cap1' noticeably greater than 'cap2' 121 * 122 * (default: ~5%) 123 */ 124 #define capacity_greater(cap1, cap2) ((cap1) * 1024 > (cap2) * 1078) 125 #endif 126 127 #ifdef CONFIG_CFS_BANDWIDTH 128 /* 129 * Amount of runtime to allocate from global (tg) to local (per-cfs_rq) pool 130 * each time a cfs_rq requests quota. 131 * 132 * Note: in the case that the slice exceeds the runtime remaining (either due 133 * to consumption or the quota being specified to be smaller than the slice) 134 * we will always only issue the remaining available time. 135 * 136 * (default: 5 msec, units: microseconds) 137 */ 138 static unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; 139 #endif 140 141 #ifdef CONFIG_NUMA_BALANCING 142 /* Restrict the NUMA promotion throughput (MB/s) for each target node. */ 143 static unsigned int sysctl_numa_balancing_promote_rate_limit = 65536; 144 #endif 145 146 #ifdef CONFIG_SYSCTL 147 static struct ctl_table sched_fair_sysctls[] = { 148 { 149 .procname = "sched_child_runs_first", 150 .data = &sysctl_sched_child_runs_first, 151 .maxlen = sizeof(unsigned int), 152 .mode = 0644, 153 .proc_handler = proc_dointvec, 154 }, 155 #ifdef CONFIG_CFS_BANDWIDTH 156 { 157 .procname = "sched_cfs_bandwidth_slice_us", 158 .data = &sysctl_sched_cfs_bandwidth_slice, 159 .maxlen = sizeof(unsigned int), 160 .mode = 0644, 161 .proc_handler = proc_dointvec_minmax, 162 .extra1 = SYSCTL_ONE, 163 }, 164 #endif 165 #ifdef CONFIG_NUMA_BALANCING 166 { 167 .procname = "numa_balancing_promote_rate_limit_MBps", 168 .data = &sysctl_numa_balancing_promote_rate_limit, 169 .maxlen = sizeof(unsigned int), 170 .mode = 0644, 171 .proc_handler = proc_dointvec_minmax, 172 .extra1 = SYSCTL_ZERO, 173 }, 174 #endif /* CONFIG_NUMA_BALANCING */ 175 {} 176 }; 177 178 static int __init sched_fair_sysctl_init(void) 179 { 180 register_sysctl_init("kernel", sched_fair_sysctls); 181 return 0; 182 } 183 late_initcall(sched_fair_sysctl_init); 184 #endif 185 186 static inline void update_load_add(struct load_weight *lw, unsigned long inc) 187 { 188 lw->weight += inc; 189 lw->inv_weight = 0; 190 } 191 192 static inline void update_load_sub(struct load_weight *lw, unsigned long dec) 193 { 194 lw->weight -= dec; 195 lw->inv_weight = 0; 196 } 197 198 static inline void update_load_set(struct load_weight *lw, unsigned long w) 199 { 200 lw->weight = w; 201 lw->inv_weight = 0; 202 } 203 204 /* 205 * Increase the granularity value when there are more CPUs, 206 * because with more CPUs the 'effective latency' as visible 207 * to users decreases. But the relationship is not linear, 208 * so pick a second-best guess by going with the log2 of the 209 * number of CPUs. 210 * 211 * This idea comes from the SD scheduler of Con Kolivas: 212 */ 213 static unsigned int get_update_sysctl_factor(void) 214 { 215 unsigned int cpus = min_t(unsigned int, num_online_cpus(), 8); 216 unsigned int factor; 217 218 switch (sysctl_sched_tunable_scaling) { 219 case SCHED_TUNABLESCALING_NONE: 220 factor = 1; 221 break; 222 case SCHED_TUNABLESCALING_LINEAR: 223 factor = cpus; 224 break; 225 case SCHED_TUNABLESCALING_LOG: 226 default: 227 factor = 1 + ilog2(cpus); 228 break; 229 } 230 231 return factor; 232 } 233 234 static void update_sysctl(void) 235 { 236 unsigned int factor = get_update_sysctl_factor(); 237 238 #define SET_SYSCTL(name) \ 239 (sysctl_##name = (factor) * normalized_sysctl_##name) 240 SET_SYSCTL(sched_base_slice); 241 #undef SET_SYSCTL 242 } 243 244 void __init sched_init_granularity(void) 245 { 246 update_sysctl(); 247 } 248 249 #define WMULT_CONST (~0U) 250 #define WMULT_SHIFT 32 251 252 static void __update_inv_weight(struct load_weight *lw) 253 { 254 unsigned long w; 255 256 if (likely(lw->inv_weight)) 257 return; 258 259 w = scale_load_down(lw->weight); 260 261 if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST)) 262 lw->inv_weight = 1; 263 else if (unlikely(!w)) 264 lw->inv_weight = WMULT_CONST; 265 else 266 lw->inv_weight = WMULT_CONST / w; 267 } 268 269 /* 270 * delta_exec * weight / lw.weight 271 * OR 272 * (delta_exec * (weight * lw->inv_weight)) >> WMULT_SHIFT 273 * 274 * Either weight := NICE_0_LOAD and lw \e sched_prio_to_wmult[], in which case 275 * we're guaranteed shift stays positive because inv_weight is guaranteed to 276 * fit 32 bits, and NICE_0_LOAD gives another 10 bits; therefore shift >= 22. 277 * 278 * Or, weight =< lw.weight (because lw.weight is the runqueue weight), thus 279 * weight/lw.weight <= 1, and therefore our shift will also be positive. 280 */ 281 static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw) 282 { 283 u64 fact = scale_load_down(weight); 284 u32 fact_hi = (u32)(fact >> 32); 285 int shift = WMULT_SHIFT; 286 int fs; 287 288 __update_inv_weight(lw); 289 290 if (unlikely(fact_hi)) { 291 fs = fls(fact_hi); 292 shift -= fs; 293 fact >>= fs; 294 } 295 296 fact = mul_u32_u32(fact, lw->inv_weight); 297 298 fact_hi = (u32)(fact >> 32); 299 if (fact_hi) { 300 fs = fls(fact_hi); 301 shift -= fs; 302 fact >>= fs; 303 } 304 305 return mul_u64_u32_shr(delta_exec, fact, shift); 306 } 307 308 /* 309 * delta /= w 310 */ 311 static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se) 312 { 313 if (unlikely(se->load.weight != NICE_0_LOAD)) 314 delta = __calc_delta(delta, NICE_0_LOAD, &se->load); 315 316 return delta; 317 } 318 319 const struct sched_class fair_sched_class; 320 321 /************************************************************** 322 * CFS operations on generic schedulable entities: 323 */ 324 325 #ifdef CONFIG_FAIR_GROUP_SCHED 326 327 /* Walk up scheduling entities hierarchy */ 328 #define for_each_sched_entity(se) \ 329 for (; se; se = se->parent) 330 331 static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) 332 { 333 struct rq *rq = rq_of(cfs_rq); 334 int cpu = cpu_of(rq); 335 336 if (cfs_rq->on_list) 337 return rq->tmp_alone_branch == &rq->leaf_cfs_rq_list; 338 339 cfs_rq->on_list = 1; 340 341 /* 342 * Ensure we either appear before our parent (if already 343 * enqueued) or force our parent to appear after us when it is 344 * enqueued. The fact that we always enqueue bottom-up 345 * reduces this to two cases and a special case for the root 346 * cfs_rq. Furthermore, it also means that we will always reset 347 * tmp_alone_branch either when the branch is connected 348 * to a tree or when we reach the top of the tree 349 */ 350 if (cfs_rq->tg->parent && 351 cfs_rq->tg->parent->cfs_rq[cpu]->on_list) { 352 /* 353 * If parent is already on the list, we add the child 354 * just before. Thanks to circular linked property of 355 * the list, this means to put the child at the tail 356 * of the list that starts by parent. 357 */ 358 list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, 359 &(cfs_rq->tg->parent->cfs_rq[cpu]->leaf_cfs_rq_list)); 360 /* 361 * The branch is now connected to its tree so we can 362 * reset tmp_alone_branch to the beginning of the 363 * list. 364 */ 365 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 366 return true; 367 } 368 369 if (!cfs_rq->tg->parent) { 370 /* 371 * cfs rq without parent should be put 372 * at the tail of the list. 373 */ 374 list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, 375 &rq->leaf_cfs_rq_list); 376 /* 377 * We have reach the top of a tree so we can reset 378 * tmp_alone_branch to the beginning of the list. 379 */ 380 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 381 return true; 382 } 383 384 /* 385 * The parent has not already been added so we want to 386 * make sure that it will be put after us. 387 * tmp_alone_branch points to the begin of the branch 388 * where we will add parent. 389 */ 390 list_add_rcu(&cfs_rq->leaf_cfs_rq_list, rq->tmp_alone_branch); 391 /* 392 * update tmp_alone_branch to points to the new begin 393 * of the branch 394 */ 395 rq->tmp_alone_branch = &cfs_rq->leaf_cfs_rq_list; 396 return false; 397 } 398 399 static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) 400 { 401 if (cfs_rq->on_list) { 402 struct rq *rq = rq_of(cfs_rq); 403 404 /* 405 * With cfs_rq being unthrottled/throttled during an enqueue, 406 * it can happen the tmp_alone_branch points the a leaf that 407 * we finally want to del. In this case, tmp_alone_branch moves 408 * to the prev element but it will point to rq->leaf_cfs_rq_list 409 * at the end of the enqueue. 410 */ 411 if (rq->tmp_alone_branch == &cfs_rq->leaf_cfs_rq_list) 412 rq->tmp_alone_branch = cfs_rq->leaf_cfs_rq_list.prev; 413 414 list_del_rcu(&cfs_rq->leaf_cfs_rq_list); 415 cfs_rq->on_list = 0; 416 } 417 } 418 419 static inline void assert_list_leaf_cfs_rq(struct rq *rq) 420 { 421 SCHED_WARN_ON(rq->tmp_alone_branch != &rq->leaf_cfs_rq_list); 422 } 423 424 /* Iterate thr' all leaf cfs_rq's on a runqueue */ 425 #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \ 426 list_for_each_entry_safe(cfs_rq, pos, &rq->leaf_cfs_rq_list, \ 427 leaf_cfs_rq_list) 428 429 /* Do the two (enqueued) entities belong to the same group ? */ 430 static inline struct cfs_rq * 431 is_same_group(struct sched_entity *se, struct sched_entity *pse) 432 { 433 if (se->cfs_rq == pse->cfs_rq) 434 return se->cfs_rq; 435 436 return NULL; 437 } 438 439 static inline struct sched_entity *parent_entity(const struct sched_entity *se) 440 { 441 return se->parent; 442 } 443 444 static void 445 find_matching_se(struct sched_entity **se, struct sched_entity **pse) 446 { 447 int se_depth, pse_depth; 448 449 /* 450 * preemption test can be made between sibling entities who are in the 451 * same cfs_rq i.e who have a common parent. Walk up the hierarchy of 452 * both tasks until we find their ancestors who are siblings of common 453 * parent. 454 */ 455 456 /* First walk up until both entities are at same depth */ 457 se_depth = (*se)->depth; 458 pse_depth = (*pse)->depth; 459 460 while (se_depth > pse_depth) { 461 se_depth--; 462 *se = parent_entity(*se); 463 } 464 465 while (pse_depth > se_depth) { 466 pse_depth--; 467 *pse = parent_entity(*pse); 468 } 469 470 while (!is_same_group(*se, *pse)) { 471 *se = parent_entity(*se); 472 *pse = parent_entity(*pse); 473 } 474 } 475 476 static int tg_is_idle(struct task_group *tg) 477 { 478 return tg->idle > 0; 479 } 480 481 static int cfs_rq_is_idle(struct cfs_rq *cfs_rq) 482 { 483 return cfs_rq->idle > 0; 484 } 485 486 static int se_is_idle(struct sched_entity *se) 487 { 488 if (entity_is_task(se)) 489 return task_has_idle_policy(task_of(se)); 490 return cfs_rq_is_idle(group_cfs_rq(se)); 491 } 492 493 #else /* !CONFIG_FAIR_GROUP_SCHED */ 494 495 #define for_each_sched_entity(se) \ 496 for (; se; se = NULL) 497 498 static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) 499 { 500 return true; 501 } 502 503 static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) 504 { 505 } 506 507 static inline void assert_list_leaf_cfs_rq(struct rq *rq) 508 { 509 } 510 511 #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \ 512 for (cfs_rq = &rq->cfs, pos = NULL; cfs_rq; cfs_rq = pos) 513 514 static inline struct sched_entity *parent_entity(struct sched_entity *se) 515 { 516 return NULL; 517 } 518 519 static inline void 520 find_matching_se(struct sched_entity **se, struct sched_entity **pse) 521 { 522 } 523 524 static inline int tg_is_idle(struct task_group *tg) 525 { 526 return 0; 527 } 528 529 static int cfs_rq_is_idle(struct cfs_rq *cfs_rq) 530 { 531 return 0; 532 } 533 534 static int se_is_idle(struct sched_entity *se) 535 { 536 return 0; 537 } 538 539 #endif /* CONFIG_FAIR_GROUP_SCHED */ 540 541 static __always_inline 542 void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec); 543 544 /************************************************************** 545 * Scheduling class tree data structure manipulation methods: 546 */ 547 548 static inline u64 max_vruntime(u64 max_vruntime, u64 vruntime) 549 { 550 s64 delta = (s64)(vruntime - max_vruntime); 551 if (delta > 0) 552 max_vruntime = vruntime; 553 554 return max_vruntime; 555 } 556 557 static inline u64 min_vruntime(u64 min_vruntime, u64 vruntime) 558 { 559 s64 delta = (s64)(vruntime - min_vruntime); 560 if (delta < 0) 561 min_vruntime = vruntime; 562 563 return min_vruntime; 564 } 565 566 static inline bool entity_before(const struct sched_entity *a, 567 const struct sched_entity *b) 568 { 569 return (s64)(a->vruntime - b->vruntime) < 0; 570 } 571 572 static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se) 573 { 574 return (s64)(se->vruntime - cfs_rq->min_vruntime); 575 } 576 577 #define __node_2_se(node) \ 578 rb_entry((node), struct sched_entity, run_node) 579 580 /* 581 * Compute virtual time from the per-task service numbers: 582 * 583 * Fair schedulers conserve lag: 584 * 585 * \Sum lag_i = 0 586 * 587 * Where lag_i is given by: 588 * 589 * lag_i = S - s_i = w_i * (V - v_i) 590 * 591 * Where S is the ideal service time and V is it's virtual time counterpart. 592 * Therefore: 593 * 594 * \Sum lag_i = 0 595 * \Sum w_i * (V - v_i) = 0 596 * \Sum w_i * V - w_i * v_i = 0 597 * 598 * From which we can solve an expression for V in v_i (which we have in 599 * se->vruntime): 600 * 601 * \Sum v_i * w_i \Sum v_i * w_i 602 * V = -------------- = -------------- 603 * \Sum w_i W 604 * 605 * Specifically, this is the weighted average of all entity virtual runtimes. 606 * 607 * [[ NOTE: this is only equal to the ideal scheduler under the condition 608 * that join/leave operations happen at lag_i = 0, otherwise the 609 * virtual time has non-continguous motion equivalent to: 610 * 611 * V +-= lag_i / W 612 * 613 * Also see the comment in place_entity() that deals with this. ]] 614 * 615 * However, since v_i is u64, and the multiplcation could easily overflow 616 * transform it into a relative form that uses smaller quantities: 617 * 618 * Substitute: v_i == (v_i - v0) + v0 619 * 620 * \Sum ((v_i - v0) + v0) * w_i \Sum (v_i - v0) * w_i 621 * V = ---------------------------- = --------------------- + v0 622 * W W 623 * 624 * Which we track using: 625 * 626 * v0 := cfs_rq->min_vruntime 627 * \Sum (v_i - v0) * w_i := cfs_rq->avg_vruntime 628 * \Sum w_i := cfs_rq->avg_load 629 * 630 * Since min_vruntime is a monotonic increasing variable that closely tracks 631 * the per-task service, these deltas: (v_i - v), will be in the order of the 632 * maximal (virtual) lag induced in the system due to quantisation. 633 * 634 * Also, we use scale_load_down() to reduce the size. 635 * 636 * As measured, the max (key * weight) value was ~44 bits for a kernel build. 637 */ 638 static void 639 avg_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se) 640 { 641 unsigned long weight = scale_load_down(se->load.weight); 642 s64 key = entity_key(cfs_rq, se); 643 644 cfs_rq->avg_vruntime += key * weight; 645 cfs_rq->avg_load += weight; 646 } 647 648 static void 649 avg_vruntime_sub(struct cfs_rq *cfs_rq, struct sched_entity *se) 650 { 651 unsigned long weight = scale_load_down(se->load.weight); 652 s64 key = entity_key(cfs_rq, se); 653 654 cfs_rq->avg_vruntime -= key * weight; 655 cfs_rq->avg_load -= weight; 656 } 657 658 static inline 659 void avg_vruntime_update(struct cfs_rq *cfs_rq, s64 delta) 660 { 661 /* 662 * v' = v + d ==> avg_vruntime' = avg_runtime - d*avg_load 663 */ 664 cfs_rq->avg_vruntime -= cfs_rq->avg_load * delta; 665 } 666 667 /* 668 * Specifically: avg_runtime() + 0 must result in entity_eligible() := true 669 * For this to be so, the result of this function must have a left bias. 670 */ 671 u64 avg_vruntime(struct cfs_rq *cfs_rq) 672 { 673 struct sched_entity *curr = cfs_rq->curr; 674 s64 avg = cfs_rq->avg_vruntime; 675 long load = cfs_rq->avg_load; 676 677 if (curr && curr->on_rq) { 678 unsigned long weight = scale_load_down(curr->load.weight); 679 680 avg += entity_key(cfs_rq, curr) * weight; 681 load += weight; 682 } 683 684 if (load) { 685 /* sign flips effective floor / ceil */ 686 if (avg < 0) 687 avg -= (load - 1); 688 avg = div_s64(avg, load); 689 } 690 691 return cfs_rq->min_vruntime + avg; 692 } 693 694 /* 695 * lag_i = S - s_i = w_i * (V - v_i) 696 * 697 * However, since V is approximated by the weighted average of all entities it 698 * is possible -- by addition/removal/reweight to the tree -- to move V around 699 * and end up with a larger lag than we started with. 700 * 701 * Limit this to either double the slice length with a minimum of TICK_NSEC 702 * since that is the timing granularity. 703 * 704 * EEVDF gives the following limit for a steady state system: 705 * 706 * -r_max < lag < max(r_max, q) 707 * 708 * XXX could add max_slice to the augmented data to track this. 709 */ 710 static void update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se) 711 { 712 s64 lag, limit; 713 714 SCHED_WARN_ON(!se->on_rq); 715 lag = avg_vruntime(cfs_rq) - se->vruntime; 716 717 limit = calc_delta_fair(max_t(u64, 2*se->slice, TICK_NSEC), se); 718 se->vlag = clamp(lag, -limit, limit); 719 } 720 721 /* 722 * Entity is eligible once it received less service than it ought to have, 723 * eg. lag >= 0. 724 * 725 * lag_i = S - s_i = w_i*(V - v_i) 726 * 727 * lag_i >= 0 -> V >= v_i 728 * 729 * \Sum (v_i - v)*w_i 730 * V = ------------------ + v 731 * \Sum w_i 732 * 733 * lag_i >= 0 -> \Sum (v_i - v)*w_i >= (v_i - v)*(\Sum w_i) 734 * 735 * Note: using 'avg_vruntime() > se->vruntime' is inacurate due 736 * to the loss in precision caused by the division. 737 */ 738 int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se) 739 { 740 struct sched_entity *curr = cfs_rq->curr; 741 s64 avg = cfs_rq->avg_vruntime; 742 long load = cfs_rq->avg_load; 743 744 if (curr && curr->on_rq) { 745 unsigned long weight = scale_load_down(curr->load.weight); 746 747 avg += entity_key(cfs_rq, curr) * weight; 748 load += weight; 749 } 750 751 return avg >= entity_key(cfs_rq, se) * load; 752 } 753 754 static u64 __update_min_vruntime(struct cfs_rq *cfs_rq, u64 vruntime) 755 { 756 u64 min_vruntime = cfs_rq->min_vruntime; 757 /* 758 * open coded max_vruntime() to allow updating avg_vruntime 759 */ 760 s64 delta = (s64)(vruntime - min_vruntime); 761 if (delta > 0) { 762 avg_vruntime_update(cfs_rq, delta); 763 min_vruntime = vruntime; 764 } 765 return min_vruntime; 766 } 767 768 static void update_min_vruntime(struct cfs_rq *cfs_rq) 769 { 770 struct sched_entity *se = __pick_first_entity(cfs_rq); 771 struct sched_entity *curr = cfs_rq->curr; 772 773 u64 vruntime = cfs_rq->min_vruntime; 774 775 if (curr) { 776 if (curr->on_rq) 777 vruntime = curr->vruntime; 778 else 779 curr = NULL; 780 } 781 782 if (se) { 783 if (!curr) 784 vruntime = se->vruntime; 785 else 786 vruntime = min_vruntime(vruntime, se->vruntime); 787 } 788 789 /* ensure we never gain time by being placed backwards. */ 790 u64_u32_store(cfs_rq->min_vruntime, 791 __update_min_vruntime(cfs_rq, vruntime)); 792 } 793 794 static inline bool __entity_less(struct rb_node *a, const struct rb_node *b) 795 { 796 return entity_before(__node_2_se(a), __node_2_se(b)); 797 } 798 799 #define deadline_gt(field, lse, rse) ({ (s64)((lse)->field - (rse)->field) > 0; }) 800 801 static inline void __update_min_deadline(struct sched_entity *se, struct rb_node *node) 802 { 803 if (node) { 804 struct sched_entity *rse = __node_2_se(node); 805 if (deadline_gt(min_deadline, se, rse)) 806 se->min_deadline = rse->min_deadline; 807 } 808 } 809 810 /* 811 * se->min_deadline = min(se->deadline, left->min_deadline, right->min_deadline) 812 */ 813 static inline bool min_deadline_update(struct sched_entity *se, bool exit) 814 { 815 u64 old_min_deadline = se->min_deadline; 816 struct rb_node *node = &se->run_node; 817 818 se->min_deadline = se->deadline; 819 __update_min_deadline(se, node->rb_right); 820 __update_min_deadline(se, node->rb_left); 821 822 return se->min_deadline == old_min_deadline; 823 } 824 825 RB_DECLARE_CALLBACKS(static, min_deadline_cb, struct sched_entity, 826 run_node, min_deadline, min_deadline_update); 827 828 /* 829 * Enqueue an entity into the rb-tree: 830 */ 831 static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 832 { 833 avg_vruntime_add(cfs_rq, se); 834 se->min_deadline = se->deadline; 835 rb_add_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, 836 __entity_less, &min_deadline_cb); 837 } 838 839 static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 840 { 841 rb_erase_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, 842 &min_deadline_cb); 843 avg_vruntime_sub(cfs_rq, se); 844 } 845 846 struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq) 847 { 848 struct rb_node *left = rb_first_cached(&cfs_rq->tasks_timeline); 849 850 if (!left) 851 return NULL; 852 853 return __node_2_se(left); 854 } 855 856 /* 857 * Earliest Eligible Virtual Deadline First 858 * 859 * In order to provide latency guarantees for different request sizes 860 * EEVDF selects the best runnable task from two criteria: 861 * 862 * 1) the task must be eligible (must be owed service) 863 * 864 * 2) from those tasks that meet 1), we select the one 865 * with the earliest virtual deadline. 866 * 867 * We can do this in O(log n) time due to an augmented RB-tree. The 868 * tree keeps the entries sorted on service, but also functions as a 869 * heap based on the deadline by keeping: 870 * 871 * se->min_deadline = min(se->deadline, se->{left,right}->min_deadline) 872 * 873 * Which allows an EDF like search on (sub)trees. 874 */ 875 static struct sched_entity *__pick_eevdf(struct cfs_rq *cfs_rq) 876 { 877 struct rb_node *node = cfs_rq->tasks_timeline.rb_root.rb_node; 878 struct sched_entity *curr = cfs_rq->curr; 879 struct sched_entity *best = NULL; 880 struct sched_entity *best_left = NULL; 881 882 if (curr && (!curr->on_rq || !entity_eligible(cfs_rq, curr))) 883 curr = NULL; 884 best = curr; 885 886 /* 887 * Once selected, run a task until it either becomes non-eligible or 888 * until it gets a new slice. See the HACK in set_next_entity(). 889 */ 890 if (sched_feat(RUN_TO_PARITY) && curr && curr->vlag == curr->deadline) 891 return curr; 892 893 while (node) { 894 struct sched_entity *se = __node_2_se(node); 895 896 /* 897 * If this entity is not eligible, try the left subtree. 898 */ 899 if (!entity_eligible(cfs_rq, se)) { 900 node = node->rb_left; 901 continue; 902 } 903 904 /* 905 * Now we heap search eligible trees for the best (min_)deadline 906 */ 907 if (!best || deadline_gt(deadline, best, se)) 908 best = se; 909 910 /* 911 * Every se in a left branch is eligible, keep track of the 912 * branch with the best min_deadline 913 */ 914 if (node->rb_left) { 915 struct sched_entity *left = __node_2_se(node->rb_left); 916 917 if (!best_left || deadline_gt(min_deadline, best_left, left)) 918 best_left = left; 919 920 /* 921 * min_deadline is in the left branch. rb_left and all 922 * descendants are eligible, so immediately switch to the second 923 * loop. 924 */ 925 if (left->min_deadline == se->min_deadline) 926 break; 927 } 928 929 /* min_deadline is at this node, no need to look right */ 930 if (se->deadline == se->min_deadline) 931 break; 932 933 /* else min_deadline is in the right branch. */ 934 node = node->rb_right; 935 } 936 937 /* 938 * We ran into an eligible node which is itself the best. 939 * (Or nr_running == 0 and both are NULL) 940 */ 941 if (!best_left || (s64)(best_left->min_deadline - best->deadline) > 0) 942 return best; 943 944 /* 945 * Now best_left and all of its children are eligible, and we are just 946 * looking for deadline == min_deadline 947 */ 948 node = &best_left->run_node; 949 while (node) { 950 struct sched_entity *se = __node_2_se(node); 951 952 /* min_deadline is the current node */ 953 if (se->deadline == se->min_deadline) 954 return se; 955 956 /* min_deadline is in the left branch */ 957 if (node->rb_left && 958 __node_2_se(node->rb_left)->min_deadline == se->min_deadline) { 959 node = node->rb_left; 960 continue; 961 } 962 963 /* else min_deadline is in the right branch */ 964 node = node->rb_right; 965 } 966 return NULL; 967 } 968 969 static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq) 970 { 971 struct sched_entity *se = __pick_eevdf(cfs_rq); 972 973 if (!se) { 974 struct sched_entity *left = __pick_first_entity(cfs_rq); 975 if (left) { 976 pr_err("EEVDF scheduling fail, picking leftmost\n"); 977 return left; 978 } 979 } 980 981 return se; 982 } 983 984 #ifdef CONFIG_SCHED_DEBUG 985 struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq) 986 { 987 struct rb_node *last = rb_last(&cfs_rq->tasks_timeline.rb_root); 988 989 if (!last) 990 return NULL; 991 992 return __node_2_se(last); 993 } 994 995 /************************************************************** 996 * Scheduling class statistics methods: 997 */ 998 #ifdef CONFIG_SMP 999 int sched_update_scaling(void) 1000 { 1001 unsigned int factor = get_update_sysctl_factor(); 1002 1003 #define WRT_SYSCTL(name) \ 1004 (normalized_sysctl_##name = sysctl_##name / (factor)) 1005 WRT_SYSCTL(sched_base_slice); 1006 #undef WRT_SYSCTL 1007 1008 return 0; 1009 } 1010 #endif 1011 #endif 1012 1013 static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se); 1014 1015 /* 1016 * XXX: strictly: vd_i += N*r_i/w_i such that: vd_i > ve_i 1017 * this is probably good enough. 1018 */ 1019 static void update_deadline(struct cfs_rq *cfs_rq, struct sched_entity *se) 1020 { 1021 if ((s64)(se->vruntime - se->deadline) < 0) 1022 return; 1023 1024 /* 1025 * For EEVDF the virtual time slope is determined by w_i (iow. 1026 * nice) while the request time r_i is determined by 1027 * sysctl_sched_base_slice. 1028 */ 1029 se->slice = sysctl_sched_base_slice; 1030 1031 /* 1032 * EEVDF: vd_i = ve_i + r_i / w_i 1033 */ 1034 se->deadline = se->vruntime + calc_delta_fair(se->slice, se); 1035 1036 /* 1037 * The task has consumed its request, reschedule. 1038 */ 1039 if (cfs_rq->nr_running > 1) { 1040 resched_curr(rq_of(cfs_rq)); 1041 clear_buddies(cfs_rq, se); 1042 } 1043 } 1044 1045 #include "pelt.h" 1046 #ifdef CONFIG_SMP 1047 1048 static int select_idle_sibling(struct task_struct *p, int prev_cpu, int cpu); 1049 static unsigned long task_h_load(struct task_struct *p); 1050 static unsigned long capacity_of(int cpu); 1051 1052 /* Give new sched_entity start runnable values to heavy its load in infant time */ 1053 void init_entity_runnable_average(struct sched_entity *se) 1054 { 1055 struct sched_avg *sa = &se->avg; 1056 1057 memset(sa, 0, sizeof(*sa)); 1058 1059 /* 1060 * Tasks are initialized with full load to be seen as heavy tasks until 1061 * they get a chance to stabilize to their real load level. 1062 * Group entities are initialized with zero load to reflect the fact that 1063 * nothing has been attached to the task group yet. 1064 */ 1065 if (entity_is_task(se)) 1066 sa->load_avg = scale_load_down(se->load.weight); 1067 1068 /* when this task enqueue'ed, it will contribute to its cfs_rq's load_avg */ 1069 } 1070 1071 /* 1072 * With new tasks being created, their initial util_avgs are extrapolated 1073 * based on the cfs_rq's current util_avg: 1074 * 1075 * util_avg = cfs_rq->util_avg / (cfs_rq->load_avg + 1) * se.load.weight 1076 * 1077 * However, in many cases, the above util_avg does not give a desired 1078 * value. Moreover, the sum of the util_avgs may be divergent, such 1079 * as when the series is a harmonic series. 1080 * 1081 * To solve this problem, we also cap the util_avg of successive tasks to 1082 * only 1/2 of the left utilization budget: 1083 * 1084 * util_avg_cap = (cpu_scale - cfs_rq->avg.util_avg) / 2^n 1085 * 1086 * where n denotes the nth task and cpu_scale the CPU capacity. 1087 * 1088 * For example, for a CPU with 1024 of capacity, a simplest series from 1089 * the beginning would be like: 1090 * 1091 * task util_avg: 512, 256, 128, 64, 32, 16, 8, ... 1092 * cfs_rq util_avg: 512, 768, 896, 960, 992, 1008, 1016, ... 1093 * 1094 * Finally, that extrapolated util_avg is clamped to the cap (util_avg_cap) 1095 * if util_avg > util_avg_cap. 1096 */ 1097 void post_init_entity_util_avg(struct task_struct *p) 1098 { 1099 struct sched_entity *se = &p->se; 1100 struct cfs_rq *cfs_rq = cfs_rq_of(se); 1101 struct sched_avg *sa = &se->avg; 1102 long cpu_scale = arch_scale_cpu_capacity(cpu_of(rq_of(cfs_rq))); 1103 long cap = (long)(cpu_scale - cfs_rq->avg.util_avg) / 2; 1104 1105 if (p->sched_class != &fair_sched_class) { 1106 /* 1107 * For !fair tasks do: 1108 * 1109 update_cfs_rq_load_avg(now, cfs_rq); 1110 attach_entity_load_avg(cfs_rq, se); 1111 switched_from_fair(rq, p); 1112 * 1113 * such that the next switched_to_fair() has the 1114 * expected state. 1115 */ 1116 se->avg.last_update_time = cfs_rq_clock_pelt(cfs_rq); 1117 return; 1118 } 1119 1120 if (cap > 0) { 1121 if (cfs_rq->avg.util_avg != 0) { 1122 sa->util_avg = cfs_rq->avg.util_avg * se->load.weight; 1123 sa->util_avg /= (cfs_rq->avg.load_avg + 1); 1124 1125 if (sa->util_avg > cap) 1126 sa->util_avg = cap; 1127 } else { 1128 sa->util_avg = cap; 1129 } 1130 } 1131 1132 sa->runnable_avg = sa->util_avg; 1133 } 1134 1135 #else /* !CONFIG_SMP */ 1136 void init_entity_runnable_average(struct sched_entity *se) 1137 { 1138 } 1139 void post_init_entity_util_avg(struct task_struct *p) 1140 { 1141 } 1142 static void update_tg_load_avg(struct cfs_rq *cfs_rq) 1143 { 1144 } 1145 #endif /* CONFIG_SMP */ 1146 1147 /* 1148 * Update the current task's runtime statistics. 1149 */ 1150 static void update_curr(struct cfs_rq *cfs_rq) 1151 { 1152 struct sched_entity *curr = cfs_rq->curr; 1153 u64 now = rq_clock_task(rq_of(cfs_rq)); 1154 u64 delta_exec; 1155 1156 if (unlikely(!curr)) 1157 return; 1158 1159 delta_exec = now - curr->exec_start; 1160 if (unlikely((s64)delta_exec <= 0)) 1161 return; 1162 1163 curr->exec_start = now; 1164 1165 if (schedstat_enabled()) { 1166 struct sched_statistics *stats; 1167 1168 stats = __schedstats_from_se(curr); 1169 __schedstat_set(stats->exec_max, 1170 max(delta_exec, stats->exec_max)); 1171 } 1172 1173 curr->sum_exec_runtime += delta_exec; 1174 schedstat_add(cfs_rq->exec_clock, delta_exec); 1175 1176 curr->vruntime += calc_delta_fair(delta_exec, curr); 1177 update_deadline(cfs_rq, curr); 1178 update_min_vruntime(cfs_rq); 1179 1180 if (entity_is_task(curr)) { 1181 struct task_struct *curtask = task_of(curr); 1182 1183 trace_sched_stat_runtime(curtask, delta_exec, curr->vruntime); 1184 cgroup_account_cputime(curtask, delta_exec); 1185 account_group_exec_runtime(curtask, delta_exec); 1186 } 1187 1188 account_cfs_rq_runtime(cfs_rq, delta_exec); 1189 } 1190 1191 static void update_curr_fair(struct rq *rq) 1192 { 1193 update_curr(cfs_rq_of(&rq->curr->se)); 1194 } 1195 1196 static inline void 1197 update_stats_wait_start_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) 1198 { 1199 struct sched_statistics *stats; 1200 struct task_struct *p = NULL; 1201 1202 if (!schedstat_enabled()) 1203 return; 1204 1205 stats = __schedstats_from_se(se); 1206 1207 if (entity_is_task(se)) 1208 p = task_of(se); 1209 1210 __update_stats_wait_start(rq_of(cfs_rq), p, stats); 1211 } 1212 1213 static inline void 1214 update_stats_wait_end_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) 1215 { 1216 struct sched_statistics *stats; 1217 struct task_struct *p = NULL; 1218 1219 if (!schedstat_enabled()) 1220 return; 1221 1222 stats = __schedstats_from_se(se); 1223 1224 /* 1225 * When the sched_schedstat changes from 0 to 1, some sched se 1226 * maybe already in the runqueue, the se->statistics.wait_start 1227 * will be 0.So it will let the delta wrong. We need to avoid this 1228 * scenario. 1229 */ 1230 if (unlikely(!schedstat_val(stats->wait_start))) 1231 return; 1232 1233 if (entity_is_task(se)) 1234 p = task_of(se); 1235 1236 __update_stats_wait_end(rq_of(cfs_rq), p, stats); 1237 } 1238 1239 static inline void 1240 update_stats_enqueue_sleeper_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) 1241 { 1242 struct sched_statistics *stats; 1243 struct task_struct *tsk = NULL; 1244 1245 if (!schedstat_enabled()) 1246 return; 1247 1248 stats = __schedstats_from_se(se); 1249 1250 if (entity_is_task(se)) 1251 tsk = task_of(se); 1252 1253 __update_stats_enqueue_sleeper(rq_of(cfs_rq), tsk, stats); 1254 } 1255 1256 /* 1257 * Task is being enqueued - update stats: 1258 */ 1259 static inline void 1260 update_stats_enqueue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 1261 { 1262 if (!schedstat_enabled()) 1263 return; 1264 1265 /* 1266 * Are we enqueueing a waiting task? (for current tasks 1267 * a dequeue/enqueue event is a NOP) 1268 */ 1269 if (se != cfs_rq->curr) 1270 update_stats_wait_start_fair(cfs_rq, se); 1271 1272 if (flags & ENQUEUE_WAKEUP) 1273 update_stats_enqueue_sleeper_fair(cfs_rq, se); 1274 } 1275 1276 static inline void 1277 update_stats_dequeue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 1278 { 1279 1280 if (!schedstat_enabled()) 1281 return; 1282 1283 /* 1284 * Mark the end of the wait period if dequeueing a 1285 * waiting task: 1286 */ 1287 if (se != cfs_rq->curr) 1288 update_stats_wait_end_fair(cfs_rq, se); 1289 1290 if ((flags & DEQUEUE_SLEEP) && entity_is_task(se)) { 1291 struct task_struct *tsk = task_of(se); 1292 unsigned int state; 1293 1294 /* XXX racy against TTWU */ 1295 state = READ_ONCE(tsk->__state); 1296 if (state & TASK_INTERRUPTIBLE) 1297 __schedstat_set(tsk->stats.sleep_start, 1298 rq_clock(rq_of(cfs_rq))); 1299 if (state & TASK_UNINTERRUPTIBLE) 1300 __schedstat_set(tsk->stats.block_start, 1301 rq_clock(rq_of(cfs_rq))); 1302 } 1303 } 1304 1305 /* 1306 * We are picking a new current task - update its stats: 1307 */ 1308 static inline void 1309 update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se) 1310 { 1311 /* 1312 * We are starting a new run period: 1313 */ 1314 se->exec_start = rq_clock_task(rq_of(cfs_rq)); 1315 } 1316 1317 /************************************************** 1318 * Scheduling class queueing methods: 1319 */ 1320 1321 static inline bool is_core_idle(int cpu) 1322 { 1323 #ifdef CONFIG_SCHED_SMT 1324 int sibling; 1325 1326 for_each_cpu(sibling, cpu_smt_mask(cpu)) { 1327 if (cpu == sibling) 1328 continue; 1329 1330 if (!idle_cpu(sibling)) 1331 return false; 1332 } 1333 #endif 1334 1335 return true; 1336 } 1337 1338 #ifdef CONFIG_NUMA 1339 #define NUMA_IMBALANCE_MIN 2 1340 1341 static inline long 1342 adjust_numa_imbalance(int imbalance, int dst_running, int imb_numa_nr) 1343 { 1344 /* 1345 * Allow a NUMA imbalance if busy CPUs is less than the maximum 1346 * threshold. Above this threshold, individual tasks may be contending 1347 * for both memory bandwidth and any shared HT resources. This is an 1348 * approximation as the number of running tasks may not be related to 1349 * the number of busy CPUs due to sched_setaffinity. 1350 */ 1351 if (dst_running > imb_numa_nr) 1352 return imbalance; 1353 1354 /* 1355 * Allow a small imbalance based on a simple pair of communicating 1356 * tasks that remain local when the destination is lightly loaded. 1357 */ 1358 if (imbalance <= NUMA_IMBALANCE_MIN) 1359 return 0; 1360 1361 return imbalance; 1362 } 1363 #endif /* CONFIG_NUMA */ 1364 1365 #ifdef CONFIG_NUMA_BALANCING 1366 /* 1367 * Approximate time to scan a full NUMA task in ms. The task scan period is 1368 * calculated based on the tasks virtual memory size and 1369 * numa_balancing_scan_size. 1370 */ 1371 unsigned int sysctl_numa_balancing_scan_period_min = 1000; 1372 unsigned int sysctl_numa_balancing_scan_period_max = 60000; 1373 1374 /* Portion of address space to scan in MB */ 1375 unsigned int sysctl_numa_balancing_scan_size = 256; 1376 1377 /* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */ 1378 unsigned int sysctl_numa_balancing_scan_delay = 1000; 1379 1380 /* The page with hint page fault latency < threshold in ms is considered hot */ 1381 unsigned int sysctl_numa_balancing_hot_threshold = MSEC_PER_SEC; 1382 1383 struct numa_group { 1384 refcount_t refcount; 1385 1386 spinlock_t lock; /* nr_tasks, tasks */ 1387 int nr_tasks; 1388 pid_t gid; 1389 int active_nodes; 1390 1391 struct rcu_head rcu; 1392 unsigned long total_faults; 1393 unsigned long max_faults_cpu; 1394 /* 1395 * faults[] array is split into two regions: faults_mem and faults_cpu. 1396 * 1397 * Faults_cpu is used to decide whether memory should move 1398 * towards the CPU. As a consequence, these stats are weighted 1399 * more by CPU use than by memory faults. 1400 */ 1401 unsigned long faults[]; 1402 }; 1403 1404 /* 1405 * For functions that can be called in multiple contexts that permit reading 1406 * ->numa_group (see struct task_struct for locking rules). 1407 */ 1408 static struct numa_group *deref_task_numa_group(struct task_struct *p) 1409 { 1410 return rcu_dereference_check(p->numa_group, p == current || 1411 (lockdep_is_held(__rq_lockp(task_rq(p))) && !READ_ONCE(p->on_cpu))); 1412 } 1413 1414 static struct numa_group *deref_curr_numa_group(struct task_struct *p) 1415 { 1416 return rcu_dereference_protected(p->numa_group, p == current); 1417 } 1418 1419 static inline unsigned long group_faults_priv(struct numa_group *ng); 1420 static inline unsigned long group_faults_shared(struct numa_group *ng); 1421 1422 static unsigned int task_nr_scan_windows(struct task_struct *p) 1423 { 1424 unsigned long rss = 0; 1425 unsigned long nr_scan_pages; 1426 1427 /* 1428 * Calculations based on RSS as non-present and empty pages are skipped 1429 * by the PTE scanner and NUMA hinting faults should be trapped based 1430 * on resident pages 1431 */ 1432 nr_scan_pages = sysctl_numa_balancing_scan_size << (20 - PAGE_SHIFT); 1433 rss = get_mm_rss(p->mm); 1434 if (!rss) 1435 rss = nr_scan_pages; 1436 1437 rss = round_up(rss, nr_scan_pages); 1438 return rss / nr_scan_pages; 1439 } 1440 1441 /* For sanity's sake, never scan more PTEs than MAX_SCAN_WINDOW MB/sec. */ 1442 #define MAX_SCAN_WINDOW 2560 1443 1444 static unsigned int task_scan_min(struct task_struct *p) 1445 { 1446 unsigned int scan_size = READ_ONCE(sysctl_numa_balancing_scan_size); 1447 unsigned int scan, floor; 1448 unsigned int windows = 1; 1449 1450 if (scan_size < MAX_SCAN_WINDOW) 1451 windows = MAX_SCAN_WINDOW / scan_size; 1452 floor = 1000 / windows; 1453 1454 scan = sysctl_numa_balancing_scan_period_min / task_nr_scan_windows(p); 1455 return max_t(unsigned int, floor, scan); 1456 } 1457 1458 static unsigned int task_scan_start(struct task_struct *p) 1459 { 1460 unsigned long smin = task_scan_min(p); 1461 unsigned long period = smin; 1462 struct numa_group *ng; 1463 1464 /* Scale the maximum scan period with the amount of shared memory. */ 1465 rcu_read_lock(); 1466 ng = rcu_dereference(p->numa_group); 1467 if (ng) { 1468 unsigned long shared = group_faults_shared(ng); 1469 unsigned long private = group_faults_priv(ng); 1470 1471 period *= refcount_read(&ng->refcount); 1472 period *= shared + 1; 1473 period /= private + shared + 1; 1474 } 1475 rcu_read_unlock(); 1476 1477 return max(smin, period); 1478 } 1479 1480 static unsigned int task_scan_max(struct task_struct *p) 1481 { 1482 unsigned long smin = task_scan_min(p); 1483 unsigned long smax; 1484 struct numa_group *ng; 1485 1486 /* Watch for min being lower than max due to floor calculations */ 1487 smax = sysctl_numa_balancing_scan_period_max / task_nr_scan_windows(p); 1488 1489 /* Scale the maximum scan period with the amount of shared memory. */ 1490 ng = deref_curr_numa_group(p); 1491 if (ng) { 1492 unsigned long shared = group_faults_shared(ng); 1493 unsigned long private = group_faults_priv(ng); 1494 unsigned long period = smax; 1495 1496 period *= refcount_read(&ng->refcount); 1497 period *= shared + 1; 1498 period /= private + shared + 1; 1499 1500 smax = max(smax, period); 1501 } 1502 1503 return max(smin, smax); 1504 } 1505 1506 static void account_numa_enqueue(struct rq *rq, struct task_struct *p) 1507 { 1508 rq->nr_numa_running += (p->numa_preferred_nid != NUMA_NO_NODE); 1509 rq->nr_preferred_running += (p->numa_preferred_nid == task_node(p)); 1510 } 1511 1512 static void account_numa_dequeue(struct rq *rq, struct task_struct *p) 1513 { 1514 rq->nr_numa_running -= (p->numa_preferred_nid != NUMA_NO_NODE); 1515 rq->nr_preferred_running -= (p->numa_preferred_nid == task_node(p)); 1516 } 1517 1518 /* Shared or private faults. */ 1519 #define NR_NUMA_HINT_FAULT_TYPES 2 1520 1521 /* Memory and CPU locality */ 1522 #define NR_NUMA_HINT_FAULT_STATS (NR_NUMA_HINT_FAULT_TYPES * 2) 1523 1524 /* Averaged statistics, and temporary buffers. */ 1525 #define NR_NUMA_HINT_FAULT_BUCKETS (NR_NUMA_HINT_FAULT_STATS * 2) 1526 1527 pid_t task_numa_group_id(struct task_struct *p) 1528 { 1529 struct numa_group *ng; 1530 pid_t gid = 0; 1531 1532 rcu_read_lock(); 1533 ng = rcu_dereference(p->numa_group); 1534 if (ng) 1535 gid = ng->gid; 1536 rcu_read_unlock(); 1537 1538 return gid; 1539 } 1540 1541 /* 1542 * The averaged statistics, shared & private, memory & CPU, 1543 * occupy the first half of the array. The second half of the 1544 * array is for current counters, which are averaged into the 1545 * first set by task_numa_placement. 1546 */ 1547 static inline int task_faults_idx(enum numa_faults_stats s, int nid, int priv) 1548 { 1549 return NR_NUMA_HINT_FAULT_TYPES * (s * nr_node_ids + nid) + priv; 1550 } 1551 1552 static inline unsigned long task_faults(struct task_struct *p, int nid) 1553 { 1554 if (!p->numa_faults) 1555 return 0; 1556 1557 return p->numa_faults[task_faults_idx(NUMA_MEM, nid, 0)] + 1558 p->numa_faults[task_faults_idx(NUMA_MEM, nid, 1)]; 1559 } 1560 1561 static inline unsigned long group_faults(struct task_struct *p, int nid) 1562 { 1563 struct numa_group *ng = deref_task_numa_group(p); 1564 1565 if (!ng) 1566 return 0; 1567 1568 return ng->faults[task_faults_idx(NUMA_MEM, nid, 0)] + 1569 ng->faults[task_faults_idx(NUMA_MEM, nid, 1)]; 1570 } 1571 1572 static inline unsigned long group_faults_cpu(struct numa_group *group, int nid) 1573 { 1574 return group->faults[task_faults_idx(NUMA_CPU, nid, 0)] + 1575 group->faults[task_faults_idx(NUMA_CPU, nid, 1)]; 1576 } 1577 1578 static inline unsigned long group_faults_priv(struct numa_group *ng) 1579 { 1580 unsigned long faults = 0; 1581 int node; 1582 1583 for_each_online_node(node) { 1584 faults += ng->faults[task_faults_idx(NUMA_MEM, node, 1)]; 1585 } 1586 1587 return faults; 1588 } 1589 1590 static inline unsigned long group_faults_shared(struct numa_group *ng) 1591 { 1592 unsigned long faults = 0; 1593 int node; 1594 1595 for_each_online_node(node) { 1596 faults += ng->faults[task_faults_idx(NUMA_MEM, node, 0)]; 1597 } 1598 1599 return faults; 1600 } 1601 1602 /* 1603 * A node triggering more than 1/3 as many NUMA faults as the maximum is 1604 * considered part of a numa group's pseudo-interleaving set. Migrations 1605 * between these nodes are slowed down, to allow things to settle down. 1606 */ 1607 #define ACTIVE_NODE_FRACTION 3 1608 1609 static bool numa_is_active_node(int nid, struct numa_group *ng) 1610 { 1611 return group_faults_cpu(ng, nid) * ACTIVE_NODE_FRACTION > ng->max_faults_cpu; 1612 } 1613 1614 /* Handle placement on systems where not all nodes are directly connected. */ 1615 static unsigned long score_nearby_nodes(struct task_struct *p, int nid, 1616 int lim_dist, bool task) 1617 { 1618 unsigned long score = 0; 1619 int node, max_dist; 1620 1621 /* 1622 * All nodes are directly connected, and the same distance 1623 * from each other. No need for fancy placement algorithms. 1624 */ 1625 if (sched_numa_topology_type == NUMA_DIRECT) 1626 return 0; 1627 1628 /* sched_max_numa_distance may be changed in parallel. */ 1629 max_dist = READ_ONCE(sched_max_numa_distance); 1630 /* 1631 * This code is called for each node, introducing N^2 complexity, 1632 * which should be ok given the number of nodes rarely exceeds 8. 1633 */ 1634 for_each_online_node(node) { 1635 unsigned long faults; 1636 int dist = node_distance(nid, node); 1637 1638 /* 1639 * The furthest away nodes in the system are not interesting 1640 * for placement; nid was already counted. 1641 */ 1642 if (dist >= max_dist || node == nid) 1643 continue; 1644 1645 /* 1646 * On systems with a backplane NUMA topology, compare groups 1647 * of nodes, and move tasks towards the group with the most 1648 * memory accesses. When comparing two nodes at distance 1649 * "hoplimit", only nodes closer by than "hoplimit" are part 1650 * of each group. Skip other nodes. 1651 */ 1652 if (sched_numa_topology_type == NUMA_BACKPLANE && dist >= lim_dist) 1653 continue; 1654 1655 /* Add up the faults from nearby nodes. */ 1656 if (task) 1657 faults = task_faults(p, node); 1658 else 1659 faults = group_faults(p, node); 1660 1661 /* 1662 * On systems with a glueless mesh NUMA topology, there are 1663 * no fixed "groups of nodes". Instead, nodes that are not 1664 * directly connected bounce traffic through intermediate 1665 * nodes; a numa_group can occupy any set of nodes. 1666 * The further away a node is, the less the faults count. 1667 * This seems to result in good task placement. 1668 */ 1669 if (sched_numa_topology_type == NUMA_GLUELESS_MESH) { 1670 faults *= (max_dist - dist); 1671 faults /= (max_dist - LOCAL_DISTANCE); 1672 } 1673 1674 score += faults; 1675 } 1676 1677 return score; 1678 } 1679 1680 /* 1681 * These return the fraction of accesses done by a particular task, or 1682 * task group, on a particular numa node. The group weight is given a 1683 * larger multiplier, in order to group tasks together that are almost 1684 * evenly spread out between numa nodes. 1685 */ 1686 static inline unsigned long task_weight(struct task_struct *p, int nid, 1687 int dist) 1688 { 1689 unsigned long faults, total_faults; 1690 1691 if (!p->numa_faults) 1692 return 0; 1693 1694 total_faults = p->total_numa_faults; 1695 1696 if (!total_faults) 1697 return 0; 1698 1699 faults = task_faults(p, nid); 1700 faults += score_nearby_nodes(p, nid, dist, true); 1701 1702 return 1000 * faults / total_faults; 1703 } 1704 1705 static inline unsigned long group_weight(struct task_struct *p, int nid, 1706 int dist) 1707 { 1708 struct numa_group *ng = deref_task_numa_group(p); 1709 unsigned long faults, total_faults; 1710 1711 if (!ng) 1712 return 0; 1713 1714 total_faults = ng->total_faults; 1715 1716 if (!total_faults) 1717 return 0; 1718 1719 faults = group_faults(p, nid); 1720 faults += score_nearby_nodes(p, nid, dist, false); 1721 1722 return 1000 * faults / total_faults; 1723 } 1724 1725 /* 1726 * If memory tiering mode is enabled, cpupid of slow memory page is 1727 * used to record scan time instead of CPU and PID. When tiering mode 1728 * is disabled at run time, the scan time (in cpupid) will be 1729 * interpreted as CPU and PID. So CPU needs to be checked to avoid to 1730 * access out of array bound. 1731 */ 1732 static inline bool cpupid_valid(int cpupid) 1733 { 1734 return cpupid_to_cpu(cpupid) < nr_cpu_ids; 1735 } 1736 1737 /* 1738 * For memory tiering mode, if there are enough free pages (more than 1739 * enough watermark defined here) in fast memory node, to take full 1740 * advantage of fast memory capacity, all recently accessed slow 1741 * memory pages will be migrated to fast memory node without 1742 * considering hot threshold. 1743 */ 1744 static bool pgdat_free_space_enough(struct pglist_data *pgdat) 1745 { 1746 int z; 1747 unsigned long enough_wmark; 1748 1749 enough_wmark = max(1UL * 1024 * 1024 * 1024 >> PAGE_SHIFT, 1750 pgdat->node_present_pages >> 4); 1751 for (z = pgdat->nr_zones - 1; z >= 0; z--) { 1752 struct zone *zone = pgdat->node_zones + z; 1753 1754 if (!populated_zone(zone)) 1755 continue; 1756 1757 if (zone_watermark_ok(zone, 0, 1758 wmark_pages(zone, WMARK_PROMO) + enough_wmark, 1759 ZONE_MOVABLE, 0)) 1760 return true; 1761 } 1762 return false; 1763 } 1764 1765 /* 1766 * For memory tiering mode, when page tables are scanned, the scan 1767 * time will be recorded in struct page in addition to make page 1768 * PROT_NONE for slow memory page. So when the page is accessed, in 1769 * hint page fault handler, the hint page fault latency is calculated 1770 * via, 1771 * 1772 * hint page fault latency = hint page fault time - scan time 1773 * 1774 * The smaller the hint page fault latency, the higher the possibility 1775 * for the page to be hot. 1776 */ 1777 static int numa_hint_fault_latency(struct page *page) 1778 { 1779 int last_time, time; 1780 1781 time = jiffies_to_msecs(jiffies); 1782 last_time = xchg_page_access_time(page, time); 1783 1784 return (time - last_time) & PAGE_ACCESS_TIME_MASK; 1785 } 1786 1787 /* 1788 * For memory tiering mode, too high promotion/demotion throughput may 1789 * hurt application latency. So we provide a mechanism to rate limit 1790 * the number of pages that are tried to be promoted. 1791 */ 1792 static bool numa_promotion_rate_limit(struct pglist_data *pgdat, 1793 unsigned long rate_limit, int nr) 1794 { 1795 unsigned long nr_cand; 1796 unsigned int now, start; 1797 1798 now = jiffies_to_msecs(jiffies); 1799 mod_node_page_state(pgdat, PGPROMOTE_CANDIDATE, nr); 1800 nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE); 1801 start = pgdat->nbp_rl_start; 1802 if (now - start > MSEC_PER_SEC && 1803 cmpxchg(&pgdat->nbp_rl_start, start, now) == start) 1804 pgdat->nbp_rl_nr_cand = nr_cand; 1805 if (nr_cand - pgdat->nbp_rl_nr_cand >= rate_limit) 1806 return true; 1807 return false; 1808 } 1809 1810 #define NUMA_MIGRATION_ADJUST_STEPS 16 1811 1812 static void numa_promotion_adjust_threshold(struct pglist_data *pgdat, 1813 unsigned long rate_limit, 1814 unsigned int ref_th) 1815 { 1816 unsigned int now, start, th_period, unit_th, th; 1817 unsigned long nr_cand, ref_cand, diff_cand; 1818 1819 now = jiffies_to_msecs(jiffies); 1820 th_period = sysctl_numa_balancing_scan_period_max; 1821 start = pgdat->nbp_th_start; 1822 if (now - start > th_period && 1823 cmpxchg(&pgdat->nbp_th_start, start, now) == start) { 1824 ref_cand = rate_limit * 1825 sysctl_numa_balancing_scan_period_max / MSEC_PER_SEC; 1826 nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE); 1827 diff_cand = nr_cand - pgdat->nbp_th_nr_cand; 1828 unit_th = ref_th * 2 / NUMA_MIGRATION_ADJUST_STEPS; 1829 th = pgdat->nbp_threshold ? : ref_th; 1830 if (diff_cand > ref_cand * 11 / 10) 1831 th = max(th - unit_th, unit_th); 1832 else if (diff_cand < ref_cand * 9 / 10) 1833 th = min(th + unit_th, ref_th * 2); 1834 pgdat->nbp_th_nr_cand = nr_cand; 1835 pgdat->nbp_threshold = th; 1836 } 1837 } 1838 1839 bool should_numa_migrate_memory(struct task_struct *p, struct page * page, 1840 int src_nid, int dst_cpu) 1841 { 1842 struct numa_group *ng = deref_curr_numa_group(p); 1843 int dst_nid = cpu_to_node(dst_cpu); 1844 int last_cpupid, this_cpupid; 1845 1846 /* 1847 * The pages in slow memory node should be migrated according 1848 * to hot/cold instead of private/shared. 1849 */ 1850 if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING && 1851 !node_is_toptier(src_nid)) { 1852 struct pglist_data *pgdat; 1853 unsigned long rate_limit; 1854 unsigned int latency, th, def_th; 1855 1856 pgdat = NODE_DATA(dst_nid); 1857 if (pgdat_free_space_enough(pgdat)) { 1858 /* workload changed, reset hot threshold */ 1859 pgdat->nbp_threshold = 0; 1860 return true; 1861 } 1862 1863 def_th = sysctl_numa_balancing_hot_threshold; 1864 rate_limit = sysctl_numa_balancing_promote_rate_limit << \ 1865 (20 - PAGE_SHIFT); 1866 numa_promotion_adjust_threshold(pgdat, rate_limit, def_th); 1867 1868 th = pgdat->nbp_threshold ? : def_th; 1869 latency = numa_hint_fault_latency(page); 1870 if (latency >= th) 1871 return false; 1872 1873 return !numa_promotion_rate_limit(pgdat, rate_limit, 1874 thp_nr_pages(page)); 1875 } 1876 1877 this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid); 1878 last_cpupid = page_cpupid_xchg_last(page, this_cpupid); 1879 1880 if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) && 1881 !node_is_toptier(src_nid) && !cpupid_valid(last_cpupid)) 1882 return false; 1883 1884 /* 1885 * Allow first faults or private faults to migrate immediately early in 1886 * the lifetime of a task. The magic number 4 is based on waiting for 1887 * two full passes of the "multi-stage node selection" test that is 1888 * executed below. 1889 */ 1890 if ((p->numa_preferred_nid == NUMA_NO_NODE || p->numa_scan_seq <= 4) && 1891 (cpupid_pid_unset(last_cpupid) || cpupid_match_pid(p, last_cpupid))) 1892 return true; 1893 1894 /* 1895 * Multi-stage node selection is used in conjunction with a periodic 1896 * migration fault to build a temporal task<->page relation. By using 1897 * a two-stage filter we remove short/unlikely relations. 1898 * 1899 * Using P(p) ~ n_p / n_t as per frequentist probability, we can equate 1900 * a task's usage of a particular page (n_p) per total usage of this 1901 * page (n_t) (in a given time-span) to a probability. 1902 * 1903 * Our periodic faults will sample this probability and getting the 1904 * same result twice in a row, given these samples are fully 1905 * independent, is then given by P(n)^2, provided our sample period 1906 * is sufficiently short compared to the usage pattern. 1907 * 1908 * This quadric squishes small probabilities, making it less likely we 1909 * act on an unlikely task<->page relation. 1910 */ 1911 if (!cpupid_pid_unset(last_cpupid) && 1912 cpupid_to_nid(last_cpupid) != dst_nid) 1913 return false; 1914 1915 /* Always allow migrate on private faults */ 1916 if (cpupid_match_pid(p, last_cpupid)) 1917 return true; 1918 1919 /* A shared fault, but p->numa_group has not been set up yet. */ 1920 if (!ng) 1921 return true; 1922 1923 /* 1924 * Destination node is much more heavily used than the source 1925 * node? Allow migration. 1926 */ 1927 if (group_faults_cpu(ng, dst_nid) > group_faults_cpu(ng, src_nid) * 1928 ACTIVE_NODE_FRACTION) 1929 return true; 1930 1931 /* 1932 * Distribute memory according to CPU & memory use on each node, 1933 * with 3/4 hysteresis to avoid unnecessary memory migrations: 1934 * 1935 * faults_cpu(dst) 3 faults_cpu(src) 1936 * --------------- * - > --------------- 1937 * faults_mem(dst) 4 faults_mem(src) 1938 */ 1939 return group_faults_cpu(ng, dst_nid) * group_faults(p, src_nid) * 3 > 1940 group_faults_cpu(ng, src_nid) * group_faults(p, dst_nid) * 4; 1941 } 1942 1943 /* 1944 * 'numa_type' describes the node at the moment of load balancing. 1945 */ 1946 enum numa_type { 1947 /* The node has spare capacity that can be used to run more tasks. */ 1948 node_has_spare = 0, 1949 /* 1950 * The node is fully used and the tasks don't compete for more CPU 1951 * cycles. Nevertheless, some tasks might wait before running. 1952 */ 1953 node_fully_busy, 1954 /* 1955 * The node is overloaded and can't provide expected CPU cycles to all 1956 * tasks. 1957 */ 1958 node_overloaded 1959 }; 1960 1961 /* Cached statistics for all CPUs within a node */ 1962 struct numa_stats { 1963 unsigned long load; 1964 unsigned long runnable; 1965 unsigned long util; 1966 /* Total compute capacity of CPUs on a node */ 1967 unsigned long compute_capacity; 1968 unsigned int nr_running; 1969 unsigned int weight; 1970 enum numa_type node_type; 1971 int idle_cpu; 1972 }; 1973 1974 struct task_numa_env { 1975 struct task_struct *p; 1976 1977 int src_cpu, src_nid; 1978 int dst_cpu, dst_nid; 1979 int imb_numa_nr; 1980 1981 struct numa_stats src_stats, dst_stats; 1982 1983 int imbalance_pct; 1984 int dist; 1985 1986 struct task_struct *best_task; 1987 long best_imp; 1988 int best_cpu; 1989 }; 1990 1991 static unsigned long cpu_load(struct rq *rq); 1992 static unsigned long cpu_runnable(struct rq *rq); 1993 1994 static inline enum 1995 numa_type numa_classify(unsigned int imbalance_pct, 1996 struct numa_stats *ns) 1997 { 1998 if ((ns->nr_running > ns->weight) && 1999 (((ns->compute_capacity * 100) < (ns->util * imbalance_pct)) || 2000 ((ns->compute_capacity * imbalance_pct) < (ns->runnable * 100)))) 2001 return node_overloaded; 2002 2003 if ((ns->nr_running < ns->weight) || 2004 (((ns->compute_capacity * 100) > (ns->util * imbalance_pct)) && 2005 ((ns->compute_capacity * imbalance_pct) > (ns->runnable * 100)))) 2006 return node_has_spare; 2007 2008 return node_fully_busy; 2009 } 2010 2011 #ifdef CONFIG_SCHED_SMT 2012 /* Forward declarations of select_idle_sibling helpers */ 2013 static inline bool test_idle_cores(int cpu); 2014 static inline int numa_idle_core(int idle_core, int cpu) 2015 { 2016 if (!static_branch_likely(&sched_smt_present) || 2017 idle_core >= 0 || !test_idle_cores(cpu)) 2018 return idle_core; 2019 2020 /* 2021 * Prefer cores instead of packing HT siblings 2022 * and triggering future load balancing. 2023 */ 2024 if (is_core_idle(cpu)) 2025 idle_core = cpu; 2026 2027 return idle_core; 2028 } 2029 #else 2030 static inline int numa_idle_core(int idle_core, int cpu) 2031 { 2032 return idle_core; 2033 } 2034 #endif 2035 2036 /* 2037 * Gather all necessary information to make NUMA balancing placement 2038 * decisions that are compatible with standard load balancer. This 2039 * borrows code and logic from update_sg_lb_stats but sharing a 2040 * common implementation is impractical. 2041 */ 2042 static void update_numa_stats(struct task_numa_env *env, 2043 struct numa_stats *ns, int nid, 2044 bool find_idle) 2045 { 2046 int cpu, idle_core = -1; 2047 2048 memset(ns, 0, sizeof(*ns)); 2049 ns->idle_cpu = -1; 2050 2051 rcu_read_lock(); 2052 for_each_cpu(cpu, cpumask_of_node(nid)) { 2053 struct rq *rq = cpu_rq(cpu); 2054 2055 ns->load += cpu_load(rq); 2056 ns->runnable += cpu_runnable(rq); 2057 ns->util += cpu_util_cfs(cpu); 2058 ns->nr_running += rq->cfs.h_nr_running; 2059 ns->compute_capacity += capacity_of(cpu); 2060 2061 if (find_idle && idle_core < 0 && !rq->nr_running && idle_cpu(cpu)) { 2062 if (READ_ONCE(rq->numa_migrate_on) || 2063 !cpumask_test_cpu(cpu, env->p->cpus_ptr)) 2064 continue; 2065 2066 if (ns->idle_cpu == -1) 2067 ns->idle_cpu = cpu; 2068 2069 idle_core = numa_idle_core(idle_core, cpu); 2070 } 2071 } 2072 rcu_read_unlock(); 2073 2074 ns->weight = cpumask_weight(cpumask_of_node(nid)); 2075 2076 ns->node_type = numa_classify(env->imbalance_pct, ns); 2077 2078 if (idle_core >= 0) 2079 ns->idle_cpu = idle_core; 2080 } 2081 2082 static void task_numa_assign(struct task_numa_env *env, 2083 struct task_struct *p, long imp) 2084 { 2085 struct rq *rq = cpu_rq(env->dst_cpu); 2086 2087 /* Check if run-queue part of active NUMA balance. */ 2088 if (env->best_cpu != env->dst_cpu && xchg(&rq->numa_migrate_on, 1)) { 2089 int cpu; 2090 int start = env->dst_cpu; 2091 2092 /* Find alternative idle CPU. */ 2093 for_each_cpu_wrap(cpu, cpumask_of_node(env->dst_nid), start + 1) { 2094 if (cpu == env->best_cpu || !idle_cpu(cpu) || 2095 !cpumask_test_cpu(cpu, env->p->cpus_ptr)) { 2096 continue; 2097 } 2098 2099 env->dst_cpu = cpu; 2100 rq = cpu_rq(env->dst_cpu); 2101 if (!xchg(&rq->numa_migrate_on, 1)) 2102 goto assign; 2103 } 2104 2105 /* Failed to find an alternative idle CPU */ 2106 return; 2107 } 2108 2109 assign: 2110 /* 2111 * Clear previous best_cpu/rq numa-migrate flag, since task now 2112 * found a better CPU to move/swap. 2113 */ 2114 if (env->best_cpu != -1 && env->best_cpu != env->dst_cpu) { 2115 rq = cpu_rq(env->best_cpu); 2116 WRITE_ONCE(rq->numa_migrate_on, 0); 2117 } 2118 2119 if (env->best_task) 2120 put_task_struct(env->best_task); 2121 if (p) 2122 get_task_struct(p); 2123 2124 env->best_task = p; 2125 env->best_imp = imp; 2126 env->best_cpu = env->dst_cpu; 2127 } 2128 2129 static bool load_too_imbalanced(long src_load, long dst_load, 2130 struct task_numa_env *env) 2131 { 2132 long imb, old_imb; 2133 long orig_src_load, orig_dst_load; 2134 long src_capacity, dst_capacity; 2135 2136 /* 2137 * The load is corrected for the CPU capacity available on each node. 2138 * 2139 * src_load dst_load 2140 * ------------ vs --------- 2141 * src_capacity dst_capacity 2142 */ 2143 src_capacity = env->src_stats.compute_capacity; 2144 dst_capacity = env->dst_stats.compute_capacity; 2145 2146 imb = abs(dst_load * src_capacity - src_load * dst_capacity); 2147 2148 orig_src_load = env->src_stats.load; 2149 orig_dst_load = env->dst_stats.load; 2150 2151 old_imb = abs(orig_dst_load * src_capacity - orig_src_load * dst_capacity); 2152 2153 /* Would this change make things worse? */ 2154 return (imb > old_imb); 2155 } 2156 2157 /* 2158 * Maximum NUMA importance can be 1998 (2*999); 2159 * SMALLIMP @ 30 would be close to 1998/64. 2160 * Used to deter task migration. 2161 */ 2162 #define SMALLIMP 30 2163 2164 /* 2165 * This checks if the overall compute and NUMA accesses of the system would 2166 * be improved if the source tasks was migrated to the target dst_cpu taking 2167 * into account that it might be best if task running on the dst_cpu should 2168 * be exchanged with the source task 2169 */ 2170 static bool task_numa_compare(struct task_numa_env *env, 2171 long taskimp, long groupimp, bool maymove) 2172 { 2173 struct numa_group *cur_ng, *p_ng = deref_curr_numa_group(env->p); 2174 struct rq *dst_rq = cpu_rq(env->dst_cpu); 2175 long imp = p_ng ? groupimp : taskimp; 2176 struct task_struct *cur; 2177 long src_load, dst_load; 2178 int dist = env->dist; 2179 long moveimp = imp; 2180 long load; 2181 bool stopsearch = false; 2182 2183 if (READ_ONCE(dst_rq->numa_migrate_on)) 2184 return false; 2185 2186 rcu_read_lock(); 2187 cur = rcu_dereference(dst_rq->curr); 2188 if (cur && ((cur->flags & PF_EXITING) || is_idle_task(cur))) 2189 cur = NULL; 2190 2191 /* 2192 * Because we have preemption enabled we can get migrated around and 2193 * end try selecting ourselves (current == env->p) as a swap candidate. 2194 */ 2195 if (cur == env->p) { 2196 stopsearch = true; 2197 goto unlock; 2198 } 2199 2200 if (!cur) { 2201 if (maymove && moveimp >= env->best_imp) 2202 goto assign; 2203 else 2204 goto unlock; 2205 } 2206 2207 /* Skip this swap candidate if cannot move to the source cpu. */ 2208 if (!cpumask_test_cpu(env->src_cpu, cur->cpus_ptr)) 2209 goto unlock; 2210 2211 /* 2212 * Skip this swap candidate if it is not moving to its preferred 2213 * node and the best task is. 2214 */ 2215 if (env->best_task && 2216 env->best_task->numa_preferred_nid == env->src_nid && 2217 cur->numa_preferred_nid != env->src_nid) { 2218 goto unlock; 2219 } 2220 2221 /* 2222 * "imp" is the fault differential for the source task between the 2223 * source and destination node. Calculate the total differential for 2224 * the source task and potential destination task. The more negative 2225 * the value is, the more remote accesses that would be expected to 2226 * be incurred if the tasks were swapped. 2227 * 2228 * If dst and source tasks are in the same NUMA group, or not 2229 * in any group then look only at task weights. 2230 */ 2231 cur_ng = rcu_dereference(cur->numa_group); 2232 if (cur_ng == p_ng) { 2233 /* 2234 * Do not swap within a group or between tasks that have 2235 * no group if there is spare capacity. Swapping does 2236 * not address the load imbalance and helps one task at 2237 * the cost of punishing another. 2238 */ 2239 if (env->dst_stats.node_type == node_has_spare) 2240 goto unlock; 2241 2242 imp = taskimp + task_weight(cur, env->src_nid, dist) - 2243 task_weight(cur, env->dst_nid, dist); 2244 /* 2245 * Add some hysteresis to prevent swapping the 2246 * tasks within a group over tiny differences. 2247 */ 2248 if (cur_ng) 2249 imp -= imp / 16; 2250 } else { 2251 /* 2252 * Compare the group weights. If a task is all by itself 2253 * (not part of a group), use the task weight instead. 2254 */ 2255 if (cur_ng && p_ng) 2256 imp += group_weight(cur, env->src_nid, dist) - 2257 group_weight(cur, env->dst_nid, dist); 2258 else 2259 imp += task_weight(cur, env->src_nid, dist) - 2260 task_weight(cur, env->dst_nid, dist); 2261 } 2262 2263 /* Discourage picking a task already on its preferred node */ 2264 if (cur->numa_preferred_nid == env->dst_nid) 2265 imp -= imp / 16; 2266 2267 /* 2268 * Encourage picking a task that moves to its preferred node. 2269 * This potentially makes imp larger than it's maximum of 2270 * 1998 (see SMALLIMP and task_weight for why) but in this 2271 * case, it does not matter. 2272 */ 2273 if (cur->numa_preferred_nid == env->src_nid) 2274 imp += imp / 8; 2275 2276 if (maymove && moveimp > imp && moveimp > env->best_imp) { 2277 imp = moveimp; 2278 cur = NULL; 2279 goto assign; 2280 } 2281 2282 /* 2283 * Prefer swapping with a task moving to its preferred node over a 2284 * task that is not. 2285 */ 2286 if (env->best_task && cur->numa_preferred_nid == env->src_nid && 2287 env->best_task->numa_preferred_nid != env->src_nid) { 2288 goto assign; 2289 } 2290 2291 /* 2292 * If the NUMA importance is less than SMALLIMP, 2293 * task migration might only result in ping pong 2294 * of tasks and also hurt performance due to cache 2295 * misses. 2296 */ 2297 if (imp < SMALLIMP || imp <= env->best_imp + SMALLIMP / 2) 2298 goto unlock; 2299 2300 /* 2301 * In the overloaded case, try and keep the load balanced. 2302 */ 2303 load = task_h_load(env->p) - task_h_load(cur); 2304 if (!load) 2305 goto assign; 2306 2307 dst_load = env->dst_stats.load + load; 2308 src_load = env->src_stats.load - load; 2309 2310 if (load_too_imbalanced(src_load, dst_load, env)) 2311 goto unlock; 2312 2313 assign: 2314 /* Evaluate an idle CPU for a task numa move. */ 2315 if (!cur) { 2316 int cpu = env->dst_stats.idle_cpu; 2317 2318 /* Nothing cached so current CPU went idle since the search. */ 2319 if (cpu < 0) 2320 cpu = env->dst_cpu; 2321 2322 /* 2323 * If the CPU is no longer truly idle and the previous best CPU 2324 * is, keep using it. 2325 */ 2326 if (!idle_cpu(cpu) && env->best_cpu >= 0 && 2327 idle_cpu(env->best_cpu)) { 2328 cpu = env->best_cpu; 2329 } 2330 2331 env->dst_cpu = cpu; 2332 } 2333 2334 task_numa_assign(env, cur, imp); 2335 2336 /* 2337 * If a move to idle is allowed because there is capacity or load 2338 * balance improves then stop the search. While a better swap 2339 * candidate may exist, a search is not free. 2340 */ 2341 if (maymove && !cur && env->best_cpu >= 0 && idle_cpu(env->best_cpu)) 2342 stopsearch = true; 2343 2344 /* 2345 * If a swap candidate must be identified and the current best task 2346 * moves its preferred node then stop the search. 2347 */ 2348 if (!maymove && env->best_task && 2349 env->best_task->numa_preferred_nid == env->src_nid) { 2350 stopsearch = true; 2351 } 2352 unlock: 2353 rcu_read_unlock(); 2354 2355 return stopsearch; 2356 } 2357 2358 static void task_numa_find_cpu(struct task_numa_env *env, 2359 long taskimp, long groupimp) 2360 { 2361 bool maymove = false; 2362 int cpu; 2363 2364 /* 2365 * If dst node has spare capacity, then check if there is an 2366 * imbalance that would be overruled by the load balancer. 2367 */ 2368 if (env->dst_stats.node_type == node_has_spare) { 2369 unsigned int imbalance; 2370 int src_running, dst_running; 2371 2372 /* 2373 * Would movement cause an imbalance? Note that if src has 2374 * more running tasks that the imbalance is ignored as the 2375 * move improves the imbalance from the perspective of the 2376 * CPU load balancer. 2377 * */ 2378 src_running = env->src_stats.nr_running - 1; 2379 dst_running = env->dst_stats.nr_running + 1; 2380 imbalance = max(0, dst_running - src_running); 2381 imbalance = adjust_numa_imbalance(imbalance, dst_running, 2382 env->imb_numa_nr); 2383 2384 /* Use idle CPU if there is no imbalance */ 2385 if (!imbalance) { 2386 maymove = true; 2387 if (env->dst_stats.idle_cpu >= 0) { 2388 env->dst_cpu = env->dst_stats.idle_cpu; 2389 task_numa_assign(env, NULL, 0); 2390 return; 2391 } 2392 } 2393 } else { 2394 long src_load, dst_load, load; 2395 /* 2396 * If the improvement from just moving env->p direction is better 2397 * than swapping tasks around, check if a move is possible. 2398 */ 2399 load = task_h_load(env->p); 2400 dst_load = env->dst_stats.load + load; 2401 src_load = env->src_stats.load - load; 2402 maymove = !load_too_imbalanced(src_load, dst_load, env); 2403 } 2404 2405 for_each_cpu(cpu, cpumask_of_node(env->dst_nid)) { 2406 /* Skip this CPU if the source task cannot migrate */ 2407 if (!cpumask_test_cpu(cpu, env->p->cpus_ptr)) 2408 continue; 2409 2410 env->dst_cpu = cpu; 2411 if (task_numa_compare(env, taskimp, groupimp, maymove)) 2412 break; 2413 } 2414 } 2415 2416 static int task_numa_migrate(struct task_struct *p) 2417 { 2418 struct task_numa_env env = { 2419 .p = p, 2420 2421 .src_cpu = task_cpu(p), 2422 .src_nid = task_node(p), 2423 2424 .imbalance_pct = 112, 2425 2426 .best_task = NULL, 2427 .best_imp = 0, 2428 .best_cpu = -1, 2429 }; 2430 unsigned long taskweight, groupweight; 2431 struct sched_domain *sd; 2432 long taskimp, groupimp; 2433 struct numa_group *ng; 2434 struct rq *best_rq; 2435 int nid, ret, dist; 2436 2437 /* 2438 * Pick the lowest SD_NUMA domain, as that would have the smallest 2439 * imbalance and would be the first to start moving tasks about. 2440 * 2441 * And we want to avoid any moving of tasks about, as that would create 2442 * random movement of tasks -- counter the numa conditions we're trying 2443 * to satisfy here. 2444 */ 2445 rcu_read_lock(); 2446 sd = rcu_dereference(per_cpu(sd_numa, env.src_cpu)); 2447 if (sd) { 2448 env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2; 2449 env.imb_numa_nr = sd->imb_numa_nr; 2450 } 2451 rcu_read_unlock(); 2452 2453 /* 2454 * Cpusets can break the scheduler domain tree into smaller 2455 * balance domains, some of which do not cross NUMA boundaries. 2456 * Tasks that are "trapped" in such domains cannot be migrated 2457 * elsewhere, so there is no point in (re)trying. 2458 */ 2459 if (unlikely(!sd)) { 2460 sched_setnuma(p, task_node(p)); 2461 return -EINVAL; 2462 } 2463 2464 env.dst_nid = p->numa_preferred_nid; 2465 dist = env.dist = node_distance(env.src_nid, env.dst_nid); 2466 taskweight = task_weight(p, env.src_nid, dist); 2467 groupweight = group_weight(p, env.src_nid, dist); 2468 update_numa_stats(&env, &env.src_stats, env.src_nid, false); 2469 taskimp = task_weight(p, env.dst_nid, dist) - taskweight; 2470 groupimp = group_weight(p, env.dst_nid, dist) - groupweight; 2471 update_numa_stats(&env, &env.dst_stats, env.dst_nid, true); 2472 2473 /* Try to find a spot on the preferred nid. */ 2474 task_numa_find_cpu(&env, taskimp, groupimp); 2475 2476 /* 2477 * Look at other nodes in these cases: 2478 * - there is no space available on the preferred_nid 2479 * - the task is part of a numa_group that is interleaved across 2480 * multiple NUMA nodes; in order to better consolidate the group, 2481 * we need to check other locations. 2482 */ 2483 ng = deref_curr_numa_group(p); 2484 if (env.best_cpu == -1 || (ng && ng->active_nodes > 1)) { 2485 for_each_node_state(nid, N_CPU) { 2486 if (nid == env.src_nid || nid == p->numa_preferred_nid) 2487 continue; 2488 2489 dist = node_distance(env.src_nid, env.dst_nid); 2490 if (sched_numa_topology_type == NUMA_BACKPLANE && 2491 dist != env.dist) { 2492 taskweight = task_weight(p, env.src_nid, dist); 2493 groupweight = group_weight(p, env.src_nid, dist); 2494 } 2495 2496 /* Only consider nodes where both task and groups benefit */ 2497 taskimp = task_weight(p, nid, dist) - taskweight; 2498 groupimp = group_weight(p, nid, dist) - groupweight; 2499 if (taskimp < 0 && groupimp < 0) 2500 continue; 2501 2502 env.dist = dist; 2503 env.dst_nid = nid; 2504 update_numa_stats(&env, &env.dst_stats, env.dst_nid, true); 2505 task_numa_find_cpu(&env, taskimp, groupimp); 2506 } 2507 } 2508 2509 /* 2510 * If the task is part of a workload that spans multiple NUMA nodes, 2511 * and is migrating into one of the workload's active nodes, remember 2512 * this node as the task's preferred numa node, so the workload can 2513 * settle down. 2514 * A task that migrated to a second choice node will be better off 2515 * trying for a better one later. Do not set the preferred node here. 2516 */ 2517 if (ng) { 2518 if (env.best_cpu == -1) 2519 nid = env.src_nid; 2520 else 2521 nid = cpu_to_node(env.best_cpu); 2522 2523 if (nid != p->numa_preferred_nid) 2524 sched_setnuma(p, nid); 2525 } 2526 2527 /* No better CPU than the current one was found. */ 2528 if (env.best_cpu == -1) { 2529 trace_sched_stick_numa(p, env.src_cpu, NULL, -1); 2530 return -EAGAIN; 2531 } 2532 2533 best_rq = cpu_rq(env.best_cpu); 2534 if (env.best_task == NULL) { 2535 ret = migrate_task_to(p, env.best_cpu); 2536 WRITE_ONCE(best_rq->numa_migrate_on, 0); 2537 if (ret != 0) 2538 trace_sched_stick_numa(p, env.src_cpu, NULL, env.best_cpu); 2539 return ret; 2540 } 2541 2542 ret = migrate_swap(p, env.best_task, env.best_cpu, env.src_cpu); 2543 WRITE_ONCE(best_rq->numa_migrate_on, 0); 2544 2545 if (ret != 0) 2546 trace_sched_stick_numa(p, env.src_cpu, env.best_task, env.best_cpu); 2547 put_task_struct(env.best_task); 2548 return ret; 2549 } 2550 2551 /* Attempt to migrate a task to a CPU on the preferred node. */ 2552 static void numa_migrate_preferred(struct task_struct *p) 2553 { 2554 unsigned long interval = HZ; 2555 2556 /* This task has no NUMA fault statistics yet */ 2557 if (unlikely(p->numa_preferred_nid == NUMA_NO_NODE || !p->numa_faults)) 2558 return; 2559 2560 /* Periodically retry migrating the task to the preferred node */ 2561 interval = min(interval, msecs_to_jiffies(p->numa_scan_period) / 16); 2562 p->numa_migrate_retry = jiffies + interval; 2563 2564 /* Success if task is already running on preferred CPU */ 2565 if (task_node(p) == p->numa_preferred_nid) 2566 return; 2567 2568 /* Otherwise, try migrate to a CPU on the preferred node */ 2569 task_numa_migrate(p); 2570 } 2571 2572 /* 2573 * Find out how many nodes the workload is actively running on. Do this by 2574 * tracking the nodes from which NUMA hinting faults are triggered. This can 2575 * be different from the set of nodes where the workload's memory is currently 2576 * located. 2577 */ 2578 static void numa_group_count_active_nodes(struct numa_group *numa_group) 2579 { 2580 unsigned long faults, max_faults = 0; 2581 int nid, active_nodes = 0; 2582 2583 for_each_node_state(nid, N_CPU) { 2584 faults = group_faults_cpu(numa_group, nid); 2585 if (faults > max_faults) 2586 max_faults = faults; 2587 } 2588 2589 for_each_node_state(nid, N_CPU) { 2590 faults = group_faults_cpu(numa_group, nid); 2591 if (faults * ACTIVE_NODE_FRACTION > max_faults) 2592 active_nodes++; 2593 } 2594 2595 numa_group->max_faults_cpu = max_faults; 2596 numa_group->active_nodes = active_nodes; 2597 } 2598 2599 /* 2600 * When adapting the scan rate, the period is divided into NUMA_PERIOD_SLOTS 2601 * increments. The more local the fault statistics are, the higher the scan 2602 * period will be for the next scan window. If local/(local+remote) ratio is 2603 * below NUMA_PERIOD_THRESHOLD (where range of ratio is 1..NUMA_PERIOD_SLOTS) 2604 * the scan period will decrease. Aim for 70% local accesses. 2605 */ 2606 #define NUMA_PERIOD_SLOTS 10 2607 #define NUMA_PERIOD_THRESHOLD 7 2608 2609 /* 2610 * Increase the scan period (slow down scanning) if the majority of 2611 * our memory is already on our local node, or if the majority of 2612 * the page accesses are shared with other processes. 2613 * Otherwise, decrease the scan period. 2614 */ 2615 static void update_task_scan_period(struct task_struct *p, 2616 unsigned long shared, unsigned long private) 2617 { 2618 unsigned int period_slot; 2619 int lr_ratio, ps_ratio; 2620 int diff; 2621 2622 unsigned long remote = p->numa_faults_locality[0]; 2623 unsigned long local = p->numa_faults_locality[1]; 2624 2625 /* 2626 * If there were no record hinting faults then either the task is 2627 * completely idle or all activity is in areas that are not of interest 2628 * to automatic numa balancing. Related to that, if there were failed 2629 * migration then it implies we are migrating too quickly or the local 2630 * node is overloaded. In either case, scan slower 2631 */ 2632 if (local + shared == 0 || p->numa_faults_locality[2]) { 2633 p->numa_scan_period = min(p->numa_scan_period_max, 2634 p->numa_scan_period << 1); 2635 2636 p->mm->numa_next_scan = jiffies + 2637 msecs_to_jiffies(p->numa_scan_period); 2638 2639 return; 2640 } 2641 2642 /* 2643 * Prepare to scale scan period relative to the current period. 2644 * == NUMA_PERIOD_THRESHOLD scan period stays the same 2645 * < NUMA_PERIOD_THRESHOLD scan period decreases (scan faster) 2646 * >= NUMA_PERIOD_THRESHOLD scan period increases (scan slower) 2647 */ 2648 period_slot = DIV_ROUND_UP(p->numa_scan_period, NUMA_PERIOD_SLOTS); 2649 lr_ratio = (local * NUMA_PERIOD_SLOTS) / (local + remote); 2650 ps_ratio = (private * NUMA_PERIOD_SLOTS) / (private + shared); 2651 2652 if (ps_ratio >= NUMA_PERIOD_THRESHOLD) { 2653 /* 2654 * Most memory accesses are local. There is no need to 2655 * do fast NUMA scanning, since memory is already local. 2656 */ 2657 int slot = ps_ratio - NUMA_PERIOD_THRESHOLD; 2658 if (!slot) 2659 slot = 1; 2660 diff = slot * period_slot; 2661 } else if (lr_ratio >= NUMA_PERIOD_THRESHOLD) { 2662 /* 2663 * Most memory accesses are shared with other tasks. 2664 * There is no point in continuing fast NUMA scanning, 2665 * since other tasks may just move the memory elsewhere. 2666 */ 2667 int slot = lr_ratio - NUMA_PERIOD_THRESHOLD; 2668 if (!slot) 2669 slot = 1; 2670 diff = slot * period_slot; 2671 } else { 2672 /* 2673 * Private memory faults exceed (SLOTS-THRESHOLD)/SLOTS, 2674 * yet they are not on the local NUMA node. Speed up 2675 * NUMA scanning to get the memory moved over. 2676 */ 2677 int ratio = max(lr_ratio, ps_ratio); 2678 diff = -(NUMA_PERIOD_THRESHOLD - ratio) * period_slot; 2679 } 2680 2681 p->numa_scan_period = clamp(p->numa_scan_period + diff, 2682 task_scan_min(p), task_scan_max(p)); 2683 memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); 2684 } 2685 2686 /* 2687 * Get the fraction of time the task has been running since the last 2688 * NUMA placement cycle. The scheduler keeps similar statistics, but 2689 * decays those on a 32ms period, which is orders of magnitude off 2690 * from the dozens-of-seconds NUMA balancing period. Use the scheduler 2691 * stats only if the task is so new there are no NUMA statistics yet. 2692 */ 2693 static u64 numa_get_avg_runtime(struct task_struct *p, u64 *period) 2694 { 2695 u64 runtime, delta, now; 2696 /* Use the start of this time slice to avoid calculations. */ 2697 now = p->se.exec_start; 2698 runtime = p->se.sum_exec_runtime; 2699 2700 if (p->last_task_numa_placement) { 2701 delta = runtime - p->last_sum_exec_runtime; 2702 *period = now - p->last_task_numa_placement; 2703 2704 /* Avoid time going backwards, prevent potential divide error: */ 2705 if (unlikely((s64)*period < 0)) 2706 *period = 0; 2707 } else { 2708 delta = p->se.avg.load_sum; 2709 *period = LOAD_AVG_MAX; 2710 } 2711 2712 p->last_sum_exec_runtime = runtime; 2713 p->last_task_numa_placement = now; 2714 2715 return delta; 2716 } 2717 2718 /* 2719 * Determine the preferred nid for a task in a numa_group. This needs to 2720 * be done in a way that produces consistent results with group_weight, 2721 * otherwise workloads might not converge. 2722 */ 2723 static int preferred_group_nid(struct task_struct *p, int nid) 2724 { 2725 nodemask_t nodes; 2726 int dist; 2727 2728 /* Direct connections between all NUMA nodes. */ 2729 if (sched_numa_topology_type == NUMA_DIRECT) 2730 return nid; 2731 2732 /* 2733 * On a system with glueless mesh NUMA topology, group_weight 2734 * scores nodes according to the number of NUMA hinting faults on 2735 * both the node itself, and on nearby nodes. 2736 */ 2737 if (sched_numa_topology_type == NUMA_GLUELESS_MESH) { 2738 unsigned long score, max_score = 0; 2739 int node, max_node = nid; 2740 2741 dist = sched_max_numa_distance; 2742 2743 for_each_node_state(node, N_CPU) { 2744 score = group_weight(p, node, dist); 2745 if (score > max_score) { 2746 max_score = score; 2747 max_node = node; 2748 } 2749 } 2750 return max_node; 2751 } 2752 2753 /* 2754 * Finding the preferred nid in a system with NUMA backplane 2755 * interconnect topology is more involved. The goal is to locate 2756 * tasks from numa_groups near each other in the system, and 2757 * untangle workloads from different sides of the system. This requires 2758 * searching down the hierarchy of node groups, recursively searching 2759 * inside the highest scoring group of nodes. The nodemask tricks 2760 * keep the complexity of the search down. 2761 */ 2762 nodes = node_states[N_CPU]; 2763 for (dist = sched_max_numa_distance; dist > LOCAL_DISTANCE; dist--) { 2764 unsigned long max_faults = 0; 2765 nodemask_t max_group = NODE_MASK_NONE; 2766 int a, b; 2767 2768 /* Are there nodes at this distance from each other? */ 2769 if (!find_numa_distance(dist)) 2770 continue; 2771 2772 for_each_node_mask(a, nodes) { 2773 unsigned long faults = 0; 2774 nodemask_t this_group; 2775 nodes_clear(this_group); 2776 2777 /* Sum group's NUMA faults; includes a==b case. */ 2778 for_each_node_mask(b, nodes) { 2779 if (node_distance(a, b) < dist) { 2780 faults += group_faults(p, b); 2781 node_set(b, this_group); 2782 node_clear(b, nodes); 2783 } 2784 } 2785 2786 /* Remember the top group. */ 2787 if (faults > max_faults) { 2788 max_faults = faults; 2789 max_group = this_group; 2790 /* 2791 * subtle: at the smallest distance there is 2792 * just one node left in each "group", the 2793 * winner is the preferred nid. 2794 */ 2795 nid = a; 2796 } 2797 } 2798 /* Next round, evaluate the nodes within max_group. */ 2799 if (!max_faults) 2800 break; 2801 nodes = max_group; 2802 } 2803 return nid; 2804 } 2805 2806 static void task_numa_placement(struct task_struct *p) 2807 { 2808 int seq, nid, max_nid = NUMA_NO_NODE; 2809 unsigned long max_faults = 0; 2810 unsigned long fault_types[2] = { 0, 0 }; 2811 unsigned long total_faults; 2812 u64 runtime, period; 2813 spinlock_t *group_lock = NULL; 2814 struct numa_group *ng; 2815 2816 /* 2817 * The p->mm->numa_scan_seq field gets updated without 2818 * exclusive access. Use READ_ONCE() here to ensure 2819 * that the field is read in a single access: 2820 */ 2821 seq = READ_ONCE(p->mm->numa_scan_seq); 2822 if (p->numa_scan_seq == seq) 2823 return; 2824 p->numa_scan_seq = seq; 2825 p->numa_scan_period_max = task_scan_max(p); 2826 2827 total_faults = p->numa_faults_locality[0] + 2828 p->numa_faults_locality[1]; 2829 runtime = numa_get_avg_runtime(p, &period); 2830 2831 /* If the task is part of a group prevent parallel updates to group stats */ 2832 ng = deref_curr_numa_group(p); 2833 if (ng) { 2834 group_lock = &ng->lock; 2835 spin_lock_irq(group_lock); 2836 } 2837 2838 /* Find the node with the highest number of faults */ 2839 for_each_online_node(nid) { 2840 /* Keep track of the offsets in numa_faults array */ 2841 int mem_idx, membuf_idx, cpu_idx, cpubuf_idx; 2842 unsigned long faults = 0, group_faults = 0; 2843 int priv; 2844 2845 for (priv = 0; priv < NR_NUMA_HINT_FAULT_TYPES; priv++) { 2846 long diff, f_diff, f_weight; 2847 2848 mem_idx = task_faults_idx(NUMA_MEM, nid, priv); 2849 membuf_idx = task_faults_idx(NUMA_MEMBUF, nid, priv); 2850 cpu_idx = task_faults_idx(NUMA_CPU, nid, priv); 2851 cpubuf_idx = task_faults_idx(NUMA_CPUBUF, nid, priv); 2852 2853 /* Decay existing window, copy faults since last scan */ 2854 diff = p->numa_faults[membuf_idx] - p->numa_faults[mem_idx] / 2; 2855 fault_types[priv] += p->numa_faults[membuf_idx]; 2856 p->numa_faults[membuf_idx] = 0; 2857 2858 /* 2859 * Normalize the faults_from, so all tasks in a group 2860 * count according to CPU use, instead of by the raw 2861 * number of faults. Tasks with little runtime have 2862 * little over-all impact on throughput, and thus their 2863 * faults are less important. 2864 */ 2865 f_weight = div64_u64(runtime << 16, period + 1); 2866 f_weight = (f_weight * p->numa_faults[cpubuf_idx]) / 2867 (total_faults + 1); 2868 f_diff = f_weight - p->numa_faults[cpu_idx] / 2; 2869 p->numa_faults[cpubuf_idx] = 0; 2870 2871 p->numa_faults[mem_idx] += diff; 2872 p->numa_faults[cpu_idx] += f_diff; 2873 faults += p->numa_faults[mem_idx]; 2874 p->total_numa_faults += diff; 2875 if (ng) { 2876 /* 2877 * safe because we can only change our own group 2878 * 2879 * mem_idx represents the offset for a given 2880 * nid and priv in a specific region because it 2881 * is at the beginning of the numa_faults array. 2882 */ 2883 ng->faults[mem_idx] += diff; 2884 ng->faults[cpu_idx] += f_diff; 2885 ng->total_faults += diff; 2886 group_faults += ng->faults[mem_idx]; 2887 } 2888 } 2889 2890 if (!ng) { 2891 if (faults > max_faults) { 2892 max_faults = faults; 2893 max_nid = nid; 2894 } 2895 } else if (group_faults > max_faults) { 2896 max_faults = group_faults; 2897 max_nid = nid; 2898 } 2899 } 2900 2901 /* Cannot migrate task to CPU-less node */ 2902 if (max_nid != NUMA_NO_NODE && !node_state(max_nid, N_CPU)) { 2903 int near_nid = max_nid; 2904 int distance, near_distance = INT_MAX; 2905 2906 for_each_node_state(nid, N_CPU) { 2907 distance = node_distance(max_nid, nid); 2908 if (distance < near_distance) { 2909 near_nid = nid; 2910 near_distance = distance; 2911 } 2912 } 2913 max_nid = near_nid; 2914 } 2915 2916 if (ng) { 2917 numa_group_count_active_nodes(ng); 2918 spin_unlock_irq(group_lock); 2919 max_nid = preferred_group_nid(p, max_nid); 2920 } 2921 2922 if (max_faults) { 2923 /* Set the new preferred node */ 2924 if (max_nid != p->numa_preferred_nid) 2925 sched_setnuma(p, max_nid); 2926 } 2927 2928 update_task_scan_period(p, fault_types[0], fault_types[1]); 2929 } 2930 2931 static inline int get_numa_group(struct numa_group *grp) 2932 { 2933 return refcount_inc_not_zero(&grp->refcount); 2934 } 2935 2936 static inline void put_numa_group(struct numa_group *grp) 2937 { 2938 if (refcount_dec_and_test(&grp->refcount)) 2939 kfree_rcu(grp, rcu); 2940 } 2941 2942 static void task_numa_group(struct task_struct *p, int cpupid, int flags, 2943 int *priv) 2944 { 2945 struct numa_group *grp, *my_grp; 2946 struct task_struct *tsk; 2947 bool join = false; 2948 int cpu = cpupid_to_cpu(cpupid); 2949 int i; 2950 2951 if (unlikely(!deref_curr_numa_group(p))) { 2952 unsigned int size = sizeof(struct numa_group) + 2953 NR_NUMA_HINT_FAULT_STATS * 2954 nr_node_ids * sizeof(unsigned long); 2955 2956 grp = kzalloc(size, GFP_KERNEL | __GFP_NOWARN); 2957 if (!grp) 2958 return; 2959 2960 refcount_set(&grp->refcount, 1); 2961 grp->active_nodes = 1; 2962 grp->max_faults_cpu = 0; 2963 spin_lock_init(&grp->lock); 2964 grp->gid = p->pid; 2965 2966 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 2967 grp->faults[i] = p->numa_faults[i]; 2968 2969 grp->total_faults = p->total_numa_faults; 2970 2971 grp->nr_tasks++; 2972 rcu_assign_pointer(p->numa_group, grp); 2973 } 2974 2975 rcu_read_lock(); 2976 tsk = READ_ONCE(cpu_rq(cpu)->curr); 2977 2978 if (!cpupid_match_pid(tsk, cpupid)) 2979 goto no_join; 2980 2981 grp = rcu_dereference(tsk->numa_group); 2982 if (!grp) 2983 goto no_join; 2984 2985 my_grp = deref_curr_numa_group(p); 2986 if (grp == my_grp) 2987 goto no_join; 2988 2989 /* 2990 * Only join the other group if its bigger; if we're the bigger group, 2991 * the other task will join us. 2992 */ 2993 if (my_grp->nr_tasks > grp->nr_tasks) 2994 goto no_join; 2995 2996 /* 2997 * Tie-break on the grp address. 2998 */ 2999 if (my_grp->nr_tasks == grp->nr_tasks && my_grp > grp) 3000 goto no_join; 3001 3002 /* Always join threads in the same process. */ 3003 if (tsk->mm == current->mm) 3004 join = true; 3005 3006 /* Simple filter to avoid false positives due to PID collisions */ 3007 if (flags & TNF_SHARED) 3008 join = true; 3009 3010 /* Update priv based on whether false sharing was detected */ 3011 *priv = !join; 3012 3013 if (join && !get_numa_group(grp)) 3014 goto no_join; 3015 3016 rcu_read_unlock(); 3017 3018 if (!join) 3019 return; 3020 3021 WARN_ON_ONCE(irqs_disabled()); 3022 double_lock_irq(&my_grp->lock, &grp->lock); 3023 3024 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) { 3025 my_grp->faults[i] -= p->numa_faults[i]; 3026 grp->faults[i] += p->numa_faults[i]; 3027 } 3028 my_grp->total_faults -= p->total_numa_faults; 3029 grp->total_faults += p->total_numa_faults; 3030 3031 my_grp->nr_tasks--; 3032 grp->nr_tasks++; 3033 3034 spin_unlock(&my_grp->lock); 3035 spin_unlock_irq(&grp->lock); 3036 3037 rcu_assign_pointer(p->numa_group, grp); 3038 3039 put_numa_group(my_grp); 3040 return; 3041 3042 no_join: 3043 rcu_read_unlock(); 3044 return; 3045 } 3046 3047 /* 3048 * Get rid of NUMA statistics associated with a task (either current or dead). 3049 * If @final is set, the task is dead and has reached refcount zero, so we can 3050 * safely free all relevant data structures. Otherwise, there might be 3051 * concurrent reads from places like load balancing and procfs, and we should 3052 * reset the data back to default state without freeing ->numa_faults. 3053 */ 3054 void task_numa_free(struct task_struct *p, bool final) 3055 { 3056 /* safe: p either is current or is being freed by current */ 3057 struct numa_group *grp = rcu_dereference_raw(p->numa_group); 3058 unsigned long *numa_faults = p->numa_faults; 3059 unsigned long flags; 3060 int i; 3061 3062 if (!numa_faults) 3063 return; 3064 3065 if (grp) { 3066 spin_lock_irqsave(&grp->lock, flags); 3067 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 3068 grp->faults[i] -= p->numa_faults[i]; 3069 grp->total_faults -= p->total_numa_faults; 3070 3071 grp->nr_tasks--; 3072 spin_unlock_irqrestore(&grp->lock, flags); 3073 RCU_INIT_POINTER(p->numa_group, NULL); 3074 put_numa_group(grp); 3075 } 3076 3077 if (final) { 3078 p->numa_faults = NULL; 3079 kfree(numa_faults); 3080 } else { 3081 p->total_numa_faults = 0; 3082 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 3083 numa_faults[i] = 0; 3084 } 3085 } 3086 3087 /* 3088 * Got a PROT_NONE fault for a page on @node. 3089 */ 3090 void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags) 3091 { 3092 struct task_struct *p = current; 3093 bool migrated = flags & TNF_MIGRATED; 3094 int cpu_node = task_node(current); 3095 int local = !!(flags & TNF_FAULT_LOCAL); 3096 struct numa_group *ng; 3097 int priv; 3098 3099 if (!static_branch_likely(&sched_numa_balancing)) 3100 return; 3101 3102 /* for example, ksmd faulting in a user's mm */ 3103 if (!p->mm) 3104 return; 3105 3106 /* 3107 * NUMA faults statistics are unnecessary for the slow memory 3108 * node for memory tiering mode. 3109 */ 3110 if (!node_is_toptier(mem_node) && 3111 (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING || 3112 !cpupid_valid(last_cpupid))) 3113 return; 3114 3115 /* Allocate buffer to track faults on a per-node basis */ 3116 if (unlikely(!p->numa_faults)) { 3117 int size = sizeof(*p->numa_faults) * 3118 NR_NUMA_HINT_FAULT_BUCKETS * nr_node_ids; 3119 3120 p->numa_faults = kzalloc(size, GFP_KERNEL|__GFP_NOWARN); 3121 if (!p->numa_faults) 3122 return; 3123 3124 p->total_numa_faults = 0; 3125 memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); 3126 } 3127 3128 /* 3129 * First accesses are treated as private, otherwise consider accesses 3130 * to be private if the accessing pid has not changed 3131 */ 3132 if (unlikely(last_cpupid == (-1 & LAST_CPUPID_MASK))) { 3133 priv = 1; 3134 } else { 3135 priv = cpupid_match_pid(p, last_cpupid); 3136 if (!priv && !(flags & TNF_NO_GROUP)) 3137 task_numa_group(p, last_cpupid, flags, &priv); 3138 } 3139 3140 /* 3141 * If a workload spans multiple NUMA nodes, a shared fault that 3142 * occurs wholly within the set of nodes that the workload is 3143 * actively using should be counted as local. This allows the 3144 * scan rate to slow down when a workload has settled down. 3145 */ 3146 ng = deref_curr_numa_group(p); 3147 if (!priv && !local && ng && ng->active_nodes > 1 && 3148 numa_is_active_node(cpu_node, ng) && 3149 numa_is_active_node(mem_node, ng)) 3150 local = 1; 3151 3152 /* 3153 * Retry to migrate task to preferred node periodically, in case it 3154 * previously failed, or the scheduler moved us. 3155 */ 3156 if (time_after(jiffies, p->numa_migrate_retry)) { 3157 task_numa_placement(p); 3158 numa_migrate_preferred(p); 3159 } 3160 3161 if (migrated) 3162 p->numa_pages_migrated += pages; 3163 if (flags & TNF_MIGRATE_FAIL) 3164 p->numa_faults_locality[2] += pages; 3165 3166 p->numa_faults[task_faults_idx(NUMA_MEMBUF, mem_node, priv)] += pages; 3167 p->numa_faults[task_faults_idx(NUMA_CPUBUF, cpu_node, priv)] += pages; 3168 p->numa_faults_locality[local] += pages; 3169 } 3170 3171 static void reset_ptenuma_scan(struct task_struct *p) 3172 { 3173 /* 3174 * We only did a read acquisition of the mmap sem, so 3175 * p->mm->numa_scan_seq is written to without exclusive access 3176 * and the update is not guaranteed to be atomic. That's not 3177 * much of an issue though, since this is just used for 3178 * statistical sampling. Use READ_ONCE/WRITE_ONCE, which are not 3179 * expensive, to avoid any form of compiler optimizations: 3180 */ 3181 WRITE_ONCE(p->mm->numa_scan_seq, READ_ONCE(p->mm->numa_scan_seq) + 1); 3182 p->mm->numa_scan_offset = 0; 3183 } 3184 3185 static bool vma_is_accessed(struct vm_area_struct *vma) 3186 { 3187 unsigned long pids; 3188 /* 3189 * Allow unconditional access first two times, so that all the (pages) 3190 * of VMAs get prot_none fault introduced irrespective of accesses. 3191 * This is also done to avoid any side effect of task scanning 3192 * amplifying the unfairness of disjoint set of VMAs' access. 3193 */ 3194 if (READ_ONCE(current->mm->numa_scan_seq) < 2) 3195 return true; 3196 3197 pids = vma->numab_state->access_pids[0] | vma->numab_state->access_pids[1]; 3198 return test_bit(hash_32(current->pid, ilog2(BITS_PER_LONG)), &pids); 3199 } 3200 3201 #define VMA_PID_RESET_PERIOD (4 * sysctl_numa_balancing_scan_delay) 3202 3203 /* 3204 * The expensive part of numa migration is done from task_work context. 3205 * Triggered from task_tick_numa(). 3206 */ 3207 static void task_numa_work(struct callback_head *work) 3208 { 3209 unsigned long migrate, next_scan, now = jiffies; 3210 struct task_struct *p = current; 3211 struct mm_struct *mm = p->mm; 3212 u64 runtime = p->se.sum_exec_runtime; 3213 struct vm_area_struct *vma; 3214 unsigned long start, end; 3215 unsigned long nr_pte_updates = 0; 3216 long pages, virtpages; 3217 struct vma_iterator vmi; 3218 3219 SCHED_WARN_ON(p != container_of(work, struct task_struct, numa_work)); 3220 3221 work->next = work; 3222 /* 3223 * Who cares about NUMA placement when they're dying. 3224 * 3225 * NOTE: make sure not to dereference p->mm before this check, 3226 * exit_task_work() happens _after_ exit_mm() so we could be called 3227 * without p->mm even though we still had it when we enqueued this 3228 * work. 3229 */ 3230 if (p->flags & PF_EXITING) 3231 return; 3232 3233 if (!mm->numa_next_scan) { 3234 mm->numa_next_scan = now + 3235 msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 3236 } 3237 3238 /* 3239 * Enforce maximal scan/migration frequency.. 3240 */ 3241 migrate = mm->numa_next_scan; 3242 if (time_before(now, migrate)) 3243 return; 3244 3245 if (p->numa_scan_period == 0) { 3246 p->numa_scan_period_max = task_scan_max(p); 3247 p->numa_scan_period = task_scan_start(p); 3248 } 3249 3250 next_scan = now + msecs_to_jiffies(p->numa_scan_period); 3251 if (!try_cmpxchg(&mm->numa_next_scan, &migrate, next_scan)) 3252 return; 3253 3254 /* 3255 * Delay this task enough that another task of this mm will likely win 3256 * the next time around. 3257 */ 3258 p->node_stamp += 2 * TICK_NSEC; 3259 3260 start = mm->numa_scan_offset; 3261 pages = sysctl_numa_balancing_scan_size; 3262 pages <<= 20 - PAGE_SHIFT; /* MB in pages */ 3263 virtpages = pages * 8; /* Scan up to this much virtual space */ 3264 if (!pages) 3265 return; 3266 3267 3268 if (!mmap_read_trylock(mm)) 3269 return; 3270 vma_iter_init(&vmi, mm, start); 3271 vma = vma_next(&vmi); 3272 if (!vma) { 3273 reset_ptenuma_scan(p); 3274 start = 0; 3275 vma_iter_set(&vmi, start); 3276 vma = vma_next(&vmi); 3277 } 3278 3279 do { 3280 if (!vma_migratable(vma) || !vma_policy_mof(vma) || 3281 is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) { 3282 continue; 3283 } 3284 3285 /* 3286 * Shared library pages mapped by multiple processes are not 3287 * migrated as it is expected they are cache replicated. Avoid 3288 * hinting faults in read-only file-backed mappings or the vdso 3289 * as migrating the pages will be of marginal benefit. 3290 */ 3291 if (!vma->vm_mm || 3292 (vma->vm_file && (vma->vm_flags & (VM_READ|VM_WRITE)) == (VM_READ))) 3293 continue; 3294 3295 /* 3296 * Skip inaccessible VMAs to avoid any confusion between 3297 * PROT_NONE and NUMA hinting ptes 3298 */ 3299 if (!vma_is_accessible(vma)) 3300 continue; 3301 3302 /* Initialise new per-VMA NUMAB state. */ 3303 if (!vma->numab_state) { 3304 vma->numab_state = kzalloc(sizeof(struct vma_numab_state), 3305 GFP_KERNEL); 3306 if (!vma->numab_state) 3307 continue; 3308 3309 vma->numab_state->next_scan = now + 3310 msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 3311 3312 /* Reset happens after 4 times scan delay of scan start */ 3313 vma->numab_state->next_pid_reset = vma->numab_state->next_scan + 3314 msecs_to_jiffies(VMA_PID_RESET_PERIOD); 3315 } 3316 3317 /* 3318 * Scanning the VMA's of short lived tasks add more overhead. So 3319 * delay the scan for new VMAs. 3320 */ 3321 if (mm->numa_scan_seq && time_before(jiffies, 3322 vma->numab_state->next_scan)) 3323 continue; 3324 3325 /* Do not scan the VMA if task has not accessed */ 3326 if (!vma_is_accessed(vma)) 3327 continue; 3328 3329 /* 3330 * RESET access PIDs regularly for old VMAs. Resetting after checking 3331 * vma for recent access to avoid clearing PID info before access.. 3332 */ 3333 if (mm->numa_scan_seq && 3334 time_after(jiffies, vma->numab_state->next_pid_reset)) { 3335 vma->numab_state->next_pid_reset = vma->numab_state->next_pid_reset + 3336 msecs_to_jiffies(VMA_PID_RESET_PERIOD); 3337 vma->numab_state->access_pids[0] = READ_ONCE(vma->numab_state->access_pids[1]); 3338 vma->numab_state->access_pids[1] = 0; 3339 } 3340 3341 do { 3342 start = max(start, vma->vm_start); 3343 end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE); 3344 end = min(end, vma->vm_end); 3345 nr_pte_updates = change_prot_numa(vma, start, end); 3346 3347 /* 3348 * Try to scan sysctl_numa_balancing_size worth of 3349 * hpages that have at least one present PTE that 3350 * is not already pte-numa. If the VMA contains 3351 * areas that are unused or already full of prot_numa 3352 * PTEs, scan up to virtpages, to skip through those 3353 * areas faster. 3354 */ 3355 if (nr_pte_updates) 3356 pages -= (end - start) >> PAGE_SHIFT; 3357 virtpages -= (end - start) >> PAGE_SHIFT; 3358 3359 start = end; 3360 if (pages <= 0 || virtpages <= 0) 3361 goto out; 3362 3363 cond_resched(); 3364 } while (end != vma->vm_end); 3365 } for_each_vma(vmi, vma); 3366 3367 out: 3368 /* 3369 * It is possible to reach the end of the VMA list but the last few 3370 * VMAs are not guaranteed to the vma_migratable. If they are not, we 3371 * would find the !migratable VMA on the next scan but not reset the 3372 * scanner to the start so check it now. 3373 */ 3374 if (vma) 3375 mm->numa_scan_offset = start; 3376 else 3377 reset_ptenuma_scan(p); 3378 mmap_read_unlock(mm); 3379 3380 /* 3381 * Make sure tasks use at least 32x as much time to run other code 3382 * than they used here, to limit NUMA PTE scanning overhead to 3% max. 3383 * Usually update_task_scan_period slows down scanning enough; on an 3384 * overloaded system we need to limit overhead on a per task basis. 3385 */ 3386 if (unlikely(p->se.sum_exec_runtime != runtime)) { 3387 u64 diff = p->se.sum_exec_runtime - runtime; 3388 p->node_stamp += 32 * diff; 3389 } 3390 } 3391 3392 void init_numa_balancing(unsigned long clone_flags, struct task_struct *p) 3393 { 3394 int mm_users = 0; 3395 struct mm_struct *mm = p->mm; 3396 3397 if (mm) { 3398 mm_users = atomic_read(&mm->mm_users); 3399 if (mm_users == 1) { 3400 mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 3401 mm->numa_scan_seq = 0; 3402 } 3403 } 3404 p->node_stamp = 0; 3405 p->numa_scan_seq = mm ? mm->numa_scan_seq : 0; 3406 p->numa_scan_period = sysctl_numa_balancing_scan_delay; 3407 p->numa_migrate_retry = 0; 3408 /* Protect against double add, see task_tick_numa and task_numa_work */ 3409 p->numa_work.next = &p->numa_work; 3410 p->numa_faults = NULL; 3411 p->numa_pages_migrated = 0; 3412 p->total_numa_faults = 0; 3413 RCU_INIT_POINTER(p->numa_group, NULL); 3414 p->last_task_numa_placement = 0; 3415 p->last_sum_exec_runtime = 0; 3416 3417 init_task_work(&p->numa_work, task_numa_work); 3418 3419 /* New address space, reset the preferred nid */ 3420 if (!(clone_flags & CLONE_VM)) { 3421 p->numa_preferred_nid = NUMA_NO_NODE; 3422 return; 3423 } 3424 3425 /* 3426 * New thread, keep existing numa_preferred_nid which should be copied 3427 * already by arch_dup_task_struct but stagger when scans start. 3428 */ 3429 if (mm) { 3430 unsigned int delay; 3431 3432 delay = min_t(unsigned int, task_scan_max(current), 3433 current->numa_scan_period * mm_users * NSEC_PER_MSEC); 3434 delay += 2 * TICK_NSEC; 3435 p->node_stamp = delay; 3436 } 3437 } 3438 3439 /* 3440 * Drive the periodic memory faults.. 3441 */ 3442 static void task_tick_numa(struct rq *rq, struct task_struct *curr) 3443 { 3444 struct callback_head *work = &curr->numa_work; 3445 u64 period, now; 3446 3447 /* 3448 * We don't care about NUMA placement if we don't have memory. 3449 */ 3450 if (!curr->mm || (curr->flags & (PF_EXITING | PF_KTHREAD)) || work->next != work) 3451 return; 3452 3453 /* 3454 * Using runtime rather than walltime has the dual advantage that 3455 * we (mostly) drive the selection from busy threads and that the 3456 * task needs to have done some actual work before we bother with 3457 * NUMA placement. 3458 */ 3459 now = curr->se.sum_exec_runtime; 3460 period = (u64)curr->numa_scan_period * NSEC_PER_MSEC; 3461 3462 if (now > curr->node_stamp + period) { 3463 if (!curr->node_stamp) 3464 curr->numa_scan_period = task_scan_start(curr); 3465 curr->node_stamp += period; 3466 3467 if (!time_before(jiffies, curr->mm->numa_next_scan)) 3468 task_work_add(curr, work, TWA_RESUME); 3469 } 3470 } 3471 3472 static void update_scan_period(struct task_struct *p, int new_cpu) 3473 { 3474 int src_nid = cpu_to_node(task_cpu(p)); 3475 int dst_nid = cpu_to_node(new_cpu); 3476 3477 if (!static_branch_likely(&sched_numa_balancing)) 3478 return; 3479 3480 if (!p->mm || !p->numa_faults || (p->flags & PF_EXITING)) 3481 return; 3482 3483 if (src_nid == dst_nid) 3484 return; 3485 3486 /* 3487 * Allow resets if faults have been trapped before one scan 3488 * has completed. This is most likely due to a new task that 3489 * is pulled cross-node due to wakeups or load balancing. 3490 */ 3491 if (p->numa_scan_seq) { 3492 /* 3493 * Avoid scan adjustments if moving to the preferred 3494 * node or if the task was not previously running on 3495 * the preferred node. 3496 */ 3497 if (dst_nid == p->numa_preferred_nid || 3498 (p->numa_preferred_nid != NUMA_NO_NODE && 3499 src_nid != p->numa_preferred_nid)) 3500 return; 3501 } 3502 3503 p->numa_scan_period = task_scan_start(p); 3504 } 3505 3506 #else 3507 static void task_tick_numa(struct rq *rq, struct task_struct *curr) 3508 { 3509 } 3510 3511 static inline void account_numa_enqueue(struct rq *rq, struct task_struct *p) 3512 { 3513 } 3514 3515 static inline void account_numa_dequeue(struct rq *rq, struct task_struct *p) 3516 { 3517 } 3518 3519 static inline void update_scan_period(struct task_struct *p, int new_cpu) 3520 { 3521 } 3522 3523 #endif /* CONFIG_NUMA_BALANCING */ 3524 3525 static void 3526 account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) 3527 { 3528 update_load_add(&cfs_rq->load, se->load.weight); 3529 #ifdef CONFIG_SMP 3530 if (entity_is_task(se)) { 3531 struct rq *rq = rq_of(cfs_rq); 3532 3533 account_numa_enqueue(rq, task_of(se)); 3534 list_add(&se->group_node, &rq->cfs_tasks); 3535 } 3536 #endif 3537 cfs_rq->nr_running++; 3538 if (se_is_idle(se)) 3539 cfs_rq->idle_nr_running++; 3540 } 3541 3542 static void 3543 account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) 3544 { 3545 update_load_sub(&cfs_rq->load, se->load.weight); 3546 #ifdef CONFIG_SMP 3547 if (entity_is_task(se)) { 3548 account_numa_dequeue(rq_of(cfs_rq), task_of(se)); 3549 list_del_init(&se->group_node); 3550 } 3551 #endif 3552 cfs_rq->nr_running--; 3553 if (se_is_idle(se)) 3554 cfs_rq->idle_nr_running--; 3555 } 3556 3557 /* 3558 * Signed add and clamp on underflow. 3559 * 3560 * Explicitly do a load-store to ensure the intermediate value never hits 3561 * memory. This allows lockless observations without ever seeing the negative 3562 * values. 3563 */ 3564 #define add_positive(_ptr, _val) do { \ 3565 typeof(_ptr) ptr = (_ptr); \ 3566 typeof(_val) val = (_val); \ 3567 typeof(*ptr) res, var = READ_ONCE(*ptr); \ 3568 \ 3569 res = var + val; \ 3570 \ 3571 if (val < 0 && res > var) \ 3572 res = 0; \ 3573 \ 3574 WRITE_ONCE(*ptr, res); \ 3575 } while (0) 3576 3577 /* 3578 * Unsigned subtract and clamp on underflow. 3579 * 3580 * Explicitly do a load-store to ensure the intermediate value never hits 3581 * memory. This allows lockless observations without ever seeing the negative 3582 * values. 3583 */ 3584 #define sub_positive(_ptr, _val) do { \ 3585 typeof(_ptr) ptr = (_ptr); \ 3586 typeof(*ptr) val = (_val); \ 3587 typeof(*ptr) res, var = READ_ONCE(*ptr); \ 3588 res = var - val; \ 3589 if (res > var) \ 3590 res = 0; \ 3591 WRITE_ONCE(*ptr, res); \ 3592 } while (0) 3593 3594 /* 3595 * Remove and clamp on negative, from a local variable. 3596 * 3597 * A variant of sub_positive(), which does not use explicit load-store 3598 * and is thus optimized for local variable updates. 3599 */ 3600 #define lsub_positive(_ptr, _val) do { \ 3601 typeof(_ptr) ptr = (_ptr); \ 3602 *ptr -= min_t(typeof(*ptr), *ptr, _val); \ 3603 } while (0) 3604 3605 #ifdef CONFIG_SMP 3606 static inline void 3607 enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 3608 { 3609 cfs_rq->avg.load_avg += se->avg.load_avg; 3610 cfs_rq->avg.load_sum += se_weight(se) * se->avg.load_sum; 3611 } 3612 3613 static inline void 3614 dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 3615 { 3616 sub_positive(&cfs_rq->avg.load_avg, se->avg.load_avg); 3617 sub_positive(&cfs_rq->avg.load_sum, se_weight(se) * se->avg.load_sum); 3618 /* See update_cfs_rq_load_avg() */ 3619 cfs_rq->avg.load_sum = max_t(u32, cfs_rq->avg.load_sum, 3620 cfs_rq->avg.load_avg * PELT_MIN_DIVIDER); 3621 } 3622 #else 3623 static inline void 3624 enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { } 3625 static inline void 3626 dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { } 3627 #endif 3628 3629 static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, 3630 unsigned long weight) 3631 { 3632 unsigned long old_weight = se->load.weight; 3633 3634 if (se->on_rq) { 3635 /* commit outstanding execution time */ 3636 if (cfs_rq->curr == se) 3637 update_curr(cfs_rq); 3638 else 3639 avg_vruntime_sub(cfs_rq, se); 3640 update_load_sub(&cfs_rq->load, se->load.weight); 3641 } 3642 dequeue_load_avg(cfs_rq, se); 3643 3644 update_load_set(&se->load, weight); 3645 3646 if (!se->on_rq) { 3647 /* 3648 * Because we keep se->vlag = V - v_i, while: lag_i = w_i*(V - v_i), 3649 * we need to scale se->vlag when w_i changes. 3650 */ 3651 se->vlag = div_s64(se->vlag * old_weight, weight); 3652 } else { 3653 s64 deadline = se->deadline - se->vruntime; 3654 /* 3655 * When the weight changes, the virtual time slope changes and 3656 * we should adjust the relative virtual deadline accordingly. 3657 */ 3658 deadline = div_s64(deadline * old_weight, weight); 3659 se->deadline = se->vruntime + deadline; 3660 if (se != cfs_rq->curr) 3661 min_deadline_cb_propagate(&se->run_node, NULL); 3662 } 3663 3664 #ifdef CONFIG_SMP 3665 do { 3666 u32 divider = get_pelt_divider(&se->avg); 3667 3668 se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider); 3669 } while (0); 3670 #endif 3671 3672 enqueue_load_avg(cfs_rq, se); 3673 if (se->on_rq) { 3674 update_load_add(&cfs_rq->load, se->load.weight); 3675 if (cfs_rq->curr != se) 3676 avg_vruntime_add(cfs_rq, se); 3677 } 3678 } 3679 3680 void reweight_task(struct task_struct *p, int prio) 3681 { 3682 struct sched_entity *se = &p->se; 3683 struct cfs_rq *cfs_rq = cfs_rq_of(se); 3684 struct load_weight *load = &se->load; 3685 unsigned long weight = scale_load(sched_prio_to_weight[prio]); 3686 3687 reweight_entity(cfs_rq, se, weight); 3688 load->inv_weight = sched_prio_to_wmult[prio]; 3689 } 3690 3691 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); 3692 3693 #ifdef CONFIG_FAIR_GROUP_SCHED 3694 #ifdef CONFIG_SMP 3695 /* 3696 * All this does is approximate the hierarchical proportion which includes that 3697 * global sum we all love to hate. 3698 * 3699 * That is, the weight of a group entity, is the proportional share of the 3700 * group weight based on the group runqueue weights. That is: 3701 * 3702 * tg->weight * grq->load.weight 3703 * ge->load.weight = ----------------------------- (1) 3704 * \Sum grq->load.weight 3705 * 3706 * Now, because computing that sum is prohibitively expensive to compute (been 3707 * there, done that) we approximate it with this average stuff. The average 3708 * moves slower and therefore the approximation is cheaper and more stable. 3709 * 3710 * So instead of the above, we substitute: 3711 * 3712 * grq->load.weight -> grq->avg.load_avg (2) 3713 * 3714 * which yields the following: 3715 * 3716 * tg->weight * grq->avg.load_avg 3717 * ge->load.weight = ------------------------------ (3) 3718 * tg->load_avg 3719 * 3720 * Where: tg->load_avg ~= \Sum grq->avg.load_avg 3721 * 3722 * That is shares_avg, and it is right (given the approximation (2)). 3723 * 3724 * The problem with it is that because the average is slow -- it was designed 3725 * to be exactly that of course -- this leads to transients in boundary 3726 * conditions. In specific, the case where the group was idle and we start the 3727 * one task. It takes time for our CPU's grq->avg.load_avg to build up, 3728 * yielding bad latency etc.. 3729 * 3730 * Now, in that special case (1) reduces to: 3731 * 3732 * tg->weight * grq->load.weight 3733 * ge->load.weight = ----------------------------- = tg->weight (4) 3734 * grp->load.weight 3735 * 3736 * That is, the sum collapses because all other CPUs are idle; the UP scenario. 3737 * 3738 * So what we do is modify our approximation (3) to approach (4) in the (near) 3739 * UP case, like: 3740 * 3741 * ge->load.weight = 3742 * 3743 * tg->weight * grq->load.weight 3744 * --------------------------------------------------- (5) 3745 * tg->load_avg - grq->avg.load_avg + grq->load.weight 3746 * 3747 * But because grq->load.weight can drop to 0, resulting in a divide by zero, 3748 * we need to use grq->avg.load_avg as its lower bound, which then gives: 3749 * 3750 * 3751 * tg->weight * grq->load.weight 3752 * ge->load.weight = ----------------------------- (6) 3753 * tg_load_avg' 3754 * 3755 * Where: 3756 * 3757 * tg_load_avg' = tg->load_avg - grq->avg.load_avg + 3758 * max(grq->load.weight, grq->avg.load_avg) 3759 * 3760 * And that is shares_weight and is icky. In the (near) UP case it approaches 3761 * (4) while in the normal case it approaches (3). It consistently 3762 * overestimates the ge->load.weight and therefore: 3763 * 3764 * \Sum ge->load.weight >= tg->weight 3765 * 3766 * hence icky! 3767 */ 3768 static long calc_group_shares(struct cfs_rq *cfs_rq) 3769 { 3770 long tg_weight, tg_shares, load, shares; 3771 struct task_group *tg = cfs_rq->tg; 3772 3773 tg_shares = READ_ONCE(tg->shares); 3774 3775 load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg); 3776 3777 tg_weight = atomic_long_read(&tg->load_avg); 3778 3779 /* Ensure tg_weight >= load */ 3780 tg_weight -= cfs_rq->tg_load_avg_contrib; 3781 tg_weight += load; 3782 3783 shares = (tg_shares * load); 3784 if (tg_weight) 3785 shares /= tg_weight; 3786 3787 /* 3788 * MIN_SHARES has to be unscaled here to support per-CPU partitioning 3789 * of a group with small tg->shares value. It is a floor value which is 3790 * assigned as a minimum load.weight to the sched_entity representing 3791 * the group on a CPU. 3792 * 3793 * E.g. on 64-bit for a group with tg->shares of scale_load(15)=15*1024 3794 * on an 8-core system with 8 tasks each runnable on one CPU shares has 3795 * to be 15*1024*1/8=1920 instead of scale_load(MIN_SHARES)=2*1024. In 3796 * case no task is runnable on a CPU MIN_SHARES=2 should be returned 3797 * instead of 0. 3798 */ 3799 return clamp_t(long, shares, MIN_SHARES, tg_shares); 3800 } 3801 #endif /* CONFIG_SMP */ 3802 3803 /* 3804 * Recomputes the group entity based on the current state of its group 3805 * runqueue. 3806 */ 3807 static void update_cfs_group(struct sched_entity *se) 3808 { 3809 struct cfs_rq *gcfs_rq = group_cfs_rq(se); 3810 long shares; 3811 3812 if (!gcfs_rq) 3813 return; 3814 3815 if (throttled_hierarchy(gcfs_rq)) 3816 return; 3817 3818 #ifndef CONFIG_SMP 3819 shares = READ_ONCE(gcfs_rq->tg->shares); 3820 3821 if (likely(se->load.weight == shares)) 3822 return; 3823 #else 3824 shares = calc_group_shares(gcfs_rq); 3825 #endif 3826 3827 reweight_entity(cfs_rq_of(se), se, shares); 3828 } 3829 3830 #else /* CONFIG_FAIR_GROUP_SCHED */ 3831 static inline void update_cfs_group(struct sched_entity *se) 3832 { 3833 } 3834 #endif /* CONFIG_FAIR_GROUP_SCHED */ 3835 3836 static inline void cfs_rq_util_change(struct cfs_rq *cfs_rq, int flags) 3837 { 3838 struct rq *rq = rq_of(cfs_rq); 3839 3840 if (&rq->cfs == cfs_rq) { 3841 /* 3842 * There are a few boundary cases this might miss but it should 3843 * get called often enough that that should (hopefully) not be 3844 * a real problem. 3845 * 3846 * It will not get called when we go idle, because the idle 3847 * thread is a different class (!fair), nor will the utilization 3848 * number include things like RT tasks. 3849 * 3850 * As is, the util number is not freq-invariant (we'd have to 3851 * implement arch_scale_freq_capacity() for that). 3852 * 3853 * See cpu_util_cfs(). 3854 */ 3855 cpufreq_update_util(rq, flags); 3856 } 3857 } 3858 3859 #ifdef CONFIG_SMP 3860 static inline bool load_avg_is_decayed(struct sched_avg *sa) 3861 { 3862 if (sa->load_sum) 3863 return false; 3864 3865 if (sa->util_sum) 3866 return false; 3867 3868 if (sa->runnable_sum) 3869 return false; 3870 3871 /* 3872 * _avg must be null when _sum are null because _avg = _sum / divider 3873 * Make sure that rounding and/or propagation of PELT values never 3874 * break this. 3875 */ 3876 SCHED_WARN_ON(sa->load_avg || 3877 sa->util_avg || 3878 sa->runnable_avg); 3879 3880 return true; 3881 } 3882 3883 static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq) 3884 { 3885 return u64_u32_load_copy(cfs_rq->avg.last_update_time, 3886 cfs_rq->last_update_time_copy); 3887 } 3888 #ifdef CONFIG_FAIR_GROUP_SCHED 3889 /* 3890 * Because list_add_leaf_cfs_rq always places a child cfs_rq on the list 3891 * immediately before a parent cfs_rq, and cfs_rqs are removed from the list 3892 * bottom-up, we only have to test whether the cfs_rq before us on the list 3893 * is our child. 3894 * If cfs_rq is not on the list, test whether a child needs its to be added to 3895 * connect a branch to the tree * (see list_add_leaf_cfs_rq() for details). 3896 */ 3897 static inline bool child_cfs_rq_on_list(struct cfs_rq *cfs_rq) 3898 { 3899 struct cfs_rq *prev_cfs_rq; 3900 struct list_head *prev; 3901 3902 if (cfs_rq->on_list) { 3903 prev = cfs_rq->leaf_cfs_rq_list.prev; 3904 } else { 3905 struct rq *rq = rq_of(cfs_rq); 3906 3907 prev = rq->tmp_alone_branch; 3908 } 3909 3910 prev_cfs_rq = container_of(prev, struct cfs_rq, leaf_cfs_rq_list); 3911 3912 return (prev_cfs_rq->tg->parent == cfs_rq->tg); 3913 } 3914 3915 static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq) 3916 { 3917 if (cfs_rq->load.weight) 3918 return false; 3919 3920 if (!load_avg_is_decayed(&cfs_rq->avg)) 3921 return false; 3922 3923 if (child_cfs_rq_on_list(cfs_rq)) 3924 return false; 3925 3926 return true; 3927 } 3928 3929 /** 3930 * update_tg_load_avg - update the tg's load avg 3931 * @cfs_rq: the cfs_rq whose avg changed 3932 * 3933 * This function 'ensures': tg->load_avg := \Sum tg->cfs_rq[]->avg.load. 3934 * However, because tg->load_avg is a global value there are performance 3935 * considerations. 3936 * 3937 * In order to avoid having to look at the other cfs_rq's, we use a 3938 * differential update where we store the last value we propagated. This in 3939 * turn allows skipping updates if the differential is 'small'. 3940 * 3941 * Updating tg's load_avg is necessary before update_cfs_share(). 3942 */ 3943 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) 3944 { 3945 long delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; 3946 3947 /* 3948 * No need to update load_avg for root_task_group as it is not used. 3949 */ 3950 if (cfs_rq->tg == &root_task_group) 3951 return; 3952 3953 if (abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { 3954 atomic_long_add(delta, &cfs_rq->tg->load_avg); 3955 cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; 3956 } 3957 } 3958 3959 /* 3960 * Called within set_task_rq() right before setting a task's CPU. The 3961 * caller only guarantees p->pi_lock is held; no other assumptions, 3962 * including the state of rq->lock, should be made. 3963 */ 3964 void set_task_rq_fair(struct sched_entity *se, 3965 struct cfs_rq *prev, struct cfs_rq *next) 3966 { 3967 u64 p_last_update_time; 3968 u64 n_last_update_time; 3969 3970 if (!sched_feat(ATTACH_AGE_LOAD)) 3971 return; 3972 3973 /* 3974 * We are supposed to update the task to "current" time, then its up to 3975 * date and ready to go to new CPU/cfs_rq. But we have difficulty in 3976 * getting what current time is, so simply throw away the out-of-date 3977 * time. This will result in the wakee task is less decayed, but giving 3978 * the wakee more load sounds not bad. 3979 */ 3980 if (!(se->avg.last_update_time && prev)) 3981 return; 3982 3983 p_last_update_time = cfs_rq_last_update_time(prev); 3984 n_last_update_time = cfs_rq_last_update_time(next); 3985 3986 __update_load_avg_blocked_se(p_last_update_time, se); 3987 se->avg.last_update_time = n_last_update_time; 3988 } 3989 3990 /* 3991 * When on migration a sched_entity joins/leaves the PELT hierarchy, we need to 3992 * propagate its contribution. The key to this propagation is the invariant 3993 * that for each group: 3994 * 3995 * ge->avg == grq->avg (1) 3996 * 3997 * _IFF_ we look at the pure running and runnable sums. Because they 3998 * represent the very same entity, just at different points in the hierarchy. 3999 * 4000 * Per the above update_tg_cfs_util() and update_tg_cfs_runnable() are trivial 4001 * and simply copies the running/runnable sum over (but still wrong, because 4002 * the group entity and group rq do not have their PELT windows aligned). 4003 * 4004 * However, update_tg_cfs_load() is more complex. So we have: 4005 * 4006 * ge->avg.load_avg = ge->load.weight * ge->avg.runnable_avg (2) 4007 * 4008 * And since, like util, the runnable part should be directly transferable, 4009 * the following would _appear_ to be the straight forward approach: 4010 * 4011 * grq->avg.load_avg = grq->load.weight * grq->avg.runnable_avg (3) 4012 * 4013 * And per (1) we have: 4014 * 4015 * ge->avg.runnable_avg == grq->avg.runnable_avg 4016 * 4017 * Which gives: 4018 * 4019 * ge->load.weight * grq->avg.load_avg 4020 * ge->avg.load_avg = ----------------------------------- (4) 4021 * grq->load.weight 4022 * 4023 * Except that is wrong! 4024 * 4025 * Because while for entities historical weight is not important and we 4026 * really only care about our future and therefore can consider a pure 4027 * runnable sum, runqueues can NOT do this. 4028 * 4029 * We specifically want runqueues to have a load_avg that includes 4030 * historical weights. Those represent the blocked load, the load we expect 4031 * to (shortly) return to us. This only works by keeping the weights as 4032 * integral part of the sum. We therefore cannot decompose as per (3). 4033 * 4034 * Another reason this doesn't work is that runnable isn't a 0-sum entity. 4035 * Imagine a rq with 2 tasks that each are runnable 2/3 of the time. Then the 4036 * rq itself is runnable anywhere between 2/3 and 1 depending on how the 4037 * runnable section of these tasks overlap (or not). If they were to perfectly 4038 * align the rq as a whole would be runnable 2/3 of the time. If however we 4039 * always have at least 1 runnable task, the rq as a whole is always runnable. 4040 * 4041 * So we'll have to approximate.. :/ 4042 * 4043 * Given the constraint: 4044 * 4045 * ge->avg.running_sum <= ge->avg.runnable_sum <= LOAD_AVG_MAX 4046 * 4047 * We can construct a rule that adds runnable to a rq by assuming minimal 4048 * overlap. 4049 * 4050 * On removal, we'll assume each task is equally runnable; which yields: 4051 * 4052 * grq->avg.runnable_sum = grq->avg.load_sum / grq->load.weight 4053 * 4054 * XXX: only do this for the part of runnable > running ? 4055 * 4056 */ 4057 static inline void 4058 update_tg_cfs_util(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 4059 { 4060 long delta_sum, delta_avg = gcfs_rq->avg.util_avg - se->avg.util_avg; 4061 u32 new_sum, divider; 4062 4063 /* Nothing to update */ 4064 if (!delta_avg) 4065 return; 4066 4067 /* 4068 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 4069 * See ___update_load_avg() for details. 4070 */ 4071 divider = get_pelt_divider(&cfs_rq->avg); 4072 4073 4074 /* Set new sched_entity's utilization */ 4075 se->avg.util_avg = gcfs_rq->avg.util_avg; 4076 new_sum = se->avg.util_avg * divider; 4077 delta_sum = (long)new_sum - (long)se->avg.util_sum; 4078 se->avg.util_sum = new_sum; 4079 4080 /* Update parent cfs_rq utilization */ 4081 add_positive(&cfs_rq->avg.util_avg, delta_avg); 4082 add_positive(&cfs_rq->avg.util_sum, delta_sum); 4083 4084 /* See update_cfs_rq_load_avg() */ 4085 cfs_rq->avg.util_sum = max_t(u32, cfs_rq->avg.util_sum, 4086 cfs_rq->avg.util_avg * PELT_MIN_DIVIDER); 4087 } 4088 4089 static inline void 4090 update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 4091 { 4092 long delta_sum, delta_avg = gcfs_rq->avg.runnable_avg - se->avg.runnable_avg; 4093 u32 new_sum, divider; 4094 4095 /* Nothing to update */ 4096 if (!delta_avg) 4097 return; 4098 4099 /* 4100 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 4101 * See ___update_load_avg() for details. 4102 */ 4103 divider = get_pelt_divider(&cfs_rq->avg); 4104 4105 /* Set new sched_entity's runnable */ 4106 se->avg.runnable_avg = gcfs_rq->avg.runnable_avg; 4107 new_sum = se->avg.runnable_avg * divider; 4108 delta_sum = (long)new_sum - (long)se->avg.runnable_sum; 4109 se->avg.runnable_sum = new_sum; 4110 4111 /* Update parent cfs_rq runnable */ 4112 add_positive(&cfs_rq->avg.runnable_avg, delta_avg); 4113 add_positive(&cfs_rq->avg.runnable_sum, delta_sum); 4114 /* See update_cfs_rq_load_avg() */ 4115 cfs_rq->avg.runnable_sum = max_t(u32, cfs_rq->avg.runnable_sum, 4116 cfs_rq->avg.runnable_avg * PELT_MIN_DIVIDER); 4117 } 4118 4119 static inline void 4120 update_tg_cfs_load(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 4121 { 4122 long delta_avg, running_sum, runnable_sum = gcfs_rq->prop_runnable_sum; 4123 unsigned long load_avg; 4124 u64 load_sum = 0; 4125 s64 delta_sum; 4126 u32 divider; 4127 4128 if (!runnable_sum) 4129 return; 4130 4131 gcfs_rq->prop_runnable_sum = 0; 4132 4133 /* 4134 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 4135 * See ___update_load_avg() for details. 4136 */ 4137 divider = get_pelt_divider(&cfs_rq->avg); 4138 4139 if (runnable_sum >= 0) { 4140 /* 4141 * Add runnable; clip at LOAD_AVG_MAX. Reflects that until 4142 * the CPU is saturated running == runnable. 4143 */ 4144 runnable_sum += se->avg.load_sum; 4145 runnable_sum = min_t(long, runnable_sum, divider); 4146 } else { 4147 /* 4148 * Estimate the new unweighted runnable_sum of the gcfs_rq by 4149 * assuming all tasks are equally runnable. 4150 */ 4151 if (scale_load_down(gcfs_rq->load.weight)) { 4152 load_sum = div_u64(gcfs_rq->avg.load_sum, 4153 scale_load_down(gcfs_rq->load.weight)); 4154 } 4155 4156 /* But make sure to not inflate se's runnable */ 4157 runnable_sum = min(se->avg.load_sum, load_sum); 4158 } 4159 4160 /* 4161 * runnable_sum can't be lower than running_sum 4162 * Rescale running sum to be in the same range as runnable sum 4163 * running_sum is in [0 : LOAD_AVG_MAX << SCHED_CAPACITY_SHIFT] 4164 * runnable_sum is in [0 : LOAD_AVG_MAX] 4165 */ 4166 running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT; 4167 runnable_sum = max(runnable_sum, running_sum); 4168 4169 load_sum = se_weight(se) * runnable_sum; 4170 load_avg = div_u64(load_sum, divider); 4171 4172 delta_avg = load_avg - se->avg.load_avg; 4173 if (!delta_avg) 4174 return; 4175 4176 delta_sum = load_sum - (s64)se_weight(se) * se->avg.load_sum; 4177 4178 se->avg.load_sum = runnable_sum; 4179 se->avg.load_avg = load_avg; 4180 add_positive(&cfs_rq->avg.load_avg, delta_avg); 4181 add_positive(&cfs_rq->avg.load_sum, delta_sum); 4182 /* See update_cfs_rq_load_avg() */ 4183 cfs_rq->avg.load_sum = max_t(u32, cfs_rq->avg.load_sum, 4184 cfs_rq->avg.load_avg * PELT_MIN_DIVIDER); 4185 } 4186 4187 static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) 4188 { 4189 cfs_rq->propagate = 1; 4190 cfs_rq->prop_runnable_sum += runnable_sum; 4191 } 4192 4193 /* Update task and its cfs_rq load average */ 4194 static inline int propagate_entity_load_avg(struct sched_entity *se) 4195 { 4196 struct cfs_rq *cfs_rq, *gcfs_rq; 4197 4198 if (entity_is_task(se)) 4199 return 0; 4200 4201 gcfs_rq = group_cfs_rq(se); 4202 if (!gcfs_rq->propagate) 4203 return 0; 4204 4205 gcfs_rq->propagate = 0; 4206 4207 cfs_rq = cfs_rq_of(se); 4208 4209 add_tg_cfs_propagate(cfs_rq, gcfs_rq->prop_runnable_sum); 4210 4211 update_tg_cfs_util(cfs_rq, se, gcfs_rq); 4212 update_tg_cfs_runnable(cfs_rq, se, gcfs_rq); 4213 update_tg_cfs_load(cfs_rq, se, gcfs_rq); 4214 4215 trace_pelt_cfs_tp(cfs_rq); 4216 trace_pelt_se_tp(se); 4217 4218 return 1; 4219 } 4220 4221 /* 4222 * Check if we need to update the load and the utilization of a blocked 4223 * group_entity: 4224 */ 4225 static inline bool skip_blocked_update(struct sched_entity *se) 4226 { 4227 struct cfs_rq *gcfs_rq = group_cfs_rq(se); 4228 4229 /* 4230 * If sched_entity still have not zero load or utilization, we have to 4231 * decay it: 4232 */ 4233 if (se->avg.load_avg || se->avg.util_avg) 4234 return false; 4235 4236 /* 4237 * If there is a pending propagation, we have to update the load and 4238 * the utilization of the sched_entity: 4239 */ 4240 if (gcfs_rq->propagate) 4241 return false; 4242 4243 /* 4244 * Otherwise, the load and the utilization of the sched_entity is 4245 * already zero and there is no pending propagation, so it will be a 4246 * waste of time to try to decay it: 4247 */ 4248 return true; 4249 } 4250 4251 #else /* CONFIG_FAIR_GROUP_SCHED */ 4252 4253 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) {} 4254 4255 static inline int propagate_entity_load_avg(struct sched_entity *se) 4256 { 4257 return 0; 4258 } 4259 4260 static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) {} 4261 4262 #endif /* CONFIG_FAIR_GROUP_SCHED */ 4263 4264 #ifdef CONFIG_NO_HZ_COMMON 4265 static inline void migrate_se_pelt_lag(struct sched_entity *se) 4266 { 4267 u64 throttled = 0, now, lut; 4268 struct cfs_rq *cfs_rq; 4269 struct rq *rq; 4270 bool is_idle; 4271 4272 if (load_avg_is_decayed(&se->avg)) 4273 return; 4274 4275 cfs_rq = cfs_rq_of(se); 4276 rq = rq_of(cfs_rq); 4277 4278 rcu_read_lock(); 4279 is_idle = is_idle_task(rcu_dereference(rq->curr)); 4280 rcu_read_unlock(); 4281 4282 /* 4283 * The lag estimation comes with a cost we don't want to pay all the 4284 * time. Hence, limiting to the case where the source CPU is idle and 4285 * we know we are at the greatest risk to have an outdated clock. 4286 */ 4287 if (!is_idle) 4288 return; 4289 4290 /* 4291 * Estimated "now" is: last_update_time + cfs_idle_lag + rq_idle_lag, where: 4292 * 4293 * last_update_time (the cfs_rq's last_update_time) 4294 * = cfs_rq_clock_pelt()@cfs_rq_idle 4295 * = rq_clock_pelt()@cfs_rq_idle 4296 * - cfs->throttled_clock_pelt_time@cfs_rq_idle 4297 * 4298 * cfs_idle_lag (delta between rq's update and cfs_rq's update) 4299 * = rq_clock_pelt()@rq_idle - rq_clock_pelt()@cfs_rq_idle 4300 * 4301 * rq_idle_lag (delta between now and rq's update) 4302 * = sched_clock_cpu() - rq_clock()@rq_idle 4303 * 4304 * We can then write: 4305 * 4306 * now = rq_clock_pelt()@rq_idle - cfs->throttled_clock_pelt_time + 4307 * sched_clock_cpu() - rq_clock()@rq_idle 4308 * Where: 4309 * rq_clock_pelt()@rq_idle is rq->clock_pelt_idle 4310 * rq_clock()@rq_idle is rq->clock_idle 4311 * cfs->throttled_clock_pelt_time@cfs_rq_idle 4312 * is cfs_rq->throttled_pelt_idle 4313 */ 4314 4315 #ifdef CONFIG_CFS_BANDWIDTH 4316 throttled = u64_u32_load(cfs_rq->throttled_pelt_idle); 4317 /* The clock has been stopped for throttling */ 4318 if (throttled == U64_MAX) 4319 return; 4320 #endif 4321 now = u64_u32_load(rq->clock_pelt_idle); 4322 /* 4323 * Paired with _update_idle_rq_clock_pelt(). It ensures at the worst case 4324 * is observed the old clock_pelt_idle value and the new clock_idle, 4325 * which lead to an underestimation. The opposite would lead to an 4326 * overestimation. 4327 */ 4328 smp_rmb(); 4329 lut = cfs_rq_last_update_time(cfs_rq); 4330 4331 now -= throttled; 4332 if (now < lut) 4333 /* 4334 * cfs_rq->avg.last_update_time is more recent than our 4335 * estimation, let's use it. 4336 */ 4337 now = lut; 4338 else 4339 now += sched_clock_cpu(cpu_of(rq)) - u64_u32_load(rq->clock_idle); 4340 4341 __update_load_avg_blocked_se(now, se); 4342 } 4343 #else 4344 static void migrate_se_pelt_lag(struct sched_entity *se) {} 4345 #endif 4346 4347 /** 4348 * update_cfs_rq_load_avg - update the cfs_rq's load/util averages 4349 * @now: current time, as per cfs_rq_clock_pelt() 4350 * @cfs_rq: cfs_rq to update 4351 * 4352 * The cfs_rq avg is the direct sum of all its entities (blocked and runnable) 4353 * avg. The immediate corollary is that all (fair) tasks must be attached. 4354 * 4355 * cfs_rq->avg is used for task_h_load() and update_cfs_share() for example. 4356 * 4357 * Return: true if the load decayed or we removed load. 4358 * 4359 * Since both these conditions indicate a changed cfs_rq->avg.load we should 4360 * call update_tg_load_avg() when this function returns true. 4361 */ 4362 static inline int 4363 update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) 4364 { 4365 unsigned long removed_load = 0, removed_util = 0, removed_runnable = 0; 4366 struct sched_avg *sa = &cfs_rq->avg; 4367 int decayed = 0; 4368 4369 if (cfs_rq->removed.nr) { 4370 unsigned long r; 4371 u32 divider = get_pelt_divider(&cfs_rq->avg); 4372 4373 raw_spin_lock(&cfs_rq->removed.lock); 4374 swap(cfs_rq->removed.util_avg, removed_util); 4375 swap(cfs_rq->removed.load_avg, removed_load); 4376 swap(cfs_rq->removed.runnable_avg, removed_runnable); 4377 cfs_rq->removed.nr = 0; 4378 raw_spin_unlock(&cfs_rq->removed.lock); 4379 4380 r = removed_load; 4381 sub_positive(&sa->load_avg, r); 4382 sub_positive(&sa->load_sum, r * divider); 4383 /* See sa->util_sum below */ 4384 sa->load_sum = max_t(u32, sa->load_sum, sa->load_avg * PELT_MIN_DIVIDER); 4385 4386 r = removed_util; 4387 sub_positive(&sa->util_avg, r); 4388 sub_positive(&sa->util_sum, r * divider); 4389 /* 4390 * Because of rounding, se->util_sum might ends up being +1 more than 4391 * cfs->util_sum. Although this is not a problem by itself, detaching 4392 * a lot of tasks with the rounding problem between 2 updates of 4393 * util_avg (~1ms) can make cfs->util_sum becoming null whereas 4394 * cfs_util_avg is not. 4395 * Check that util_sum is still above its lower bound for the new 4396 * util_avg. Given that period_contrib might have moved since the last 4397 * sync, we are only sure that util_sum must be above or equal to 4398 * util_avg * minimum possible divider 4399 */ 4400 sa->util_sum = max_t(u32, sa->util_sum, sa->util_avg * PELT_MIN_DIVIDER); 4401 4402 r = removed_runnable; 4403 sub_positive(&sa->runnable_avg, r); 4404 sub_positive(&sa->runnable_sum, r * divider); 4405 /* See sa->util_sum above */ 4406 sa->runnable_sum = max_t(u32, sa->runnable_sum, 4407 sa->runnable_avg * PELT_MIN_DIVIDER); 4408 4409 /* 4410 * removed_runnable is the unweighted version of removed_load so we 4411 * can use it to estimate removed_load_sum. 4412 */ 4413 add_tg_cfs_propagate(cfs_rq, 4414 -(long)(removed_runnable * divider) >> SCHED_CAPACITY_SHIFT); 4415 4416 decayed = 1; 4417 } 4418 4419 decayed |= __update_load_avg_cfs_rq(now, cfs_rq); 4420 u64_u32_store_copy(sa->last_update_time, 4421 cfs_rq->last_update_time_copy, 4422 sa->last_update_time); 4423 return decayed; 4424 } 4425 4426 /** 4427 * attach_entity_load_avg - attach this entity to its cfs_rq load avg 4428 * @cfs_rq: cfs_rq to attach to 4429 * @se: sched_entity to attach 4430 * 4431 * Must call update_cfs_rq_load_avg() before this, since we rely on 4432 * cfs_rq->avg.last_update_time being current. 4433 */ 4434 static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 4435 { 4436 /* 4437 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 4438 * See ___update_load_avg() for details. 4439 */ 4440 u32 divider = get_pelt_divider(&cfs_rq->avg); 4441 4442 /* 4443 * When we attach the @se to the @cfs_rq, we must align the decay 4444 * window because without that, really weird and wonderful things can 4445 * happen. 4446 * 4447 * XXX illustrate 4448 */ 4449 se->avg.last_update_time = cfs_rq->avg.last_update_time; 4450 se->avg.period_contrib = cfs_rq->avg.period_contrib; 4451 4452 /* 4453 * Hell(o) Nasty stuff.. we need to recompute _sum based on the new 4454 * period_contrib. This isn't strictly correct, but since we're 4455 * entirely outside of the PELT hierarchy, nobody cares if we truncate 4456 * _sum a little. 4457 */ 4458 se->avg.util_sum = se->avg.util_avg * divider; 4459 4460 se->avg.runnable_sum = se->avg.runnable_avg * divider; 4461 4462 se->avg.load_sum = se->avg.load_avg * divider; 4463 if (se_weight(se) < se->avg.load_sum) 4464 se->avg.load_sum = div_u64(se->avg.load_sum, se_weight(se)); 4465 else 4466 se->avg.load_sum = 1; 4467 4468 enqueue_load_avg(cfs_rq, se); 4469 cfs_rq->avg.util_avg += se->avg.util_avg; 4470 cfs_rq->avg.util_sum += se->avg.util_sum; 4471 cfs_rq->avg.runnable_avg += se->avg.runnable_avg; 4472 cfs_rq->avg.runnable_sum += se->avg.runnable_sum; 4473 4474 add_tg_cfs_propagate(cfs_rq, se->avg.load_sum); 4475 4476 cfs_rq_util_change(cfs_rq, 0); 4477 4478 trace_pelt_cfs_tp(cfs_rq); 4479 } 4480 4481 /** 4482 * detach_entity_load_avg - detach this entity from its cfs_rq load avg 4483 * @cfs_rq: cfs_rq to detach from 4484 * @se: sched_entity to detach 4485 * 4486 * Must call update_cfs_rq_load_avg() before this, since we rely on 4487 * cfs_rq->avg.last_update_time being current. 4488 */ 4489 static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 4490 { 4491 dequeue_load_avg(cfs_rq, se); 4492 sub_positive(&cfs_rq->avg.util_avg, se->avg.util_avg); 4493 sub_positive(&cfs_rq->avg.util_sum, se->avg.util_sum); 4494 /* See update_cfs_rq_load_avg() */ 4495 cfs_rq->avg.util_sum = max_t(u32, cfs_rq->avg.util_sum, 4496 cfs_rq->avg.util_avg * PELT_MIN_DIVIDER); 4497 4498 sub_positive(&cfs_rq->avg.runnable_avg, se->avg.runnable_avg); 4499 sub_positive(&cfs_rq->avg.runnable_sum, se->avg.runnable_sum); 4500 /* See update_cfs_rq_load_avg() */ 4501 cfs_rq->avg.runnable_sum = max_t(u32, cfs_rq->avg.runnable_sum, 4502 cfs_rq->avg.runnable_avg * PELT_MIN_DIVIDER); 4503 4504 add_tg_cfs_propagate(cfs_rq, -se->avg.load_sum); 4505 4506 cfs_rq_util_change(cfs_rq, 0); 4507 4508 trace_pelt_cfs_tp(cfs_rq); 4509 } 4510 4511 /* 4512 * Optional action to be done while updating the load average 4513 */ 4514 #define UPDATE_TG 0x1 4515 #define SKIP_AGE_LOAD 0x2 4516 #define DO_ATTACH 0x4 4517 #define DO_DETACH 0x8 4518 4519 /* Update task and its cfs_rq load average */ 4520 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 4521 { 4522 u64 now = cfs_rq_clock_pelt(cfs_rq); 4523 int decayed; 4524 4525 /* 4526 * Track task load average for carrying it to new CPU after migrated, and 4527 * track group sched_entity load average for task_h_load calc in migration 4528 */ 4529 if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD)) 4530 __update_load_avg_se(now, cfs_rq, se); 4531 4532 decayed = update_cfs_rq_load_avg(now, cfs_rq); 4533 decayed |= propagate_entity_load_avg(se); 4534 4535 if (!se->avg.last_update_time && (flags & DO_ATTACH)) { 4536 4537 /* 4538 * DO_ATTACH means we're here from enqueue_entity(). 4539 * !last_update_time means we've passed through 4540 * migrate_task_rq_fair() indicating we migrated. 4541 * 4542 * IOW we're enqueueing a task on a new CPU. 4543 */ 4544 attach_entity_load_avg(cfs_rq, se); 4545 update_tg_load_avg(cfs_rq); 4546 4547 } else if (flags & DO_DETACH) { 4548 /* 4549 * DO_DETACH means we're here from dequeue_entity() 4550 * and we are migrating task out of the CPU. 4551 */ 4552 detach_entity_load_avg(cfs_rq, se); 4553 update_tg_load_avg(cfs_rq); 4554 } else if (decayed) { 4555 cfs_rq_util_change(cfs_rq, 0); 4556 4557 if (flags & UPDATE_TG) 4558 update_tg_load_avg(cfs_rq); 4559 } 4560 } 4561 4562 /* 4563 * Synchronize entity load avg of dequeued entity without locking 4564 * the previous rq. 4565 */ 4566 static void sync_entity_load_avg(struct sched_entity *se) 4567 { 4568 struct cfs_rq *cfs_rq = cfs_rq_of(se); 4569 u64 last_update_time; 4570 4571 last_update_time = cfs_rq_last_update_time(cfs_rq); 4572 __update_load_avg_blocked_se(last_update_time, se); 4573 } 4574 4575 /* 4576 * Task first catches up with cfs_rq, and then subtract 4577 * itself from the cfs_rq (task must be off the queue now). 4578 */ 4579 static void remove_entity_load_avg(struct sched_entity *se) 4580 { 4581 struct cfs_rq *cfs_rq = cfs_rq_of(se); 4582 unsigned long flags; 4583 4584 /* 4585 * tasks cannot exit without having gone through wake_up_new_task() -> 4586 * enqueue_task_fair() which will have added things to the cfs_rq, 4587 * so we can remove unconditionally. 4588 */ 4589 4590 sync_entity_load_avg(se); 4591 4592 raw_spin_lock_irqsave(&cfs_rq->removed.lock, flags); 4593 ++cfs_rq->removed.nr; 4594 cfs_rq->removed.util_avg += se->avg.util_avg; 4595 cfs_rq->removed.load_avg += se->avg.load_avg; 4596 cfs_rq->removed.runnable_avg += se->avg.runnable_avg; 4597 raw_spin_unlock_irqrestore(&cfs_rq->removed.lock, flags); 4598 } 4599 4600 static inline unsigned long cfs_rq_runnable_avg(struct cfs_rq *cfs_rq) 4601 { 4602 return cfs_rq->avg.runnable_avg; 4603 } 4604 4605 static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq) 4606 { 4607 return cfs_rq->avg.load_avg; 4608 } 4609 4610 static int newidle_balance(struct rq *this_rq, struct rq_flags *rf); 4611 4612 static inline unsigned long task_util(struct task_struct *p) 4613 { 4614 return READ_ONCE(p->se.avg.util_avg); 4615 } 4616 4617 static inline unsigned long _task_util_est(struct task_struct *p) 4618 { 4619 struct util_est ue = READ_ONCE(p->se.avg.util_est); 4620 4621 return max(ue.ewma, (ue.enqueued & ~UTIL_AVG_UNCHANGED)); 4622 } 4623 4624 static inline unsigned long task_util_est(struct task_struct *p) 4625 { 4626 return max(task_util(p), _task_util_est(p)); 4627 } 4628 4629 #ifdef CONFIG_UCLAMP_TASK 4630 static inline unsigned long uclamp_task_util(struct task_struct *p, 4631 unsigned long uclamp_min, 4632 unsigned long uclamp_max) 4633 { 4634 return clamp(task_util_est(p), uclamp_min, uclamp_max); 4635 } 4636 #else 4637 static inline unsigned long uclamp_task_util(struct task_struct *p, 4638 unsigned long uclamp_min, 4639 unsigned long uclamp_max) 4640 { 4641 return task_util_est(p); 4642 } 4643 #endif 4644 4645 static inline void util_est_enqueue(struct cfs_rq *cfs_rq, 4646 struct task_struct *p) 4647 { 4648 unsigned int enqueued; 4649 4650 if (!sched_feat(UTIL_EST)) 4651 return; 4652 4653 /* Update root cfs_rq's estimated utilization */ 4654 enqueued = cfs_rq->avg.util_est.enqueued; 4655 enqueued += _task_util_est(p); 4656 WRITE_ONCE(cfs_rq->avg.util_est.enqueued, enqueued); 4657 4658 trace_sched_util_est_cfs_tp(cfs_rq); 4659 } 4660 4661 static inline void util_est_dequeue(struct cfs_rq *cfs_rq, 4662 struct task_struct *p) 4663 { 4664 unsigned int enqueued; 4665 4666 if (!sched_feat(UTIL_EST)) 4667 return; 4668 4669 /* Update root cfs_rq's estimated utilization */ 4670 enqueued = cfs_rq->avg.util_est.enqueued; 4671 enqueued -= min_t(unsigned int, enqueued, _task_util_est(p)); 4672 WRITE_ONCE(cfs_rq->avg.util_est.enqueued, enqueued); 4673 4674 trace_sched_util_est_cfs_tp(cfs_rq); 4675 } 4676 4677 #define UTIL_EST_MARGIN (SCHED_CAPACITY_SCALE / 100) 4678 4679 /* 4680 * Check if a (signed) value is within a specified (unsigned) margin, 4681 * based on the observation that: 4682 * 4683 * abs(x) < y := (unsigned)(x + y - 1) < (2 * y - 1) 4684 * 4685 * NOTE: this only works when value + margin < INT_MAX. 4686 */ 4687 static inline bool within_margin(int value, int margin) 4688 { 4689 return ((unsigned int)(value + margin - 1) < (2 * margin - 1)); 4690 } 4691 4692 static inline void util_est_update(struct cfs_rq *cfs_rq, 4693 struct task_struct *p, 4694 bool task_sleep) 4695 { 4696 long last_ewma_diff, last_enqueued_diff; 4697 struct util_est ue; 4698 4699 if (!sched_feat(UTIL_EST)) 4700 return; 4701 4702 /* 4703 * Skip update of task's estimated utilization when the task has not 4704 * yet completed an activation, e.g. being migrated. 4705 */ 4706 if (!task_sleep) 4707 return; 4708 4709 /* 4710 * If the PELT values haven't changed since enqueue time, 4711 * skip the util_est update. 4712 */ 4713 ue = p->se.avg.util_est; 4714 if (ue.enqueued & UTIL_AVG_UNCHANGED) 4715 return; 4716 4717 last_enqueued_diff = ue.enqueued; 4718 4719 /* 4720 * Reset EWMA on utilization increases, the moving average is used only 4721 * to smooth utilization decreases. 4722 */ 4723 ue.enqueued = task_util(p); 4724 if (sched_feat(UTIL_EST_FASTUP)) { 4725 if (ue.ewma < ue.enqueued) { 4726 ue.ewma = ue.enqueued; 4727 goto done; 4728 } 4729 } 4730 4731 /* 4732 * Skip update of task's estimated utilization when its members are 4733 * already ~1% close to its last activation value. 4734 */ 4735 last_ewma_diff = ue.enqueued - ue.ewma; 4736 last_enqueued_diff -= ue.enqueued; 4737 if (within_margin(last_ewma_diff, UTIL_EST_MARGIN)) { 4738 if (!within_margin(last_enqueued_diff, UTIL_EST_MARGIN)) 4739 goto done; 4740 4741 return; 4742 } 4743 4744 /* 4745 * To avoid overestimation of actual task utilization, skip updates if 4746 * we cannot grant there is idle time in this CPU. 4747 */ 4748 if (task_util(p) > capacity_orig_of(cpu_of(rq_of(cfs_rq)))) 4749 return; 4750 4751 /* 4752 * Update Task's estimated utilization 4753 * 4754 * When *p completes an activation we can consolidate another sample 4755 * of the task size. This is done by storing the current PELT value 4756 * as ue.enqueued and by using this value to update the Exponential 4757 * Weighted Moving Average (EWMA): 4758 * 4759 * ewma(t) = w * task_util(p) + (1-w) * ewma(t-1) 4760 * = w * task_util(p) + ewma(t-1) - w * ewma(t-1) 4761 * = w * (task_util(p) - ewma(t-1)) + ewma(t-1) 4762 * = w * ( last_ewma_diff ) + ewma(t-1) 4763 * = w * (last_ewma_diff + ewma(t-1) / w) 4764 * 4765 * Where 'w' is the weight of new samples, which is configured to be 4766 * 0.25, thus making w=1/4 ( >>= UTIL_EST_WEIGHT_SHIFT) 4767 */ 4768 ue.ewma <<= UTIL_EST_WEIGHT_SHIFT; 4769 ue.ewma += last_ewma_diff; 4770 ue.ewma >>= UTIL_EST_WEIGHT_SHIFT; 4771 done: 4772 ue.enqueued |= UTIL_AVG_UNCHANGED; 4773 WRITE_ONCE(p->se.avg.util_est, ue); 4774 4775 trace_sched_util_est_se_tp(&p->se); 4776 } 4777 4778 static inline int util_fits_cpu(unsigned long util, 4779 unsigned long uclamp_min, 4780 unsigned long uclamp_max, 4781 int cpu) 4782 { 4783 unsigned long capacity_orig, capacity_orig_thermal; 4784 unsigned long capacity = capacity_of(cpu); 4785 bool fits, uclamp_max_fits; 4786 4787 /* 4788 * Check if the real util fits without any uclamp boost/cap applied. 4789 */ 4790 fits = fits_capacity(util, capacity); 4791 4792 if (!uclamp_is_used()) 4793 return fits; 4794 4795 /* 4796 * We must use capacity_orig_of() for comparing against uclamp_min and 4797 * uclamp_max. We only care about capacity pressure (by using 4798 * capacity_of()) for comparing against the real util. 4799 * 4800 * If a task is boosted to 1024 for example, we don't want a tiny 4801 * pressure to skew the check whether it fits a CPU or not. 4802 * 4803 * Similarly if a task is capped to capacity_orig_of(little_cpu), it 4804 * should fit a little cpu even if there's some pressure. 4805 * 4806 * Only exception is for thermal pressure since it has a direct impact 4807 * on available OPP of the system. 4808 * 4809 * We honour it for uclamp_min only as a drop in performance level 4810 * could result in not getting the requested minimum performance level. 4811 * 4812 * For uclamp_max, we can tolerate a drop in performance level as the 4813 * goal is to cap the task. So it's okay if it's getting less. 4814 */ 4815 capacity_orig = capacity_orig_of(cpu); 4816 capacity_orig_thermal = capacity_orig - arch_scale_thermal_pressure(cpu); 4817 4818 /* 4819 * We want to force a task to fit a cpu as implied by uclamp_max. 4820 * But we do have some corner cases to cater for.. 4821 * 4822 * 4823 * C=z 4824 * | ___ 4825 * | C=y | | 4826 * |_ _ _ _ _ _ _ _ _ ___ _ _ _ | _ | _ _ _ _ _ uclamp_max 4827 * | C=x | | | | 4828 * | ___ | | | | 4829 * | | | | | | | (util somewhere in this region) 4830 * | | | | | | | 4831 * | | | | | | | 4832 * +---------------------------------------- 4833 * cpu0 cpu1 cpu2 4834 * 4835 * In the above example if a task is capped to a specific performance 4836 * point, y, then when: 4837 * 4838 * * util = 80% of x then it does not fit on cpu0 and should migrate 4839 * to cpu1 4840 * * util = 80% of y then it is forced to fit on cpu1 to honour 4841 * uclamp_max request. 4842 * 4843 * which is what we're enforcing here. A task always fits if 4844 * uclamp_max <= capacity_orig. But when uclamp_max > capacity_orig, 4845 * the normal upmigration rules should withhold still. 4846 * 4847 * Only exception is when we are on max capacity, then we need to be 4848 * careful not to block overutilized state. This is so because: 4849 * 4850 * 1. There's no concept of capping at max_capacity! We can't go 4851 * beyond this performance level anyway. 4852 * 2. The system is being saturated when we're operating near 4853 * max capacity, it doesn't make sense to block overutilized. 4854 */ 4855 uclamp_max_fits = (capacity_orig == SCHED_CAPACITY_SCALE) && (uclamp_max == SCHED_CAPACITY_SCALE); 4856 uclamp_max_fits = !uclamp_max_fits && (uclamp_max <= capacity_orig); 4857 fits = fits || uclamp_max_fits; 4858 4859 /* 4860 * 4861 * C=z 4862 * | ___ (region a, capped, util >= uclamp_max) 4863 * | C=y | | 4864 * |_ _ _ _ _ _ _ _ _ ___ _ _ _ | _ | _ _ _ _ _ uclamp_max 4865 * | C=x | | | | 4866 * | ___ | | | | (region b, uclamp_min <= util <= uclamp_max) 4867 * |_ _ _|_ _|_ _ _ _| _ | _ _ _| _ | _ _ _ _ _ uclamp_min 4868 * | | | | | | | 4869 * | | | | | | | (region c, boosted, util < uclamp_min) 4870 * +---------------------------------------- 4871 * cpu0 cpu1 cpu2 4872 * 4873 * a) If util > uclamp_max, then we're capped, we don't care about 4874 * actual fitness value here. We only care if uclamp_max fits 4875 * capacity without taking margin/pressure into account. 4876 * See comment above. 4877 * 4878 * b) If uclamp_min <= util <= uclamp_max, then the normal 4879 * fits_capacity() rules apply. Except we need to ensure that we 4880 * enforce we remain within uclamp_max, see comment above. 4881 * 4882 * c) If util < uclamp_min, then we are boosted. Same as (b) but we 4883 * need to take into account the boosted value fits the CPU without 4884 * taking margin/pressure into account. 4885 * 4886 * Cases (a) and (b) are handled in the 'fits' variable already. We 4887 * just need to consider an extra check for case (c) after ensuring we 4888 * handle the case uclamp_min > uclamp_max. 4889 */ 4890 uclamp_min = min(uclamp_min, uclamp_max); 4891 if (fits && (util < uclamp_min) && (uclamp_min > capacity_orig_thermal)) 4892 return -1; 4893 4894 return fits; 4895 } 4896 4897 static inline int task_fits_cpu(struct task_struct *p, int cpu) 4898 { 4899 unsigned long uclamp_min = uclamp_eff_value(p, UCLAMP_MIN); 4900 unsigned long uclamp_max = uclamp_eff_value(p, UCLAMP_MAX); 4901 unsigned long util = task_util_est(p); 4902 /* 4903 * Return true only if the cpu fully fits the task requirements, which 4904 * include the utilization but also the performance hints. 4905 */ 4906 return (util_fits_cpu(util, uclamp_min, uclamp_max, cpu) > 0); 4907 } 4908 4909 static inline void update_misfit_status(struct task_struct *p, struct rq *rq) 4910 { 4911 if (!sched_asym_cpucap_active()) 4912 return; 4913 4914 if (!p || p->nr_cpus_allowed == 1) { 4915 rq->misfit_task_load = 0; 4916 return; 4917 } 4918 4919 if (task_fits_cpu(p, cpu_of(rq))) { 4920 rq->misfit_task_load = 0; 4921 return; 4922 } 4923 4924 /* 4925 * Make sure that misfit_task_load will not be null even if 4926 * task_h_load() returns 0. 4927 */ 4928 rq->misfit_task_load = max_t(unsigned long, task_h_load(p), 1); 4929 } 4930 4931 #else /* CONFIG_SMP */ 4932 4933 static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq) 4934 { 4935 return true; 4936 } 4937 4938 #define UPDATE_TG 0x0 4939 #define SKIP_AGE_LOAD 0x0 4940 #define DO_ATTACH 0x0 4941 #define DO_DETACH 0x0 4942 4943 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int not_used1) 4944 { 4945 cfs_rq_util_change(cfs_rq, 0); 4946 } 4947 4948 static inline void remove_entity_load_avg(struct sched_entity *se) {} 4949 4950 static inline void 4951 attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} 4952 static inline void 4953 detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} 4954 4955 static inline int newidle_balance(struct rq *rq, struct rq_flags *rf) 4956 { 4957 return 0; 4958 } 4959 4960 static inline void 4961 util_est_enqueue(struct cfs_rq *cfs_rq, struct task_struct *p) {} 4962 4963 static inline void 4964 util_est_dequeue(struct cfs_rq *cfs_rq, struct task_struct *p) {} 4965 4966 static inline void 4967 util_est_update(struct cfs_rq *cfs_rq, struct task_struct *p, 4968 bool task_sleep) {} 4969 static inline void update_misfit_status(struct task_struct *p, struct rq *rq) {} 4970 4971 #endif /* CONFIG_SMP */ 4972 4973 static void 4974 place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 4975 { 4976 u64 vslice, vruntime = avg_vruntime(cfs_rq); 4977 s64 lag = 0; 4978 4979 se->slice = sysctl_sched_base_slice; 4980 vslice = calc_delta_fair(se->slice, se); 4981 4982 /* 4983 * Due to how V is constructed as the weighted average of entities, 4984 * adding tasks with positive lag, or removing tasks with negative lag 4985 * will move 'time' backwards, this can screw around with the lag of 4986 * other tasks. 4987 * 4988 * EEVDF: placement strategy #1 / #2 4989 */ 4990 if (sched_feat(PLACE_LAG) && cfs_rq->nr_running) { 4991 struct sched_entity *curr = cfs_rq->curr; 4992 unsigned long load; 4993 4994 lag = se->vlag; 4995 4996 /* 4997 * If we want to place a task and preserve lag, we have to 4998 * consider the effect of the new entity on the weighted 4999 * average and compensate for this, otherwise lag can quickly 5000 * evaporate. 5001 * 5002 * Lag is defined as: 5003 * 5004 * lag_i = S - s_i = w_i * (V - v_i) 5005 * 5006 * To avoid the 'w_i' term all over the place, we only track 5007 * the virtual lag: 5008 * 5009 * vl_i = V - v_i <=> v_i = V - vl_i 5010 * 5011 * And we take V to be the weighted average of all v: 5012 * 5013 * V = (\Sum w_j*v_j) / W 5014 * 5015 * Where W is: \Sum w_j 5016 * 5017 * Then, the weighted average after adding an entity with lag 5018 * vl_i is given by: 5019 * 5020 * V' = (\Sum w_j*v_j + w_i*v_i) / (W + w_i) 5021 * = (W*V + w_i*(V - vl_i)) / (W + w_i) 5022 * = (W*V + w_i*V - w_i*vl_i) / (W + w_i) 5023 * = (V*(W + w_i) - w_i*l) / (W + w_i) 5024 * = V - w_i*vl_i / (W + w_i) 5025 * 5026 * And the actual lag after adding an entity with vl_i is: 5027 * 5028 * vl'_i = V' - v_i 5029 * = V - w_i*vl_i / (W + w_i) - (V - vl_i) 5030 * = vl_i - w_i*vl_i / (W + w_i) 5031 * 5032 * Which is strictly less than vl_i. So in order to preserve lag 5033 * we should inflate the lag before placement such that the 5034 * effective lag after placement comes out right. 5035 * 5036 * As such, invert the above relation for vl'_i to get the vl_i 5037 * we need to use such that the lag after placement is the lag 5038 * we computed before dequeue. 5039 * 5040 * vl'_i = vl_i - w_i*vl_i / (W + w_i) 5041 * = ((W + w_i)*vl_i - w_i*vl_i) / (W + w_i) 5042 * 5043 * (W + w_i)*vl'_i = (W + w_i)*vl_i - w_i*vl_i 5044 * = W*vl_i 5045 * 5046 * vl_i = (W + w_i)*vl'_i / W 5047 */ 5048 load = cfs_rq->avg_load; 5049 if (curr && curr->on_rq) 5050 load += scale_load_down(curr->load.weight); 5051 5052 lag *= load + scale_load_down(se->load.weight); 5053 if (WARN_ON_ONCE(!load)) 5054 load = 1; 5055 lag = div_s64(lag, load); 5056 } 5057 5058 se->vruntime = vruntime - lag; 5059 5060 /* 5061 * When joining the competition; the exisiting tasks will be, 5062 * on average, halfway through their slice, as such start tasks 5063 * off with half a slice to ease into the competition. 5064 */ 5065 if (sched_feat(PLACE_DEADLINE_INITIAL) && (flags & ENQUEUE_INITIAL)) 5066 vslice /= 2; 5067 5068 /* 5069 * EEVDF: vd_i = ve_i + r_i/w_i 5070 */ 5071 se->deadline = se->vruntime + vslice; 5072 } 5073 5074 static void check_enqueue_throttle(struct cfs_rq *cfs_rq); 5075 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq); 5076 5077 static inline bool cfs_bandwidth_used(void); 5078 5079 static void 5080 enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 5081 { 5082 bool curr = cfs_rq->curr == se; 5083 5084 /* 5085 * If we're the current task, we must renormalise before calling 5086 * update_curr(). 5087 */ 5088 if (curr) 5089 place_entity(cfs_rq, se, flags); 5090 5091 update_curr(cfs_rq); 5092 5093 /* 5094 * When enqueuing a sched_entity, we must: 5095 * - Update loads to have both entity and cfs_rq synced with now. 5096 * - For group_entity, update its runnable_weight to reflect the new 5097 * h_nr_running of its group cfs_rq. 5098 * - For group_entity, update its weight to reflect the new share of 5099 * its group cfs_rq 5100 * - Add its new weight to cfs_rq->load.weight 5101 */ 5102 update_load_avg(cfs_rq, se, UPDATE_TG | DO_ATTACH); 5103 se_update_runnable(se); 5104 /* 5105 * XXX update_load_avg() above will have attached us to the pelt sum; 5106 * but update_cfs_group() here will re-adjust the weight and have to 5107 * undo/redo all that. Seems wasteful. 5108 */ 5109 update_cfs_group(se); 5110 5111 /* 5112 * XXX now that the entity has been re-weighted, and it's lag adjusted, 5113 * we can place the entity. 5114 */ 5115 if (!curr) 5116 place_entity(cfs_rq, se, flags); 5117 5118 account_entity_enqueue(cfs_rq, se); 5119 5120 /* Entity has migrated, no longer consider this task hot */ 5121 if (flags & ENQUEUE_MIGRATED) 5122 se->exec_start = 0; 5123 5124 check_schedstat_required(); 5125 update_stats_enqueue_fair(cfs_rq, se, flags); 5126 if (!curr) 5127 __enqueue_entity(cfs_rq, se); 5128 se->on_rq = 1; 5129 5130 if (cfs_rq->nr_running == 1) { 5131 check_enqueue_throttle(cfs_rq); 5132 if (!throttled_hierarchy(cfs_rq)) { 5133 list_add_leaf_cfs_rq(cfs_rq); 5134 } else { 5135 #ifdef CONFIG_CFS_BANDWIDTH 5136 struct rq *rq = rq_of(cfs_rq); 5137 5138 if (cfs_rq_throttled(cfs_rq) && !cfs_rq->throttled_clock) 5139 cfs_rq->throttled_clock = rq_clock(rq); 5140 if (!cfs_rq->throttled_clock_self) 5141 cfs_rq->throttled_clock_self = rq_clock(rq); 5142 #endif 5143 } 5144 } 5145 } 5146 5147 static void __clear_buddies_next(struct sched_entity *se) 5148 { 5149 for_each_sched_entity(se) { 5150 struct cfs_rq *cfs_rq = cfs_rq_of(se); 5151 if (cfs_rq->next != se) 5152 break; 5153 5154 cfs_rq->next = NULL; 5155 } 5156 } 5157 5158 static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) 5159 { 5160 if (cfs_rq->next == se) 5161 __clear_buddies_next(se); 5162 } 5163 5164 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq); 5165 5166 static void 5167 dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 5168 { 5169 int action = UPDATE_TG; 5170 5171 if (entity_is_task(se) && task_on_rq_migrating(task_of(se))) 5172 action |= DO_DETACH; 5173 5174 /* 5175 * Update run-time statistics of the 'current'. 5176 */ 5177 update_curr(cfs_rq); 5178 5179 /* 5180 * When dequeuing a sched_entity, we must: 5181 * - Update loads to have both entity and cfs_rq synced with now. 5182 * - For group_entity, update its runnable_weight to reflect the new 5183 * h_nr_running of its group cfs_rq. 5184 * - Subtract its previous weight from cfs_rq->load.weight. 5185 * - For group entity, update its weight to reflect the new share 5186 * of its group cfs_rq. 5187 */ 5188 update_load_avg(cfs_rq, se, action); 5189 se_update_runnable(se); 5190 5191 update_stats_dequeue_fair(cfs_rq, se, flags); 5192 5193 clear_buddies(cfs_rq, se); 5194 5195 update_entity_lag(cfs_rq, se); 5196 if (se != cfs_rq->curr) 5197 __dequeue_entity(cfs_rq, se); 5198 se->on_rq = 0; 5199 account_entity_dequeue(cfs_rq, se); 5200 5201 /* return excess runtime on last dequeue */ 5202 return_cfs_rq_runtime(cfs_rq); 5203 5204 update_cfs_group(se); 5205 5206 /* 5207 * Now advance min_vruntime if @se was the entity holding it back, 5208 * except when: DEQUEUE_SAVE && !DEQUEUE_MOVE, in this case we'll be 5209 * put back on, and if we advance min_vruntime, we'll be placed back 5210 * further than we started -- ie. we'll be penalized. 5211 */ 5212 if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) != DEQUEUE_SAVE) 5213 update_min_vruntime(cfs_rq); 5214 5215 if (cfs_rq->nr_running == 0) 5216 update_idle_cfs_rq_clock_pelt(cfs_rq); 5217 } 5218 5219 static void 5220 set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 5221 { 5222 clear_buddies(cfs_rq, se); 5223 5224 /* 'current' is not kept within the tree. */ 5225 if (se->on_rq) { 5226 /* 5227 * Any task has to be enqueued before it get to execute on 5228 * a CPU. So account for the time it spent waiting on the 5229 * runqueue. 5230 */ 5231 update_stats_wait_end_fair(cfs_rq, se); 5232 __dequeue_entity(cfs_rq, se); 5233 update_load_avg(cfs_rq, se, UPDATE_TG); 5234 /* 5235 * HACK, stash a copy of deadline at the point of pick in vlag, 5236 * which isn't used until dequeue. 5237 */ 5238 se->vlag = se->deadline; 5239 } 5240 5241 update_stats_curr_start(cfs_rq, se); 5242 cfs_rq->curr = se; 5243 5244 /* 5245 * Track our maximum slice length, if the CPU's load is at 5246 * least twice that of our own weight (i.e. dont track it 5247 * when there are only lesser-weight tasks around): 5248 */ 5249 if (schedstat_enabled() && 5250 rq_of(cfs_rq)->cfs.load.weight >= 2*se->load.weight) { 5251 struct sched_statistics *stats; 5252 5253 stats = __schedstats_from_se(se); 5254 __schedstat_set(stats->slice_max, 5255 max((u64)stats->slice_max, 5256 se->sum_exec_runtime - se->prev_sum_exec_runtime)); 5257 } 5258 5259 se->prev_sum_exec_runtime = se->sum_exec_runtime; 5260 } 5261 5262 /* 5263 * Pick the next process, keeping these things in mind, in this order: 5264 * 1) keep things fair between processes/task groups 5265 * 2) pick the "next" process, since someone really wants that to run 5266 * 3) pick the "last" process, for cache locality 5267 * 4) do not run the "skip" process, if something else is available 5268 */ 5269 static struct sched_entity * 5270 pick_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *curr) 5271 { 5272 /* 5273 * Enabling NEXT_BUDDY will affect latency but not fairness. 5274 */ 5275 if (sched_feat(NEXT_BUDDY) && 5276 cfs_rq->next && entity_eligible(cfs_rq, cfs_rq->next)) 5277 return cfs_rq->next; 5278 5279 return pick_eevdf(cfs_rq); 5280 } 5281 5282 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq); 5283 5284 static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) 5285 { 5286 /* 5287 * If still on the runqueue then deactivate_task() 5288 * was not called and update_curr() has to be done: 5289 */ 5290 if (prev->on_rq) 5291 update_curr(cfs_rq); 5292 5293 /* throttle cfs_rqs exceeding runtime */ 5294 check_cfs_rq_runtime(cfs_rq); 5295 5296 if (prev->on_rq) { 5297 update_stats_wait_start_fair(cfs_rq, prev); 5298 /* Put 'current' back into the tree. */ 5299 __enqueue_entity(cfs_rq, prev); 5300 /* in !on_rq case, update occurred at dequeue */ 5301 update_load_avg(cfs_rq, prev, 0); 5302 } 5303 cfs_rq->curr = NULL; 5304 } 5305 5306 static void 5307 entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) 5308 { 5309 /* 5310 * Update run-time statistics of the 'current'. 5311 */ 5312 update_curr(cfs_rq); 5313 5314 /* 5315 * Ensure that runnable average is periodically updated. 5316 */ 5317 update_load_avg(cfs_rq, curr, UPDATE_TG); 5318 update_cfs_group(curr); 5319 5320 #ifdef CONFIG_SCHED_HRTICK 5321 /* 5322 * queued ticks are scheduled to match the slice, so don't bother 5323 * validating it and just reschedule. 5324 */ 5325 if (queued) { 5326 resched_curr(rq_of(cfs_rq)); 5327 return; 5328 } 5329 /* 5330 * don't let the period tick interfere with the hrtick preemption 5331 */ 5332 if (!sched_feat(DOUBLE_TICK) && 5333 hrtimer_active(&rq_of(cfs_rq)->hrtick_timer)) 5334 return; 5335 #endif 5336 } 5337 5338 5339 /************************************************** 5340 * CFS bandwidth control machinery 5341 */ 5342 5343 #ifdef CONFIG_CFS_BANDWIDTH 5344 5345 #ifdef CONFIG_JUMP_LABEL 5346 static struct static_key __cfs_bandwidth_used; 5347 5348 static inline bool cfs_bandwidth_used(void) 5349 { 5350 return static_key_false(&__cfs_bandwidth_used); 5351 } 5352 5353 void cfs_bandwidth_usage_inc(void) 5354 { 5355 static_key_slow_inc_cpuslocked(&__cfs_bandwidth_used); 5356 } 5357 5358 void cfs_bandwidth_usage_dec(void) 5359 { 5360 static_key_slow_dec_cpuslocked(&__cfs_bandwidth_used); 5361 } 5362 #else /* CONFIG_JUMP_LABEL */ 5363 static bool cfs_bandwidth_used(void) 5364 { 5365 return true; 5366 } 5367 5368 void cfs_bandwidth_usage_inc(void) {} 5369 void cfs_bandwidth_usage_dec(void) {} 5370 #endif /* CONFIG_JUMP_LABEL */ 5371 5372 /* 5373 * default period for cfs group bandwidth. 5374 * default: 0.1s, units: nanoseconds 5375 */ 5376 static inline u64 default_cfs_period(void) 5377 { 5378 return 100000000ULL; 5379 } 5380 5381 static inline u64 sched_cfs_bandwidth_slice(void) 5382 { 5383 return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC; 5384 } 5385 5386 /* 5387 * Replenish runtime according to assigned quota. We use sched_clock_cpu 5388 * directly instead of rq->clock to avoid adding additional synchronization 5389 * around rq->lock. 5390 * 5391 * requires cfs_b->lock 5392 */ 5393 void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b) 5394 { 5395 s64 runtime; 5396 5397 if (unlikely(cfs_b->quota == RUNTIME_INF)) 5398 return; 5399 5400 cfs_b->runtime += cfs_b->quota; 5401 runtime = cfs_b->runtime_snap - cfs_b->runtime; 5402 if (runtime > 0) { 5403 cfs_b->burst_time += runtime; 5404 cfs_b->nr_burst++; 5405 } 5406 5407 cfs_b->runtime = min(cfs_b->runtime, cfs_b->quota + cfs_b->burst); 5408 cfs_b->runtime_snap = cfs_b->runtime; 5409 } 5410 5411 static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) 5412 { 5413 return &tg->cfs_bandwidth; 5414 } 5415 5416 /* returns 0 on failure to allocate runtime */ 5417 static int __assign_cfs_rq_runtime(struct cfs_bandwidth *cfs_b, 5418 struct cfs_rq *cfs_rq, u64 target_runtime) 5419 { 5420 u64 min_amount, amount = 0; 5421 5422 lockdep_assert_held(&cfs_b->lock); 5423 5424 /* note: this is a positive sum as runtime_remaining <= 0 */ 5425 min_amount = target_runtime - cfs_rq->runtime_remaining; 5426 5427 if (cfs_b->quota == RUNTIME_INF) 5428 amount = min_amount; 5429 else { 5430 start_cfs_bandwidth(cfs_b); 5431 5432 if (cfs_b->runtime > 0) { 5433 amount = min(cfs_b->runtime, min_amount); 5434 cfs_b->runtime -= amount; 5435 cfs_b->idle = 0; 5436 } 5437 } 5438 5439 cfs_rq->runtime_remaining += amount; 5440 5441 return cfs_rq->runtime_remaining > 0; 5442 } 5443 5444 /* returns 0 on failure to allocate runtime */ 5445 static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq) 5446 { 5447 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 5448 int ret; 5449 5450 raw_spin_lock(&cfs_b->lock); 5451 ret = __assign_cfs_rq_runtime(cfs_b, cfs_rq, sched_cfs_bandwidth_slice()); 5452 raw_spin_unlock(&cfs_b->lock); 5453 5454 return ret; 5455 } 5456 5457 static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) 5458 { 5459 /* dock delta_exec before expiring quota (as it could span periods) */ 5460 cfs_rq->runtime_remaining -= delta_exec; 5461 5462 if (likely(cfs_rq->runtime_remaining > 0)) 5463 return; 5464 5465 if (cfs_rq->throttled) 5466 return; 5467 /* 5468 * if we're unable to extend our runtime we resched so that the active 5469 * hierarchy can be throttled 5470 */ 5471 if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr)) 5472 resched_curr(rq_of(cfs_rq)); 5473 } 5474 5475 static __always_inline 5476 void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) 5477 { 5478 if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled) 5479 return; 5480 5481 __account_cfs_rq_runtime(cfs_rq, delta_exec); 5482 } 5483 5484 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) 5485 { 5486 return cfs_bandwidth_used() && cfs_rq->throttled; 5487 } 5488 5489 /* check whether cfs_rq, or any parent, is throttled */ 5490 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) 5491 { 5492 return cfs_bandwidth_used() && cfs_rq->throttle_count; 5493 } 5494 5495 /* 5496 * Ensure that neither of the group entities corresponding to src_cpu or 5497 * dest_cpu are members of a throttled hierarchy when performing group 5498 * load-balance operations. 5499 */ 5500 static inline int throttled_lb_pair(struct task_group *tg, 5501 int src_cpu, int dest_cpu) 5502 { 5503 struct cfs_rq *src_cfs_rq, *dest_cfs_rq; 5504 5505 src_cfs_rq = tg->cfs_rq[src_cpu]; 5506 dest_cfs_rq = tg->cfs_rq[dest_cpu]; 5507 5508 return throttled_hierarchy(src_cfs_rq) || 5509 throttled_hierarchy(dest_cfs_rq); 5510 } 5511 5512 static int tg_unthrottle_up(struct task_group *tg, void *data) 5513 { 5514 struct rq *rq = data; 5515 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 5516 5517 cfs_rq->throttle_count--; 5518 if (!cfs_rq->throttle_count) { 5519 cfs_rq->throttled_clock_pelt_time += rq_clock_pelt(rq) - 5520 cfs_rq->throttled_clock_pelt; 5521 5522 /* Add cfs_rq with load or one or more already running entities to the list */ 5523 if (!cfs_rq_is_decayed(cfs_rq)) 5524 list_add_leaf_cfs_rq(cfs_rq); 5525 5526 if (cfs_rq->throttled_clock_self) { 5527 u64 delta = rq_clock(rq) - cfs_rq->throttled_clock_self; 5528 5529 cfs_rq->throttled_clock_self = 0; 5530 5531 if (SCHED_WARN_ON((s64)delta < 0)) 5532 delta = 0; 5533 5534 cfs_rq->throttled_clock_self_time += delta; 5535 } 5536 } 5537 5538 return 0; 5539 } 5540 5541 static int tg_throttle_down(struct task_group *tg, void *data) 5542 { 5543 struct rq *rq = data; 5544 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 5545 5546 /* group is entering throttled state, stop time */ 5547 if (!cfs_rq->throttle_count) { 5548 cfs_rq->throttled_clock_pelt = rq_clock_pelt(rq); 5549 list_del_leaf_cfs_rq(cfs_rq); 5550 5551 SCHED_WARN_ON(cfs_rq->throttled_clock_self); 5552 if (cfs_rq->nr_running) 5553 cfs_rq->throttled_clock_self = rq_clock(rq); 5554 } 5555 cfs_rq->throttle_count++; 5556 5557 return 0; 5558 } 5559 5560 static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) 5561 { 5562 struct rq *rq = rq_of(cfs_rq); 5563 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 5564 struct sched_entity *se; 5565 long task_delta, idle_task_delta, dequeue = 1; 5566 5567 raw_spin_lock(&cfs_b->lock); 5568 /* This will start the period timer if necessary */ 5569 if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, 1)) { 5570 /* 5571 * We have raced with bandwidth becoming available, and if we 5572 * actually throttled the timer might not unthrottle us for an 5573 * entire period. We additionally needed to make sure that any 5574 * subsequent check_cfs_rq_runtime calls agree not to throttle 5575 * us, as we may commit to do cfs put_prev+pick_next, so we ask 5576 * for 1ns of runtime rather than just check cfs_b. 5577 */ 5578 dequeue = 0; 5579 } else { 5580 list_add_tail_rcu(&cfs_rq->throttled_list, 5581 &cfs_b->throttled_cfs_rq); 5582 } 5583 raw_spin_unlock(&cfs_b->lock); 5584 5585 if (!dequeue) 5586 return false; /* Throttle no longer required. */ 5587 5588 se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))]; 5589 5590 /* freeze hierarchy runnable averages while throttled */ 5591 rcu_read_lock(); 5592 walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq); 5593 rcu_read_unlock(); 5594 5595 task_delta = cfs_rq->h_nr_running; 5596 idle_task_delta = cfs_rq->idle_h_nr_running; 5597 for_each_sched_entity(se) { 5598 struct cfs_rq *qcfs_rq = cfs_rq_of(se); 5599 /* throttled entity or throttle-on-deactivate */ 5600 if (!se->on_rq) 5601 goto done; 5602 5603 dequeue_entity(qcfs_rq, se, DEQUEUE_SLEEP); 5604 5605 if (cfs_rq_is_idle(group_cfs_rq(se))) 5606 idle_task_delta = cfs_rq->h_nr_running; 5607 5608 qcfs_rq->h_nr_running -= task_delta; 5609 qcfs_rq->idle_h_nr_running -= idle_task_delta; 5610 5611 if (qcfs_rq->load.weight) { 5612 /* Avoid re-evaluating load for this entity: */ 5613 se = parent_entity(se); 5614 break; 5615 } 5616 } 5617 5618 for_each_sched_entity(se) { 5619 struct cfs_rq *qcfs_rq = cfs_rq_of(se); 5620 /* throttled entity or throttle-on-deactivate */ 5621 if (!se->on_rq) 5622 goto done; 5623 5624 update_load_avg(qcfs_rq, se, 0); 5625 se_update_runnable(se); 5626 5627 if (cfs_rq_is_idle(group_cfs_rq(se))) 5628 idle_task_delta = cfs_rq->h_nr_running; 5629 5630 qcfs_rq->h_nr_running -= task_delta; 5631 qcfs_rq->idle_h_nr_running -= idle_task_delta; 5632 } 5633 5634 /* At this point se is NULL and we are at root level*/ 5635 sub_nr_running(rq, task_delta); 5636 5637 done: 5638 /* 5639 * Note: distribution will already see us throttled via the 5640 * throttled-list. rq->lock protects completion. 5641 */ 5642 cfs_rq->throttled = 1; 5643 SCHED_WARN_ON(cfs_rq->throttled_clock); 5644 if (cfs_rq->nr_running) 5645 cfs_rq->throttled_clock = rq_clock(rq); 5646 return true; 5647 } 5648 5649 void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) 5650 { 5651 struct rq *rq = rq_of(cfs_rq); 5652 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 5653 struct sched_entity *se; 5654 long task_delta, idle_task_delta; 5655 5656 se = cfs_rq->tg->se[cpu_of(rq)]; 5657 5658 cfs_rq->throttled = 0; 5659 5660 update_rq_clock(rq); 5661 5662 raw_spin_lock(&cfs_b->lock); 5663 if (cfs_rq->throttled_clock) { 5664 cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock; 5665 cfs_rq->throttled_clock = 0; 5666 } 5667 list_del_rcu(&cfs_rq->throttled_list); 5668 raw_spin_unlock(&cfs_b->lock); 5669 5670 /* update hierarchical throttle state */ 5671 walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq); 5672 5673 if (!cfs_rq->load.weight) { 5674 if (!cfs_rq->on_list) 5675 return; 5676 /* 5677 * Nothing to run but something to decay (on_list)? 5678 * Complete the branch. 5679 */ 5680 for_each_sched_entity(se) { 5681 if (list_add_leaf_cfs_rq(cfs_rq_of(se))) 5682 break; 5683 } 5684 goto unthrottle_throttle; 5685 } 5686 5687 task_delta = cfs_rq->h_nr_running; 5688 idle_task_delta = cfs_rq->idle_h_nr_running; 5689 for_each_sched_entity(se) { 5690 struct cfs_rq *qcfs_rq = cfs_rq_of(se); 5691 5692 if (se->on_rq) 5693 break; 5694 enqueue_entity(qcfs_rq, se, ENQUEUE_WAKEUP); 5695 5696 if (cfs_rq_is_idle(group_cfs_rq(se))) 5697 idle_task_delta = cfs_rq->h_nr_running; 5698 5699 qcfs_rq->h_nr_running += task_delta; 5700 qcfs_rq->idle_h_nr_running += idle_task_delta; 5701 5702 /* end evaluation on encountering a throttled cfs_rq */ 5703 if (cfs_rq_throttled(qcfs_rq)) 5704 goto unthrottle_throttle; 5705 } 5706 5707 for_each_sched_entity(se) { 5708 struct cfs_rq *qcfs_rq = cfs_rq_of(se); 5709 5710 update_load_avg(qcfs_rq, se, UPDATE_TG); 5711 se_update_runnable(se); 5712 5713 if (cfs_rq_is_idle(group_cfs_rq(se))) 5714 idle_task_delta = cfs_rq->h_nr_running; 5715 5716 qcfs_rq->h_nr_running += task_delta; 5717 qcfs_rq->idle_h_nr_running += idle_task_delta; 5718 5719 /* end evaluation on encountering a throttled cfs_rq */ 5720 if (cfs_rq_throttled(qcfs_rq)) 5721 goto unthrottle_throttle; 5722 } 5723 5724 /* At this point se is NULL and we are at root level*/ 5725 add_nr_running(rq, task_delta); 5726 5727 unthrottle_throttle: 5728 assert_list_leaf_cfs_rq(rq); 5729 5730 /* Determine whether we need to wake up potentially idle CPU: */ 5731 if (rq->curr == rq->idle && rq->cfs.nr_running) 5732 resched_curr(rq); 5733 } 5734 5735 #ifdef CONFIG_SMP 5736 static void __cfsb_csd_unthrottle(void *arg) 5737 { 5738 struct cfs_rq *cursor, *tmp; 5739 struct rq *rq = arg; 5740 struct rq_flags rf; 5741 5742 rq_lock(rq, &rf); 5743 5744 /* 5745 * Iterating over the list can trigger several call to 5746 * update_rq_clock() in unthrottle_cfs_rq(). 5747 * Do it once and skip the potential next ones. 5748 */ 5749 update_rq_clock(rq); 5750 rq_clock_start_loop_update(rq); 5751 5752 /* 5753 * Since we hold rq lock we're safe from concurrent manipulation of 5754 * the CSD list. However, this RCU critical section annotates the 5755 * fact that we pair with sched_free_group_rcu(), so that we cannot 5756 * race with group being freed in the window between removing it 5757 * from the list and advancing to the next entry in the list. 5758 */ 5759 rcu_read_lock(); 5760 5761 list_for_each_entry_safe(cursor, tmp, &rq->cfsb_csd_list, 5762 throttled_csd_list) { 5763 list_del_init(&cursor->throttled_csd_list); 5764 5765 if (cfs_rq_throttled(cursor)) 5766 unthrottle_cfs_rq(cursor); 5767 } 5768 5769 rcu_read_unlock(); 5770 5771 rq_clock_stop_loop_update(rq); 5772 rq_unlock(rq, &rf); 5773 } 5774 5775 static inline void __unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq) 5776 { 5777 struct rq *rq = rq_of(cfs_rq); 5778 bool first; 5779 5780 if (rq == this_rq()) { 5781 unthrottle_cfs_rq(cfs_rq); 5782 return; 5783 } 5784 5785 /* Already enqueued */ 5786 if (SCHED_WARN_ON(!list_empty(&cfs_rq->throttled_csd_list))) 5787 return; 5788 5789 first = list_empty(&rq->cfsb_csd_list); 5790 list_add_tail(&cfs_rq->throttled_csd_list, &rq->cfsb_csd_list); 5791 if (first) 5792 smp_call_function_single_async(cpu_of(rq), &rq->cfsb_csd); 5793 } 5794 #else 5795 static inline void __unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq) 5796 { 5797 unthrottle_cfs_rq(cfs_rq); 5798 } 5799 #endif 5800 5801 static void unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq) 5802 { 5803 lockdep_assert_rq_held(rq_of(cfs_rq)); 5804 5805 if (SCHED_WARN_ON(!cfs_rq_throttled(cfs_rq) || 5806 cfs_rq->runtime_remaining <= 0)) 5807 return; 5808 5809 __unthrottle_cfs_rq_async(cfs_rq); 5810 } 5811 5812 static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b) 5813 { 5814 struct cfs_rq *local_unthrottle = NULL; 5815 int this_cpu = smp_processor_id(); 5816 u64 runtime, remaining = 1; 5817 bool throttled = false; 5818 struct cfs_rq *cfs_rq; 5819 struct rq_flags rf; 5820 struct rq *rq; 5821 5822 rcu_read_lock(); 5823 list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq, 5824 throttled_list) { 5825 rq = rq_of(cfs_rq); 5826 5827 if (!remaining) { 5828 throttled = true; 5829 break; 5830 } 5831 5832 rq_lock_irqsave(rq, &rf); 5833 if (!cfs_rq_throttled(cfs_rq)) 5834 goto next; 5835 5836 #ifdef CONFIG_SMP 5837 /* Already queued for async unthrottle */ 5838 if (!list_empty(&cfs_rq->throttled_csd_list)) 5839 goto next; 5840 #endif 5841 5842 /* By the above checks, this should never be true */ 5843 SCHED_WARN_ON(cfs_rq->runtime_remaining > 0); 5844 5845 raw_spin_lock(&cfs_b->lock); 5846 runtime = -cfs_rq->runtime_remaining + 1; 5847 if (runtime > cfs_b->runtime) 5848 runtime = cfs_b->runtime; 5849 cfs_b->runtime -= runtime; 5850 remaining = cfs_b->runtime; 5851 raw_spin_unlock(&cfs_b->lock); 5852 5853 cfs_rq->runtime_remaining += runtime; 5854 5855 /* we check whether we're throttled above */ 5856 if (cfs_rq->runtime_remaining > 0) { 5857 if (cpu_of(rq) != this_cpu || 5858 SCHED_WARN_ON(local_unthrottle)) 5859 unthrottle_cfs_rq_async(cfs_rq); 5860 else 5861 local_unthrottle = cfs_rq; 5862 } else { 5863 throttled = true; 5864 } 5865 5866 next: 5867 rq_unlock_irqrestore(rq, &rf); 5868 } 5869 rcu_read_unlock(); 5870 5871 if (local_unthrottle) { 5872 rq = cpu_rq(this_cpu); 5873 rq_lock_irqsave(rq, &rf); 5874 if (cfs_rq_throttled(local_unthrottle)) 5875 unthrottle_cfs_rq(local_unthrottle); 5876 rq_unlock_irqrestore(rq, &rf); 5877 } 5878 5879 return throttled; 5880 } 5881 5882 /* 5883 * Responsible for refilling a task_group's bandwidth and unthrottling its 5884 * cfs_rqs as appropriate. If there has been no activity within the last 5885 * period the timer is deactivated until scheduling resumes; cfs_b->idle is 5886 * used to track this state. 5887 */ 5888 static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, unsigned long flags) 5889 { 5890 int throttled; 5891 5892 /* no need to continue the timer with no bandwidth constraint */ 5893 if (cfs_b->quota == RUNTIME_INF) 5894 goto out_deactivate; 5895 5896 throttled = !list_empty(&cfs_b->throttled_cfs_rq); 5897 cfs_b->nr_periods += overrun; 5898 5899 /* Refill extra burst quota even if cfs_b->idle */ 5900 __refill_cfs_bandwidth_runtime(cfs_b); 5901 5902 /* 5903 * idle depends on !throttled (for the case of a large deficit), and if 5904 * we're going inactive then everything else can be deferred 5905 */ 5906 if (cfs_b->idle && !throttled) 5907 goto out_deactivate; 5908 5909 if (!throttled) { 5910 /* mark as potentially idle for the upcoming period */ 5911 cfs_b->idle = 1; 5912 return 0; 5913 } 5914 5915 /* account preceding periods in which throttling occurred */ 5916 cfs_b->nr_throttled += overrun; 5917 5918 /* 5919 * This check is repeated as we release cfs_b->lock while we unthrottle. 5920 */ 5921 while (throttled && cfs_b->runtime > 0) { 5922 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 5923 /* we can't nest cfs_b->lock while distributing bandwidth */ 5924 throttled = distribute_cfs_runtime(cfs_b); 5925 raw_spin_lock_irqsave(&cfs_b->lock, flags); 5926 } 5927 5928 /* 5929 * While we are ensured activity in the period following an 5930 * unthrottle, this also covers the case in which the new bandwidth is 5931 * insufficient to cover the existing bandwidth deficit. (Forcing the 5932 * timer to remain active while there are any throttled entities.) 5933 */ 5934 cfs_b->idle = 0; 5935 5936 return 0; 5937 5938 out_deactivate: 5939 return 1; 5940 } 5941 5942 /* a cfs_rq won't donate quota below this amount */ 5943 static const u64 min_cfs_rq_runtime = 1 * NSEC_PER_MSEC; 5944 /* minimum remaining period time to redistribute slack quota */ 5945 static const u64 min_bandwidth_expiration = 2 * NSEC_PER_MSEC; 5946 /* how long we wait to gather additional slack before distributing */ 5947 static const u64 cfs_bandwidth_slack_period = 5 * NSEC_PER_MSEC; 5948 5949 /* 5950 * Are we near the end of the current quota period? 5951 * 5952 * Requires cfs_b->lock for hrtimer_expires_remaining to be safe against the 5953 * hrtimer base being cleared by hrtimer_start. In the case of 5954 * migrate_hrtimers, base is never cleared, so we are fine. 5955 */ 5956 static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire) 5957 { 5958 struct hrtimer *refresh_timer = &cfs_b->period_timer; 5959 s64 remaining; 5960 5961 /* if the call-back is running a quota refresh is already occurring */ 5962 if (hrtimer_callback_running(refresh_timer)) 5963 return 1; 5964 5965 /* is a quota refresh about to occur? */ 5966 remaining = ktime_to_ns(hrtimer_expires_remaining(refresh_timer)); 5967 if (remaining < (s64)min_expire) 5968 return 1; 5969 5970 return 0; 5971 } 5972 5973 static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b) 5974 { 5975 u64 min_left = cfs_bandwidth_slack_period + min_bandwidth_expiration; 5976 5977 /* if there's a quota refresh soon don't bother with slack */ 5978 if (runtime_refresh_within(cfs_b, min_left)) 5979 return; 5980 5981 /* don't push forwards an existing deferred unthrottle */ 5982 if (cfs_b->slack_started) 5983 return; 5984 cfs_b->slack_started = true; 5985 5986 hrtimer_start(&cfs_b->slack_timer, 5987 ns_to_ktime(cfs_bandwidth_slack_period), 5988 HRTIMER_MODE_REL); 5989 } 5990 5991 /* we know any runtime found here is valid as update_curr() precedes return */ 5992 static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq) 5993 { 5994 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 5995 s64 slack_runtime = cfs_rq->runtime_remaining - min_cfs_rq_runtime; 5996 5997 if (slack_runtime <= 0) 5998 return; 5999 6000 raw_spin_lock(&cfs_b->lock); 6001 if (cfs_b->quota != RUNTIME_INF) { 6002 cfs_b->runtime += slack_runtime; 6003 6004 /* we are under rq->lock, defer unthrottling using a timer */ 6005 if (cfs_b->runtime > sched_cfs_bandwidth_slice() && 6006 !list_empty(&cfs_b->throttled_cfs_rq)) 6007 start_cfs_slack_bandwidth(cfs_b); 6008 } 6009 raw_spin_unlock(&cfs_b->lock); 6010 6011 /* even if it's not valid for return we don't want to try again */ 6012 cfs_rq->runtime_remaining -= slack_runtime; 6013 } 6014 6015 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) 6016 { 6017 if (!cfs_bandwidth_used()) 6018 return; 6019 6020 if (!cfs_rq->runtime_enabled || cfs_rq->nr_running) 6021 return; 6022 6023 __return_cfs_rq_runtime(cfs_rq); 6024 } 6025 6026 /* 6027 * This is done with a timer (instead of inline with bandwidth return) since 6028 * it's necessary to juggle rq->locks to unthrottle their respective cfs_rqs. 6029 */ 6030 static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) 6031 { 6032 u64 runtime = 0, slice = sched_cfs_bandwidth_slice(); 6033 unsigned long flags; 6034 6035 /* confirm we're still not at a refresh boundary */ 6036 raw_spin_lock_irqsave(&cfs_b->lock, flags); 6037 cfs_b->slack_started = false; 6038 6039 if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) { 6040 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 6041 return; 6042 } 6043 6044 if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice) 6045 runtime = cfs_b->runtime; 6046 6047 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 6048 6049 if (!runtime) 6050 return; 6051 6052 distribute_cfs_runtime(cfs_b); 6053 } 6054 6055 /* 6056 * When a group wakes up we want to make sure that its quota is not already 6057 * expired/exceeded, otherwise it may be allowed to steal additional ticks of 6058 * runtime as update_curr() throttling can not trigger until it's on-rq. 6059 */ 6060 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) 6061 { 6062 if (!cfs_bandwidth_used()) 6063 return; 6064 6065 /* an active group must be handled by the update_curr()->put() path */ 6066 if (!cfs_rq->runtime_enabled || cfs_rq->curr) 6067 return; 6068 6069 /* ensure the group is not already throttled */ 6070 if (cfs_rq_throttled(cfs_rq)) 6071 return; 6072 6073 /* update runtime allocation */ 6074 account_cfs_rq_runtime(cfs_rq, 0); 6075 if (cfs_rq->runtime_remaining <= 0) 6076 throttle_cfs_rq(cfs_rq); 6077 } 6078 6079 static void sync_throttle(struct task_group *tg, int cpu) 6080 { 6081 struct cfs_rq *pcfs_rq, *cfs_rq; 6082 6083 if (!cfs_bandwidth_used()) 6084 return; 6085 6086 if (!tg->parent) 6087 return; 6088 6089 cfs_rq = tg->cfs_rq[cpu]; 6090 pcfs_rq = tg->parent->cfs_rq[cpu]; 6091 6092 cfs_rq->throttle_count = pcfs_rq->throttle_count; 6093 cfs_rq->throttled_clock_pelt = rq_clock_pelt(cpu_rq(cpu)); 6094 } 6095 6096 /* conditionally throttle active cfs_rq's from put_prev_entity() */ 6097 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) 6098 { 6099 if (!cfs_bandwidth_used()) 6100 return false; 6101 6102 if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0)) 6103 return false; 6104 6105 /* 6106 * it's possible for a throttled entity to be forced into a running 6107 * state (e.g. set_curr_task), in this case we're finished. 6108 */ 6109 if (cfs_rq_throttled(cfs_rq)) 6110 return true; 6111 6112 return throttle_cfs_rq(cfs_rq); 6113 } 6114 6115 static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer) 6116 { 6117 struct cfs_bandwidth *cfs_b = 6118 container_of(timer, struct cfs_bandwidth, slack_timer); 6119 6120 do_sched_cfs_slack_timer(cfs_b); 6121 6122 return HRTIMER_NORESTART; 6123 } 6124 6125 extern const u64 max_cfs_quota_period; 6126 6127 static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer) 6128 { 6129 struct cfs_bandwidth *cfs_b = 6130 container_of(timer, struct cfs_bandwidth, period_timer); 6131 unsigned long flags; 6132 int overrun; 6133 int idle = 0; 6134 int count = 0; 6135 6136 raw_spin_lock_irqsave(&cfs_b->lock, flags); 6137 for (;;) { 6138 overrun = hrtimer_forward_now(timer, cfs_b->period); 6139 if (!overrun) 6140 break; 6141 6142 idle = do_sched_cfs_period_timer(cfs_b, overrun, flags); 6143 6144 if (++count > 3) { 6145 u64 new, old = ktime_to_ns(cfs_b->period); 6146 6147 /* 6148 * Grow period by a factor of 2 to avoid losing precision. 6149 * Precision loss in the quota/period ratio can cause __cfs_schedulable 6150 * to fail. 6151 */ 6152 new = old * 2; 6153 if (new < max_cfs_quota_period) { 6154 cfs_b->period = ns_to_ktime(new); 6155 cfs_b->quota *= 2; 6156 cfs_b->burst *= 2; 6157 6158 pr_warn_ratelimited( 6159 "cfs_period_timer[cpu%d]: period too short, scaling up (new cfs_period_us = %lld, cfs_quota_us = %lld)\n", 6160 smp_processor_id(), 6161 div_u64(new, NSEC_PER_USEC), 6162 div_u64(cfs_b->quota, NSEC_PER_USEC)); 6163 } else { 6164 pr_warn_ratelimited( 6165 "cfs_period_timer[cpu%d]: period too short, but cannot scale up without losing precision (cfs_period_us = %lld, cfs_quota_us = %lld)\n", 6166 smp_processor_id(), 6167 div_u64(old, NSEC_PER_USEC), 6168 div_u64(cfs_b->quota, NSEC_PER_USEC)); 6169 } 6170 6171 /* reset count so we don't come right back in here */ 6172 count = 0; 6173 } 6174 } 6175 if (idle) 6176 cfs_b->period_active = 0; 6177 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 6178 6179 return idle ? HRTIMER_NORESTART : HRTIMER_RESTART; 6180 } 6181 6182 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b, struct cfs_bandwidth *parent) 6183 { 6184 raw_spin_lock_init(&cfs_b->lock); 6185 cfs_b->runtime = 0; 6186 cfs_b->quota = RUNTIME_INF; 6187 cfs_b->period = ns_to_ktime(default_cfs_period()); 6188 cfs_b->burst = 0; 6189 cfs_b->hierarchical_quota = parent ? parent->hierarchical_quota : RUNTIME_INF; 6190 6191 INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq); 6192 hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED); 6193 cfs_b->period_timer.function = sched_cfs_period_timer; 6194 6195 /* Add a random offset so that timers interleave */ 6196 hrtimer_set_expires(&cfs_b->period_timer, 6197 get_random_u32_below(cfs_b->period)); 6198 hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 6199 cfs_b->slack_timer.function = sched_cfs_slack_timer; 6200 cfs_b->slack_started = false; 6201 } 6202 6203 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) 6204 { 6205 cfs_rq->runtime_enabled = 0; 6206 INIT_LIST_HEAD(&cfs_rq->throttled_list); 6207 #ifdef CONFIG_SMP 6208 INIT_LIST_HEAD(&cfs_rq->throttled_csd_list); 6209 #endif 6210 } 6211 6212 void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b) 6213 { 6214 lockdep_assert_held(&cfs_b->lock); 6215 6216 if (cfs_b->period_active) 6217 return; 6218 6219 cfs_b->period_active = 1; 6220 hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period); 6221 hrtimer_start_expires(&cfs_b->period_timer, HRTIMER_MODE_ABS_PINNED); 6222 } 6223 6224 static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) 6225 { 6226 int __maybe_unused i; 6227 6228 /* init_cfs_bandwidth() was not called */ 6229 if (!cfs_b->throttled_cfs_rq.next) 6230 return; 6231 6232 hrtimer_cancel(&cfs_b->period_timer); 6233 hrtimer_cancel(&cfs_b->slack_timer); 6234 6235 /* 6236 * It is possible that we still have some cfs_rq's pending on a CSD 6237 * list, though this race is very rare. In order for this to occur, we 6238 * must have raced with the last task leaving the group while there 6239 * exist throttled cfs_rq(s), and the period_timer must have queued the 6240 * CSD item but the remote cpu has not yet processed it. To handle this, 6241 * we can simply flush all pending CSD work inline here. We're 6242 * guaranteed at this point that no additional cfs_rq of this group can 6243 * join a CSD list. 6244 */ 6245 #ifdef CONFIG_SMP 6246 for_each_possible_cpu(i) { 6247 struct rq *rq = cpu_rq(i); 6248 unsigned long flags; 6249 6250 if (list_empty(&rq->cfsb_csd_list)) 6251 continue; 6252 6253 local_irq_save(flags); 6254 __cfsb_csd_unthrottle(rq); 6255 local_irq_restore(flags); 6256 } 6257 #endif 6258 } 6259 6260 /* 6261 * Both these CPU hotplug callbacks race against unregister_fair_sched_group() 6262 * 6263 * The race is harmless, since modifying bandwidth settings of unhooked group 6264 * bits doesn't do much. 6265 */ 6266 6267 /* cpu online callback */ 6268 static void __maybe_unused update_runtime_enabled(struct rq *rq) 6269 { 6270 struct task_group *tg; 6271 6272 lockdep_assert_rq_held(rq); 6273 6274 rcu_read_lock(); 6275 list_for_each_entry_rcu(tg, &task_groups, list) { 6276 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 6277 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 6278 6279 raw_spin_lock(&cfs_b->lock); 6280 cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF; 6281 raw_spin_unlock(&cfs_b->lock); 6282 } 6283 rcu_read_unlock(); 6284 } 6285 6286 /* cpu offline callback */ 6287 static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq) 6288 { 6289 struct task_group *tg; 6290 6291 lockdep_assert_rq_held(rq); 6292 6293 /* 6294 * The rq clock has already been updated in the 6295 * set_rq_offline(), so we should skip updating 6296 * the rq clock again in unthrottle_cfs_rq(). 6297 */ 6298 rq_clock_start_loop_update(rq); 6299 6300 rcu_read_lock(); 6301 list_for_each_entry_rcu(tg, &task_groups, list) { 6302 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 6303 6304 if (!cfs_rq->runtime_enabled) 6305 continue; 6306 6307 /* 6308 * clock_task is not advancing so we just need to make sure 6309 * there's some valid quota amount 6310 */ 6311 cfs_rq->runtime_remaining = 1; 6312 /* 6313 * Offline rq is schedulable till CPU is completely disabled 6314 * in take_cpu_down(), so we prevent new cfs throttling here. 6315 */ 6316 cfs_rq->runtime_enabled = 0; 6317 6318 if (cfs_rq_throttled(cfs_rq)) 6319 unthrottle_cfs_rq(cfs_rq); 6320 } 6321 rcu_read_unlock(); 6322 6323 rq_clock_stop_loop_update(rq); 6324 } 6325 6326 bool cfs_task_bw_constrained(struct task_struct *p) 6327 { 6328 struct cfs_rq *cfs_rq = task_cfs_rq(p); 6329 6330 if (!cfs_bandwidth_used()) 6331 return false; 6332 6333 if (cfs_rq->runtime_enabled || 6334 tg_cfs_bandwidth(cfs_rq->tg)->hierarchical_quota != RUNTIME_INF) 6335 return true; 6336 6337 return false; 6338 } 6339 6340 #ifdef CONFIG_NO_HZ_FULL 6341 /* called from pick_next_task_fair() */ 6342 static void sched_fair_update_stop_tick(struct rq *rq, struct task_struct *p) 6343 { 6344 int cpu = cpu_of(rq); 6345 6346 if (!sched_feat(HZ_BW) || !cfs_bandwidth_used()) 6347 return; 6348 6349 if (!tick_nohz_full_cpu(cpu)) 6350 return; 6351 6352 if (rq->nr_running != 1) 6353 return; 6354 6355 /* 6356 * We know there is only one task runnable and we've just picked it. The 6357 * normal enqueue path will have cleared TICK_DEP_BIT_SCHED if we will 6358 * be otherwise able to stop the tick. Just need to check if we are using 6359 * bandwidth control. 6360 */ 6361 if (cfs_task_bw_constrained(p)) 6362 tick_nohz_dep_set_cpu(cpu, TICK_DEP_BIT_SCHED); 6363 } 6364 #endif 6365 6366 #else /* CONFIG_CFS_BANDWIDTH */ 6367 6368 static inline bool cfs_bandwidth_used(void) 6369 { 6370 return false; 6371 } 6372 6373 static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) {} 6374 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) { return false; } 6375 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {} 6376 static inline void sync_throttle(struct task_group *tg, int cpu) {} 6377 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} 6378 6379 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) 6380 { 6381 return 0; 6382 } 6383 6384 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) 6385 { 6386 return 0; 6387 } 6388 6389 static inline int throttled_lb_pair(struct task_group *tg, 6390 int src_cpu, int dest_cpu) 6391 { 6392 return 0; 6393 } 6394 6395 #ifdef CONFIG_FAIR_GROUP_SCHED 6396 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b, struct cfs_bandwidth *parent) {} 6397 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} 6398 #endif 6399 6400 static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) 6401 { 6402 return NULL; 6403 } 6404 static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {} 6405 static inline void update_runtime_enabled(struct rq *rq) {} 6406 static inline void unthrottle_offline_cfs_rqs(struct rq *rq) {} 6407 #ifdef CONFIG_CGROUP_SCHED 6408 bool cfs_task_bw_constrained(struct task_struct *p) 6409 { 6410 return false; 6411 } 6412 #endif 6413 #endif /* CONFIG_CFS_BANDWIDTH */ 6414 6415 #if !defined(CONFIG_CFS_BANDWIDTH) || !defined(CONFIG_NO_HZ_FULL) 6416 static inline void sched_fair_update_stop_tick(struct rq *rq, struct task_struct *p) {} 6417 #endif 6418 6419 /************************************************** 6420 * CFS operations on tasks: 6421 */ 6422 6423 #ifdef CONFIG_SCHED_HRTICK 6424 static void hrtick_start_fair(struct rq *rq, struct task_struct *p) 6425 { 6426 struct sched_entity *se = &p->se; 6427 6428 SCHED_WARN_ON(task_rq(p) != rq); 6429 6430 if (rq->cfs.h_nr_running > 1) { 6431 u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime; 6432 u64 slice = se->slice; 6433 s64 delta = slice - ran; 6434 6435 if (delta < 0) { 6436 if (task_current(rq, p)) 6437 resched_curr(rq); 6438 return; 6439 } 6440 hrtick_start(rq, delta); 6441 } 6442 } 6443 6444 /* 6445 * called from enqueue/dequeue and updates the hrtick when the 6446 * current task is from our class and nr_running is low enough 6447 * to matter. 6448 */ 6449 static void hrtick_update(struct rq *rq) 6450 { 6451 struct task_struct *curr = rq->curr; 6452 6453 if (!hrtick_enabled_fair(rq) || curr->sched_class != &fair_sched_class) 6454 return; 6455 6456 hrtick_start_fair(rq, curr); 6457 } 6458 #else /* !CONFIG_SCHED_HRTICK */ 6459 static inline void 6460 hrtick_start_fair(struct rq *rq, struct task_struct *p) 6461 { 6462 } 6463 6464 static inline void hrtick_update(struct rq *rq) 6465 { 6466 } 6467 #endif 6468 6469 #ifdef CONFIG_SMP 6470 static inline bool cpu_overutilized(int cpu) 6471 { 6472 unsigned long rq_util_min = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MIN); 6473 unsigned long rq_util_max = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MAX); 6474 6475 /* Return true only if the utilization doesn't fit CPU's capacity */ 6476 return !util_fits_cpu(cpu_util_cfs(cpu), rq_util_min, rq_util_max, cpu); 6477 } 6478 6479 static inline void update_overutilized_status(struct rq *rq) 6480 { 6481 if (!READ_ONCE(rq->rd->overutilized) && cpu_overutilized(rq->cpu)) { 6482 WRITE_ONCE(rq->rd->overutilized, SG_OVERUTILIZED); 6483 trace_sched_overutilized_tp(rq->rd, SG_OVERUTILIZED); 6484 } 6485 } 6486 #else 6487 static inline void update_overutilized_status(struct rq *rq) { } 6488 #endif 6489 6490 /* Runqueue only has SCHED_IDLE tasks enqueued */ 6491 static int sched_idle_rq(struct rq *rq) 6492 { 6493 return unlikely(rq->nr_running == rq->cfs.idle_h_nr_running && 6494 rq->nr_running); 6495 } 6496 6497 #ifdef CONFIG_SMP 6498 static int sched_idle_cpu(int cpu) 6499 { 6500 return sched_idle_rq(cpu_rq(cpu)); 6501 } 6502 #endif 6503 6504 /* 6505 * The enqueue_task method is called before nr_running is 6506 * increased. Here we update the fair scheduling stats and 6507 * then put the task into the rbtree: 6508 */ 6509 static void 6510 enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) 6511 { 6512 struct cfs_rq *cfs_rq; 6513 struct sched_entity *se = &p->se; 6514 int idle_h_nr_running = task_has_idle_policy(p); 6515 int task_new = !(flags & ENQUEUE_WAKEUP); 6516 6517 /* 6518 * The code below (indirectly) updates schedutil which looks at 6519 * the cfs_rq utilization to select a frequency. 6520 * Let's add the task's estimated utilization to the cfs_rq's 6521 * estimated utilization, before we update schedutil. 6522 */ 6523 util_est_enqueue(&rq->cfs, p); 6524 6525 /* 6526 * If in_iowait is set, the code below may not trigger any cpufreq 6527 * utilization updates, so do it here explicitly with the IOWAIT flag 6528 * passed. 6529 */ 6530 if (p->in_iowait) 6531 cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT); 6532 6533 for_each_sched_entity(se) { 6534 if (se->on_rq) 6535 break; 6536 cfs_rq = cfs_rq_of(se); 6537 enqueue_entity(cfs_rq, se, flags); 6538 6539 cfs_rq->h_nr_running++; 6540 cfs_rq->idle_h_nr_running += idle_h_nr_running; 6541 6542 if (cfs_rq_is_idle(cfs_rq)) 6543 idle_h_nr_running = 1; 6544 6545 /* end evaluation on encountering a throttled cfs_rq */ 6546 if (cfs_rq_throttled(cfs_rq)) 6547 goto enqueue_throttle; 6548 6549 flags = ENQUEUE_WAKEUP; 6550 } 6551 6552 for_each_sched_entity(se) { 6553 cfs_rq = cfs_rq_of(se); 6554 6555 update_load_avg(cfs_rq, se, UPDATE_TG); 6556 se_update_runnable(se); 6557 update_cfs_group(se); 6558 6559 cfs_rq->h_nr_running++; 6560 cfs_rq->idle_h_nr_running += idle_h_nr_running; 6561 6562 if (cfs_rq_is_idle(cfs_rq)) 6563 idle_h_nr_running = 1; 6564 6565 /* end evaluation on encountering a throttled cfs_rq */ 6566 if (cfs_rq_throttled(cfs_rq)) 6567 goto enqueue_throttle; 6568 } 6569 6570 /* At this point se is NULL and we are at root level*/ 6571 add_nr_running(rq, 1); 6572 6573 /* 6574 * Since new tasks are assigned an initial util_avg equal to 6575 * half of the spare capacity of their CPU, tiny tasks have the 6576 * ability to cross the overutilized threshold, which will 6577 * result in the load balancer ruining all the task placement 6578 * done by EAS. As a way to mitigate that effect, do not account 6579 * for the first enqueue operation of new tasks during the 6580 * overutilized flag detection. 6581 * 6582 * A better way of solving this problem would be to wait for 6583 * the PELT signals of tasks to converge before taking them 6584 * into account, but that is not straightforward to implement, 6585 * and the following generally works well enough in practice. 6586 */ 6587 if (!task_new) 6588 update_overutilized_status(rq); 6589 6590 enqueue_throttle: 6591 assert_list_leaf_cfs_rq(rq); 6592 6593 hrtick_update(rq); 6594 } 6595 6596 static void set_next_buddy(struct sched_entity *se); 6597 6598 /* 6599 * The dequeue_task method is called before nr_running is 6600 * decreased. We remove the task from the rbtree and 6601 * update the fair scheduling stats: 6602 */ 6603 static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) 6604 { 6605 struct cfs_rq *cfs_rq; 6606 struct sched_entity *se = &p->se; 6607 int task_sleep = flags & DEQUEUE_SLEEP; 6608 int idle_h_nr_running = task_has_idle_policy(p); 6609 bool was_sched_idle = sched_idle_rq(rq); 6610 6611 util_est_dequeue(&rq->cfs, p); 6612 6613 for_each_sched_entity(se) { 6614 cfs_rq = cfs_rq_of(se); 6615 dequeue_entity(cfs_rq, se, flags); 6616 6617 cfs_rq->h_nr_running--; 6618 cfs_rq->idle_h_nr_running -= idle_h_nr_running; 6619 6620 if (cfs_rq_is_idle(cfs_rq)) 6621 idle_h_nr_running = 1; 6622 6623 /* end evaluation on encountering a throttled cfs_rq */ 6624 if (cfs_rq_throttled(cfs_rq)) 6625 goto dequeue_throttle; 6626 6627 /* Don't dequeue parent if it has other entities besides us */ 6628 if (cfs_rq->load.weight) { 6629 /* Avoid re-evaluating load for this entity: */ 6630 se = parent_entity(se); 6631 /* 6632 * Bias pick_next to pick a task from this cfs_rq, as 6633 * p is sleeping when it is within its sched_slice. 6634 */ 6635 if (task_sleep && se && !throttled_hierarchy(cfs_rq)) 6636 set_next_buddy(se); 6637 break; 6638 } 6639 flags |= DEQUEUE_SLEEP; 6640 } 6641 6642 for_each_sched_entity(se) { 6643 cfs_rq = cfs_rq_of(se); 6644 6645 update_load_avg(cfs_rq, se, UPDATE_TG); 6646 se_update_runnable(se); 6647 update_cfs_group(se); 6648 6649 cfs_rq->h_nr_running--; 6650 cfs_rq->idle_h_nr_running -= idle_h_nr_running; 6651 6652 if (cfs_rq_is_idle(cfs_rq)) 6653 idle_h_nr_running = 1; 6654 6655 /* end evaluation on encountering a throttled cfs_rq */ 6656 if (cfs_rq_throttled(cfs_rq)) 6657 goto dequeue_throttle; 6658 6659 } 6660 6661 /* At this point se is NULL and we are at root level*/ 6662 sub_nr_running(rq, 1); 6663 6664 /* balance early to pull high priority tasks */ 6665 if (unlikely(!was_sched_idle && sched_idle_rq(rq))) 6666 rq->next_balance = jiffies; 6667 6668 dequeue_throttle: 6669 util_est_update(&rq->cfs, p, task_sleep); 6670 hrtick_update(rq); 6671 } 6672 6673 #ifdef CONFIG_SMP 6674 6675 /* Working cpumask for: load_balance, load_balance_newidle. */ 6676 static DEFINE_PER_CPU(cpumask_var_t, load_balance_mask); 6677 static DEFINE_PER_CPU(cpumask_var_t, select_rq_mask); 6678 static DEFINE_PER_CPU(cpumask_var_t, should_we_balance_tmpmask); 6679 6680 #ifdef CONFIG_NO_HZ_COMMON 6681 6682 static struct { 6683 cpumask_var_t idle_cpus_mask; 6684 atomic_t nr_cpus; 6685 int has_blocked; /* Idle CPUS has blocked load */ 6686 int needs_update; /* Newly idle CPUs need their next_balance collated */ 6687 unsigned long next_balance; /* in jiffy units */ 6688 unsigned long next_blocked; /* Next update of blocked load in jiffies */ 6689 } nohz ____cacheline_aligned; 6690 6691 #endif /* CONFIG_NO_HZ_COMMON */ 6692 6693 static unsigned long cpu_load(struct rq *rq) 6694 { 6695 return cfs_rq_load_avg(&rq->cfs); 6696 } 6697 6698 /* 6699 * cpu_load_without - compute CPU load without any contributions from *p 6700 * @cpu: the CPU which load is requested 6701 * @p: the task which load should be discounted 6702 * 6703 * The load of a CPU is defined by the load of tasks currently enqueued on that 6704 * CPU as well as tasks which are currently sleeping after an execution on that 6705 * CPU. 6706 * 6707 * This method returns the load of the specified CPU by discounting the load of 6708 * the specified task, whenever the task is currently contributing to the CPU 6709 * load. 6710 */ 6711 static unsigned long cpu_load_without(struct rq *rq, struct task_struct *p) 6712 { 6713 struct cfs_rq *cfs_rq; 6714 unsigned int load; 6715 6716 /* Task has no contribution or is new */ 6717 if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 6718 return cpu_load(rq); 6719 6720 cfs_rq = &rq->cfs; 6721 load = READ_ONCE(cfs_rq->avg.load_avg); 6722 6723 /* Discount task's util from CPU's util */ 6724 lsub_positive(&load, task_h_load(p)); 6725 6726 return load; 6727 } 6728 6729 static unsigned long cpu_runnable(struct rq *rq) 6730 { 6731 return cfs_rq_runnable_avg(&rq->cfs); 6732 } 6733 6734 static unsigned long cpu_runnable_without(struct rq *rq, struct task_struct *p) 6735 { 6736 struct cfs_rq *cfs_rq; 6737 unsigned int runnable; 6738 6739 /* Task has no contribution or is new */ 6740 if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 6741 return cpu_runnable(rq); 6742 6743 cfs_rq = &rq->cfs; 6744 runnable = READ_ONCE(cfs_rq->avg.runnable_avg); 6745 6746 /* Discount task's runnable from CPU's runnable */ 6747 lsub_positive(&runnable, p->se.avg.runnable_avg); 6748 6749 return runnable; 6750 } 6751 6752 static unsigned long capacity_of(int cpu) 6753 { 6754 return cpu_rq(cpu)->cpu_capacity; 6755 } 6756 6757 static void record_wakee(struct task_struct *p) 6758 { 6759 /* 6760 * Only decay a single time; tasks that have less then 1 wakeup per 6761 * jiffy will not have built up many flips. 6762 */ 6763 if (time_after(jiffies, current->wakee_flip_decay_ts + HZ)) { 6764 current->wakee_flips >>= 1; 6765 current->wakee_flip_decay_ts = jiffies; 6766 } 6767 6768 if (current->last_wakee != p) { 6769 current->last_wakee = p; 6770 current->wakee_flips++; 6771 } 6772 } 6773 6774 /* 6775 * Detect M:N waker/wakee relationships via a switching-frequency heuristic. 6776 * 6777 * A waker of many should wake a different task than the one last awakened 6778 * at a frequency roughly N times higher than one of its wakees. 6779 * 6780 * In order to determine whether we should let the load spread vs consolidating 6781 * to shared cache, we look for a minimum 'flip' frequency of llc_size in one 6782 * partner, and a factor of lls_size higher frequency in the other. 6783 * 6784 * With both conditions met, we can be relatively sure that the relationship is 6785 * non-monogamous, with partner count exceeding socket size. 6786 * 6787 * Waker/wakee being client/server, worker/dispatcher, interrupt source or 6788 * whatever is irrelevant, spread criteria is apparent partner count exceeds 6789 * socket size. 6790 */ 6791 static int wake_wide(struct task_struct *p) 6792 { 6793 unsigned int master = current->wakee_flips; 6794 unsigned int slave = p->wakee_flips; 6795 int factor = __this_cpu_read(sd_llc_size); 6796 6797 if (master < slave) 6798 swap(master, slave); 6799 if (slave < factor || master < slave * factor) 6800 return 0; 6801 return 1; 6802 } 6803 6804 /* 6805 * The purpose of wake_affine() is to quickly determine on which CPU we can run 6806 * soonest. For the purpose of speed we only consider the waking and previous 6807 * CPU. 6808 * 6809 * wake_affine_idle() - only considers 'now', it check if the waking CPU is 6810 * cache-affine and is (or will be) idle. 6811 * 6812 * wake_affine_weight() - considers the weight to reflect the average 6813 * scheduling latency of the CPUs. This seems to work 6814 * for the overloaded case. 6815 */ 6816 static int 6817 wake_affine_idle(int this_cpu, int prev_cpu, int sync) 6818 { 6819 /* 6820 * If this_cpu is idle, it implies the wakeup is from interrupt 6821 * context. Only allow the move if cache is shared. Otherwise an 6822 * interrupt intensive workload could force all tasks onto one 6823 * node depending on the IO topology or IRQ affinity settings. 6824 * 6825 * If the prev_cpu is idle and cache affine then avoid a migration. 6826 * There is no guarantee that the cache hot data from an interrupt 6827 * is more important than cache hot data on the prev_cpu and from 6828 * a cpufreq perspective, it's better to have higher utilisation 6829 * on one CPU. 6830 */ 6831 if (available_idle_cpu(this_cpu) && cpus_share_cache(this_cpu, prev_cpu)) 6832 return available_idle_cpu(prev_cpu) ? prev_cpu : this_cpu; 6833 6834 if (sync && cpu_rq(this_cpu)->nr_running == 1) 6835 return this_cpu; 6836 6837 if (available_idle_cpu(prev_cpu)) 6838 return prev_cpu; 6839 6840 return nr_cpumask_bits; 6841 } 6842 6843 static int 6844 wake_affine_weight(struct sched_domain *sd, struct task_struct *p, 6845 int this_cpu, int prev_cpu, int sync) 6846 { 6847 s64 this_eff_load, prev_eff_load; 6848 unsigned long task_load; 6849 6850 this_eff_load = cpu_load(cpu_rq(this_cpu)); 6851 6852 if (sync) { 6853 unsigned long current_load = task_h_load(current); 6854 6855 if (current_load > this_eff_load) 6856 return this_cpu; 6857 6858 this_eff_load -= current_load; 6859 } 6860 6861 task_load = task_h_load(p); 6862 6863 this_eff_load += task_load; 6864 if (sched_feat(WA_BIAS)) 6865 this_eff_load *= 100; 6866 this_eff_load *= capacity_of(prev_cpu); 6867 6868 prev_eff_load = cpu_load(cpu_rq(prev_cpu)); 6869 prev_eff_load -= task_load; 6870 if (sched_feat(WA_BIAS)) 6871 prev_eff_load *= 100 + (sd->imbalance_pct - 100) / 2; 6872 prev_eff_load *= capacity_of(this_cpu); 6873 6874 /* 6875 * If sync, adjust the weight of prev_eff_load such that if 6876 * prev_eff == this_eff that select_idle_sibling() will consider 6877 * stacking the wakee on top of the waker if no other CPU is 6878 * idle. 6879 */ 6880 if (sync) 6881 prev_eff_load += 1; 6882 6883 return this_eff_load < prev_eff_load ? this_cpu : nr_cpumask_bits; 6884 } 6885 6886 static int wake_affine(struct sched_domain *sd, struct task_struct *p, 6887 int this_cpu, int prev_cpu, int sync) 6888 { 6889 int target = nr_cpumask_bits; 6890 6891 if (sched_feat(WA_IDLE)) 6892 target = wake_affine_idle(this_cpu, prev_cpu, sync); 6893 6894 if (sched_feat(WA_WEIGHT) && target == nr_cpumask_bits) 6895 target = wake_affine_weight(sd, p, this_cpu, prev_cpu, sync); 6896 6897 schedstat_inc(p->stats.nr_wakeups_affine_attempts); 6898 if (target != this_cpu) 6899 return prev_cpu; 6900 6901 schedstat_inc(sd->ttwu_move_affine); 6902 schedstat_inc(p->stats.nr_wakeups_affine); 6903 return target; 6904 } 6905 6906 static struct sched_group * 6907 find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu); 6908 6909 /* 6910 * find_idlest_group_cpu - find the idlest CPU among the CPUs in the group. 6911 */ 6912 static int 6913 find_idlest_group_cpu(struct sched_group *group, struct task_struct *p, int this_cpu) 6914 { 6915 unsigned long load, min_load = ULONG_MAX; 6916 unsigned int min_exit_latency = UINT_MAX; 6917 u64 latest_idle_timestamp = 0; 6918 int least_loaded_cpu = this_cpu; 6919 int shallowest_idle_cpu = -1; 6920 int i; 6921 6922 /* Check if we have any choice: */ 6923 if (group->group_weight == 1) 6924 return cpumask_first(sched_group_span(group)); 6925 6926 /* Traverse only the allowed CPUs */ 6927 for_each_cpu_and(i, sched_group_span(group), p->cpus_ptr) { 6928 struct rq *rq = cpu_rq(i); 6929 6930 if (!sched_core_cookie_match(rq, p)) 6931 continue; 6932 6933 if (sched_idle_cpu(i)) 6934 return i; 6935 6936 if (available_idle_cpu(i)) { 6937 struct cpuidle_state *idle = idle_get_state(rq); 6938 if (idle && idle->exit_latency < min_exit_latency) { 6939 /* 6940 * We give priority to a CPU whose idle state 6941 * has the smallest exit latency irrespective 6942 * of any idle timestamp. 6943 */ 6944 min_exit_latency = idle->exit_latency; 6945 latest_idle_timestamp = rq->idle_stamp; 6946 shallowest_idle_cpu = i; 6947 } else if ((!idle || idle->exit_latency == min_exit_latency) && 6948 rq->idle_stamp > latest_idle_timestamp) { 6949 /* 6950 * If equal or no active idle state, then 6951 * the most recently idled CPU might have 6952 * a warmer cache. 6953 */ 6954 latest_idle_timestamp = rq->idle_stamp; 6955 shallowest_idle_cpu = i; 6956 } 6957 } else if (shallowest_idle_cpu == -1) { 6958 load = cpu_load(cpu_rq(i)); 6959 if (load < min_load) { 6960 min_load = load; 6961 least_loaded_cpu = i; 6962 } 6963 } 6964 } 6965 6966 return shallowest_idle_cpu != -1 ? shallowest_idle_cpu : least_loaded_cpu; 6967 } 6968 6969 static inline int find_idlest_cpu(struct sched_domain *sd, struct task_struct *p, 6970 int cpu, int prev_cpu, int sd_flag) 6971 { 6972 int new_cpu = cpu; 6973 6974 if (!cpumask_intersects(sched_domain_span(sd), p->cpus_ptr)) 6975 return prev_cpu; 6976 6977 /* 6978 * We need task's util for cpu_util_without, sync it up to 6979 * prev_cpu's last_update_time. 6980 */ 6981 if (!(sd_flag & SD_BALANCE_FORK)) 6982 sync_entity_load_avg(&p->se); 6983 6984 while (sd) { 6985 struct sched_group *group; 6986 struct sched_domain *tmp; 6987 int weight; 6988 6989 if (!(sd->flags & sd_flag)) { 6990 sd = sd->child; 6991 continue; 6992 } 6993 6994 group = find_idlest_group(sd, p, cpu); 6995 if (!group) { 6996 sd = sd->child; 6997 continue; 6998 } 6999 7000 new_cpu = find_idlest_group_cpu(group, p, cpu); 7001 if (new_cpu == cpu) { 7002 /* Now try balancing at a lower domain level of 'cpu': */ 7003 sd = sd->child; 7004 continue; 7005 } 7006 7007 /* Now try balancing at a lower domain level of 'new_cpu': */ 7008 cpu = new_cpu; 7009 weight = sd->span_weight; 7010 sd = NULL; 7011 for_each_domain(cpu, tmp) { 7012 if (weight <= tmp->span_weight) 7013 break; 7014 if (tmp->flags & sd_flag) 7015 sd = tmp; 7016 } 7017 } 7018 7019 return new_cpu; 7020 } 7021 7022 static inline int __select_idle_cpu(int cpu, struct task_struct *p) 7023 { 7024 if ((available_idle_cpu(cpu) || sched_idle_cpu(cpu)) && 7025 sched_cpu_cookie_match(cpu_rq(cpu), p)) 7026 return cpu; 7027 7028 return -1; 7029 } 7030 7031 #ifdef CONFIG_SCHED_SMT 7032 DEFINE_STATIC_KEY_FALSE(sched_smt_present); 7033 EXPORT_SYMBOL_GPL(sched_smt_present); 7034 7035 static inline void set_idle_cores(int cpu, int val) 7036 { 7037 struct sched_domain_shared *sds; 7038 7039 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); 7040 if (sds) 7041 WRITE_ONCE(sds->has_idle_cores, val); 7042 } 7043 7044 static inline bool test_idle_cores(int cpu) 7045 { 7046 struct sched_domain_shared *sds; 7047 7048 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); 7049 if (sds) 7050 return READ_ONCE(sds->has_idle_cores); 7051 7052 return false; 7053 } 7054 7055 /* 7056 * Scans the local SMT mask to see if the entire core is idle, and records this 7057 * information in sd_llc_shared->has_idle_cores. 7058 * 7059 * Since SMT siblings share all cache levels, inspecting this limited remote 7060 * state should be fairly cheap. 7061 */ 7062 void __update_idle_core(struct rq *rq) 7063 { 7064 int core = cpu_of(rq); 7065 int cpu; 7066 7067 rcu_read_lock(); 7068 if (test_idle_cores(core)) 7069 goto unlock; 7070 7071 for_each_cpu(cpu, cpu_smt_mask(core)) { 7072 if (cpu == core) 7073 continue; 7074 7075 if (!available_idle_cpu(cpu)) 7076 goto unlock; 7077 } 7078 7079 set_idle_cores(core, 1); 7080 unlock: 7081 rcu_read_unlock(); 7082 } 7083 7084 /* 7085 * Scan the entire LLC domain for idle cores; this dynamically switches off if 7086 * there are no idle cores left in the system; tracked through 7087 * sd_llc->shared->has_idle_cores and enabled through update_idle_core() above. 7088 */ 7089 static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu) 7090 { 7091 bool idle = true; 7092 int cpu; 7093 7094 for_each_cpu(cpu, cpu_smt_mask(core)) { 7095 if (!available_idle_cpu(cpu)) { 7096 idle = false; 7097 if (*idle_cpu == -1) { 7098 if (sched_idle_cpu(cpu) && cpumask_test_cpu(cpu, p->cpus_ptr)) { 7099 *idle_cpu = cpu; 7100 break; 7101 } 7102 continue; 7103 } 7104 break; 7105 } 7106 if (*idle_cpu == -1 && cpumask_test_cpu(cpu, p->cpus_ptr)) 7107 *idle_cpu = cpu; 7108 } 7109 7110 if (idle) 7111 return core; 7112 7113 cpumask_andnot(cpus, cpus, cpu_smt_mask(core)); 7114 return -1; 7115 } 7116 7117 /* 7118 * Scan the local SMT mask for idle CPUs. 7119 */ 7120 static int select_idle_smt(struct task_struct *p, int target) 7121 { 7122 int cpu; 7123 7124 for_each_cpu_and(cpu, cpu_smt_mask(target), p->cpus_ptr) { 7125 if (cpu == target) 7126 continue; 7127 if (available_idle_cpu(cpu) || sched_idle_cpu(cpu)) 7128 return cpu; 7129 } 7130 7131 return -1; 7132 } 7133 7134 #else /* CONFIG_SCHED_SMT */ 7135 7136 static inline void set_idle_cores(int cpu, int val) 7137 { 7138 } 7139 7140 static inline bool test_idle_cores(int cpu) 7141 { 7142 return false; 7143 } 7144 7145 static inline int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu) 7146 { 7147 return __select_idle_cpu(core, p); 7148 } 7149 7150 static inline int select_idle_smt(struct task_struct *p, int target) 7151 { 7152 return -1; 7153 } 7154 7155 #endif /* CONFIG_SCHED_SMT */ 7156 7157 /* 7158 * Scan the LLC domain for idle CPUs; this is dynamically regulated by 7159 * comparing the average scan cost (tracked in sd->avg_scan_cost) against the 7160 * average idle time for this rq (as found in rq->avg_idle). 7161 */ 7162 static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool has_idle_core, int target) 7163 { 7164 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 7165 int i, cpu, idle_cpu = -1, nr = INT_MAX; 7166 struct sched_domain_shared *sd_share; 7167 struct rq *this_rq = this_rq(); 7168 int this = smp_processor_id(); 7169 struct sched_domain *this_sd = NULL; 7170 u64 time = 0; 7171 7172 cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr); 7173 7174 if (sched_feat(SIS_PROP) && !has_idle_core) { 7175 u64 avg_cost, avg_idle, span_avg; 7176 unsigned long now = jiffies; 7177 7178 this_sd = rcu_dereference(*this_cpu_ptr(&sd_llc)); 7179 if (!this_sd) 7180 return -1; 7181 7182 /* 7183 * If we're busy, the assumption that the last idle period 7184 * predicts the future is flawed; age away the remaining 7185 * predicted idle time. 7186 */ 7187 if (unlikely(this_rq->wake_stamp < now)) { 7188 while (this_rq->wake_stamp < now && this_rq->wake_avg_idle) { 7189 this_rq->wake_stamp++; 7190 this_rq->wake_avg_idle >>= 1; 7191 } 7192 } 7193 7194 avg_idle = this_rq->wake_avg_idle; 7195 avg_cost = this_sd->avg_scan_cost + 1; 7196 7197 span_avg = sd->span_weight * avg_idle; 7198 if (span_avg > 4*avg_cost) 7199 nr = div_u64(span_avg, avg_cost); 7200 else 7201 nr = 4; 7202 7203 time = cpu_clock(this); 7204 } 7205 7206 if (sched_feat(SIS_UTIL)) { 7207 sd_share = rcu_dereference(per_cpu(sd_llc_shared, target)); 7208 if (sd_share) { 7209 /* because !--nr is the condition to stop scan */ 7210 nr = READ_ONCE(sd_share->nr_idle_scan) + 1; 7211 /* overloaded LLC is unlikely to have idle cpu/core */ 7212 if (nr == 1) 7213 return -1; 7214 } 7215 } 7216 7217 for_each_cpu_wrap(cpu, cpus, target + 1) { 7218 if (has_idle_core) { 7219 i = select_idle_core(p, cpu, cpus, &idle_cpu); 7220 if ((unsigned int)i < nr_cpumask_bits) 7221 return i; 7222 7223 } else { 7224 if (!--nr) 7225 return -1; 7226 idle_cpu = __select_idle_cpu(cpu, p); 7227 if ((unsigned int)idle_cpu < nr_cpumask_bits) 7228 break; 7229 } 7230 } 7231 7232 if (has_idle_core) 7233 set_idle_cores(target, false); 7234 7235 if (sched_feat(SIS_PROP) && this_sd && !has_idle_core) { 7236 time = cpu_clock(this) - time; 7237 7238 /* 7239 * Account for the scan cost of wakeups against the average 7240 * idle time. 7241 */ 7242 this_rq->wake_avg_idle -= min(this_rq->wake_avg_idle, time); 7243 7244 update_avg(&this_sd->avg_scan_cost, time); 7245 } 7246 7247 return idle_cpu; 7248 } 7249 7250 /* 7251 * Scan the asym_capacity domain for idle CPUs; pick the first idle one on which 7252 * the task fits. If no CPU is big enough, but there are idle ones, try to 7253 * maximize capacity. 7254 */ 7255 static int 7256 select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) 7257 { 7258 unsigned long task_util, util_min, util_max, best_cap = 0; 7259 int fits, best_fits = 0; 7260 int cpu, best_cpu = -1; 7261 struct cpumask *cpus; 7262 7263 cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 7264 cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr); 7265 7266 task_util = task_util_est(p); 7267 util_min = uclamp_eff_value(p, UCLAMP_MIN); 7268 util_max = uclamp_eff_value(p, UCLAMP_MAX); 7269 7270 for_each_cpu_wrap(cpu, cpus, target) { 7271 unsigned long cpu_cap = capacity_of(cpu); 7272 7273 if (!available_idle_cpu(cpu) && !sched_idle_cpu(cpu)) 7274 continue; 7275 7276 fits = util_fits_cpu(task_util, util_min, util_max, cpu); 7277 7278 /* This CPU fits with all requirements */ 7279 if (fits > 0) 7280 return cpu; 7281 /* 7282 * Only the min performance hint (i.e. uclamp_min) doesn't fit. 7283 * Look for the CPU with best capacity. 7284 */ 7285 else if (fits < 0) 7286 cpu_cap = capacity_orig_of(cpu) - thermal_load_avg(cpu_rq(cpu)); 7287 7288 /* 7289 * First, select CPU which fits better (-1 being better than 0). 7290 * Then, select the one with best capacity at same level. 7291 */ 7292 if ((fits < best_fits) || 7293 ((fits == best_fits) && (cpu_cap > best_cap))) { 7294 best_cap = cpu_cap; 7295 best_cpu = cpu; 7296 best_fits = fits; 7297 } 7298 } 7299 7300 return best_cpu; 7301 } 7302 7303 static inline bool asym_fits_cpu(unsigned long util, 7304 unsigned long util_min, 7305 unsigned long util_max, 7306 int cpu) 7307 { 7308 if (sched_asym_cpucap_active()) 7309 /* 7310 * Return true only if the cpu fully fits the task requirements 7311 * which include the utilization and the performance hints. 7312 */ 7313 return (util_fits_cpu(util, util_min, util_max, cpu) > 0); 7314 7315 return true; 7316 } 7317 7318 /* 7319 * Try and locate an idle core/thread in the LLC cache domain. 7320 */ 7321 static int select_idle_sibling(struct task_struct *p, int prev, int target) 7322 { 7323 bool has_idle_core = false; 7324 struct sched_domain *sd; 7325 unsigned long task_util, util_min, util_max; 7326 int i, recent_used_cpu; 7327 7328 /* 7329 * On asymmetric system, update task utilization because we will check 7330 * that the task fits with cpu's capacity. 7331 */ 7332 if (sched_asym_cpucap_active()) { 7333 sync_entity_load_avg(&p->se); 7334 task_util = task_util_est(p); 7335 util_min = uclamp_eff_value(p, UCLAMP_MIN); 7336 util_max = uclamp_eff_value(p, UCLAMP_MAX); 7337 } 7338 7339 /* 7340 * per-cpu select_rq_mask usage 7341 */ 7342 lockdep_assert_irqs_disabled(); 7343 7344 if ((available_idle_cpu(target) || sched_idle_cpu(target)) && 7345 asym_fits_cpu(task_util, util_min, util_max, target)) 7346 return target; 7347 7348 /* 7349 * If the previous CPU is cache affine and idle, don't be stupid: 7350 */ 7351 if (prev != target && cpus_share_cache(prev, target) && 7352 (available_idle_cpu(prev) || sched_idle_cpu(prev)) && 7353 asym_fits_cpu(task_util, util_min, util_max, prev)) 7354 return prev; 7355 7356 /* 7357 * Allow a per-cpu kthread to stack with the wakee if the 7358 * kworker thread and the tasks previous CPUs are the same. 7359 * The assumption is that the wakee queued work for the 7360 * per-cpu kthread that is now complete and the wakeup is 7361 * essentially a sync wakeup. An obvious example of this 7362 * pattern is IO completions. 7363 */ 7364 if (is_per_cpu_kthread(current) && 7365 in_task() && 7366 prev == smp_processor_id() && 7367 this_rq()->nr_running <= 1 && 7368 asym_fits_cpu(task_util, util_min, util_max, prev)) { 7369 return prev; 7370 } 7371 7372 /* Check a recently used CPU as a potential idle candidate: */ 7373 recent_used_cpu = p->recent_used_cpu; 7374 p->recent_used_cpu = prev; 7375 if (recent_used_cpu != prev && 7376 recent_used_cpu != target && 7377 cpus_share_cache(recent_used_cpu, target) && 7378 (available_idle_cpu(recent_used_cpu) || sched_idle_cpu(recent_used_cpu)) && 7379 cpumask_test_cpu(recent_used_cpu, p->cpus_ptr) && 7380 asym_fits_cpu(task_util, util_min, util_max, recent_used_cpu)) { 7381 return recent_used_cpu; 7382 } 7383 7384 /* 7385 * For asymmetric CPU capacity systems, our domain of interest is 7386 * sd_asym_cpucapacity rather than sd_llc. 7387 */ 7388 if (sched_asym_cpucap_active()) { 7389 sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, target)); 7390 /* 7391 * On an asymmetric CPU capacity system where an exclusive 7392 * cpuset defines a symmetric island (i.e. one unique 7393 * capacity_orig value through the cpuset), the key will be set 7394 * but the CPUs within that cpuset will not have a domain with 7395 * SD_ASYM_CPUCAPACITY. These should follow the usual symmetric 7396 * capacity path. 7397 */ 7398 if (sd) { 7399 i = select_idle_capacity(p, sd, target); 7400 return ((unsigned)i < nr_cpumask_bits) ? i : target; 7401 } 7402 } 7403 7404 sd = rcu_dereference(per_cpu(sd_llc, target)); 7405 if (!sd) 7406 return target; 7407 7408 if (sched_smt_active()) { 7409 has_idle_core = test_idle_cores(target); 7410 7411 if (!has_idle_core && cpus_share_cache(prev, target)) { 7412 i = select_idle_smt(p, prev); 7413 if ((unsigned int)i < nr_cpumask_bits) 7414 return i; 7415 } 7416 } 7417 7418 i = select_idle_cpu(p, sd, has_idle_core, target); 7419 if ((unsigned)i < nr_cpumask_bits) 7420 return i; 7421 7422 return target; 7423 } 7424 7425 /** 7426 * cpu_util() - Estimates the amount of CPU capacity used by CFS tasks. 7427 * @cpu: the CPU to get the utilization for 7428 * @p: task for which the CPU utilization should be predicted or NULL 7429 * @dst_cpu: CPU @p migrates to, -1 if @p moves from @cpu or @p == NULL 7430 * @boost: 1 to enable boosting, otherwise 0 7431 * 7432 * The unit of the return value must be the same as the one of CPU capacity 7433 * so that CPU utilization can be compared with CPU capacity. 7434 * 7435 * CPU utilization is the sum of running time of runnable tasks plus the 7436 * recent utilization of currently non-runnable tasks on that CPU. 7437 * It represents the amount of CPU capacity currently used by CFS tasks in 7438 * the range [0..max CPU capacity] with max CPU capacity being the CPU 7439 * capacity at f_max. 7440 * 7441 * The estimated CPU utilization is defined as the maximum between CPU 7442 * utilization and sum of the estimated utilization of the currently 7443 * runnable tasks on that CPU. It preserves a utilization "snapshot" of 7444 * previously-executed tasks, which helps better deduce how busy a CPU will 7445 * be when a long-sleeping task wakes up. The contribution to CPU utilization 7446 * of such a task would be significantly decayed at this point of time. 7447 * 7448 * Boosted CPU utilization is defined as max(CPU runnable, CPU utilization). 7449 * CPU contention for CFS tasks can be detected by CPU runnable > CPU 7450 * utilization. Boosting is implemented in cpu_util() so that internal 7451 * users (e.g. EAS) can use it next to external users (e.g. schedutil), 7452 * latter via cpu_util_cfs_boost(). 7453 * 7454 * CPU utilization can be higher than the current CPU capacity 7455 * (f_curr/f_max * max CPU capacity) or even the max CPU capacity because 7456 * of rounding errors as well as task migrations or wakeups of new tasks. 7457 * CPU utilization has to be capped to fit into the [0..max CPU capacity] 7458 * range. Otherwise a group of CPUs (CPU0 util = 121% + CPU1 util = 80%) 7459 * could be seen as over-utilized even though CPU1 has 20% of spare CPU 7460 * capacity. CPU utilization is allowed to overshoot current CPU capacity 7461 * though since this is useful for predicting the CPU capacity required 7462 * after task migrations (scheduler-driven DVFS). 7463 * 7464 * Return: (Boosted) (estimated) utilization for the specified CPU. 7465 */ 7466 static unsigned long 7467 cpu_util(int cpu, struct task_struct *p, int dst_cpu, int boost) 7468 { 7469 struct cfs_rq *cfs_rq = &cpu_rq(cpu)->cfs; 7470 unsigned long util = READ_ONCE(cfs_rq->avg.util_avg); 7471 unsigned long runnable; 7472 7473 if (boost) { 7474 runnable = READ_ONCE(cfs_rq->avg.runnable_avg); 7475 util = max(util, runnable); 7476 } 7477 7478 /* 7479 * If @dst_cpu is -1 or @p migrates from @cpu to @dst_cpu remove its 7480 * contribution. If @p migrates from another CPU to @cpu add its 7481 * contribution. In all the other cases @cpu is not impacted by the 7482 * migration so its util_avg is already correct. 7483 */ 7484 if (p && task_cpu(p) == cpu && dst_cpu != cpu) 7485 lsub_positive(&util, task_util(p)); 7486 else if (p && task_cpu(p) != cpu && dst_cpu == cpu) 7487 util += task_util(p); 7488 7489 if (sched_feat(UTIL_EST)) { 7490 unsigned long util_est; 7491 7492 util_est = READ_ONCE(cfs_rq->avg.util_est.enqueued); 7493 7494 /* 7495 * During wake-up @p isn't enqueued yet and doesn't contribute 7496 * to any cpu_rq(cpu)->cfs.avg.util_est.enqueued. 7497 * If @dst_cpu == @cpu add it to "simulate" cpu_util after @p 7498 * has been enqueued. 7499 * 7500 * During exec (@dst_cpu = -1) @p is enqueued and does 7501 * contribute to cpu_rq(cpu)->cfs.util_est.enqueued. 7502 * Remove it to "simulate" cpu_util without @p's contribution. 7503 * 7504 * Despite the task_on_rq_queued(@p) check there is still a 7505 * small window for a possible race when an exec 7506 * select_task_rq_fair() races with LB's detach_task(). 7507 * 7508 * detach_task() 7509 * deactivate_task() 7510 * p->on_rq = TASK_ON_RQ_MIGRATING; 7511 * -------------------------------- A 7512 * dequeue_task() \ 7513 * dequeue_task_fair() + Race Time 7514 * util_est_dequeue() / 7515 * -------------------------------- B 7516 * 7517 * The additional check "current == p" is required to further 7518 * reduce the race window. 7519 */ 7520 if (dst_cpu == cpu) 7521 util_est += _task_util_est(p); 7522 else if (p && unlikely(task_on_rq_queued(p) || current == p)) 7523 lsub_positive(&util_est, _task_util_est(p)); 7524 7525 util = max(util, util_est); 7526 } 7527 7528 return min(util, capacity_orig_of(cpu)); 7529 } 7530 7531 unsigned long cpu_util_cfs(int cpu) 7532 { 7533 return cpu_util(cpu, NULL, -1, 0); 7534 } 7535 7536 unsigned long cpu_util_cfs_boost(int cpu) 7537 { 7538 return cpu_util(cpu, NULL, -1, 1); 7539 } 7540 7541 /* 7542 * cpu_util_without: compute cpu utilization without any contributions from *p 7543 * @cpu: the CPU which utilization is requested 7544 * @p: the task which utilization should be discounted 7545 * 7546 * The utilization of a CPU is defined by the utilization of tasks currently 7547 * enqueued on that CPU as well as tasks which are currently sleeping after an 7548 * execution on that CPU. 7549 * 7550 * This method returns the utilization of the specified CPU by discounting the 7551 * utilization of the specified task, whenever the task is currently 7552 * contributing to the CPU utilization. 7553 */ 7554 static unsigned long cpu_util_without(int cpu, struct task_struct *p) 7555 { 7556 /* Task has no contribution or is new */ 7557 if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 7558 p = NULL; 7559 7560 return cpu_util(cpu, p, -1, 0); 7561 } 7562 7563 /* 7564 * energy_env - Utilization landscape for energy estimation. 7565 * @task_busy_time: Utilization contribution by the task for which we test the 7566 * placement. Given by eenv_task_busy_time(). 7567 * @pd_busy_time: Utilization of the whole perf domain without the task 7568 * contribution. Given by eenv_pd_busy_time(). 7569 * @cpu_cap: Maximum CPU capacity for the perf domain. 7570 * @pd_cap: Entire perf domain capacity. (pd->nr_cpus * cpu_cap). 7571 */ 7572 struct energy_env { 7573 unsigned long task_busy_time; 7574 unsigned long pd_busy_time; 7575 unsigned long cpu_cap; 7576 unsigned long pd_cap; 7577 }; 7578 7579 /* 7580 * Compute the task busy time for compute_energy(). This time cannot be 7581 * injected directly into effective_cpu_util() because of the IRQ scaling. 7582 * The latter only makes sense with the most recent CPUs where the task has 7583 * run. 7584 */ 7585 static inline void eenv_task_busy_time(struct energy_env *eenv, 7586 struct task_struct *p, int prev_cpu) 7587 { 7588 unsigned long busy_time, max_cap = arch_scale_cpu_capacity(prev_cpu); 7589 unsigned long irq = cpu_util_irq(cpu_rq(prev_cpu)); 7590 7591 if (unlikely(irq >= max_cap)) 7592 busy_time = max_cap; 7593 else 7594 busy_time = scale_irq_capacity(task_util_est(p), irq, max_cap); 7595 7596 eenv->task_busy_time = busy_time; 7597 } 7598 7599 /* 7600 * Compute the perf_domain (PD) busy time for compute_energy(). Based on the 7601 * utilization for each @pd_cpus, it however doesn't take into account 7602 * clamping since the ratio (utilization / cpu_capacity) is already enough to 7603 * scale the EM reported power consumption at the (eventually clamped) 7604 * cpu_capacity. 7605 * 7606 * The contribution of the task @p for which we want to estimate the 7607 * energy cost is removed (by cpu_util()) and must be calculated 7608 * separately (see eenv_task_busy_time). This ensures: 7609 * 7610 * - A stable PD utilization, no matter which CPU of that PD we want to place 7611 * the task on. 7612 * 7613 * - A fair comparison between CPUs as the task contribution (task_util()) 7614 * will always be the same no matter which CPU utilization we rely on 7615 * (util_avg or util_est). 7616 * 7617 * Set @eenv busy time for the PD that spans @pd_cpus. This busy time can't 7618 * exceed @eenv->pd_cap. 7619 */ 7620 static inline void eenv_pd_busy_time(struct energy_env *eenv, 7621 struct cpumask *pd_cpus, 7622 struct task_struct *p) 7623 { 7624 unsigned long busy_time = 0; 7625 int cpu; 7626 7627 for_each_cpu(cpu, pd_cpus) { 7628 unsigned long util = cpu_util(cpu, p, -1, 0); 7629 7630 busy_time += effective_cpu_util(cpu, util, ENERGY_UTIL, NULL); 7631 } 7632 7633 eenv->pd_busy_time = min(eenv->pd_cap, busy_time); 7634 } 7635 7636 /* 7637 * Compute the maximum utilization for compute_energy() when the task @p 7638 * is placed on the cpu @dst_cpu. 7639 * 7640 * Returns the maximum utilization among @eenv->cpus. This utilization can't 7641 * exceed @eenv->cpu_cap. 7642 */ 7643 static inline unsigned long 7644 eenv_pd_max_util(struct energy_env *eenv, struct cpumask *pd_cpus, 7645 struct task_struct *p, int dst_cpu) 7646 { 7647 unsigned long max_util = 0; 7648 int cpu; 7649 7650 for_each_cpu(cpu, pd_cpus) { 7651 struct task_struct *tsk = (cpu == dst_cpu) ? p : NULL; 7652 unsigned long util = cpu_util(cpu, p, dst_cpu, 1); 7653 unsigned long eff_util; 7654 7655 /* 7656 * Performance domain frequency: utilization clamping 7657 * must be considered since it affects the selection 7658 * of the performance domain frequency. 7659 * NOTE: in case RT tasks are running, by default the 7660 * FREQUENCY_UTIL's utilization can be max OPP. 7661 */ 7662 eff_util = effective_cpu_util(cpu, util, FREQUENCY_UTIL, tsk); 7663 max_util = max(max_util, eff_util); 7664 } 7665 7666 return min(max_util, eenv->cpu_cap); 7667 } 7668 7669 /* 7670 * compute_energy(): Use the Energy Model to estimate the energy that @pd would 7671 * consume for a given utilization landscape @eenv. When @dst_cpu < 0, the task 7672 * contribution is ignored. 7673 */ 7674 static inline unsigned long 7675 compute_energy(struct energy_env *eenv, struct perf_domain *pd, 7676 struct cpumask *pd_cpus, struct task_struct *p, int dst_cpu) 7677 { 7678 unsigned long max_util = eenv_pd_max_util(eenv, pd_cpus, p, dst_cpu); 7679 unsigned long busy_time = eenv->pd_busy_time; 7680 7681 if (dst_cpu >= 0) 7682 busy_time = min(eenv->pd_cap, busy_time + eenv->task_busy_time); 7683 7684 return em_cpu_energy(pd->em_pd, max_util, busy_time, eenv->cpu_cap); 7685 } 7686 7687 /* 7688 * find_energy_efficient_cpu(): Find most energy-efficient target CPU for the 7689 * waking task. find_energy_efficient_cpu() looks for the CPU with maximum 7690 * spare capacity in each performance domain and uses it as a potential 7691 * candidate to execute the task. Then, it uses the Energy Model to figure 7692 * out which of the CPU candidates is the most energy-efficient. 7693 * 7694 * The rationale for this heuristic is as follows. In a performance domain, 7695 * all the most energy efficient CPU candidates (according to the Energy 7696 * Model) are those for which we'll request a low frequency. When there are 7697 * several CPUs for which the frequency request will be the same, we don't 7698 * have enough data to break the tie between them, because the Energy Model 7699 * only includes active power costs. With this model, if we assume that 7700 * frequency requests follow utilization (e.g. using schedutil), the CPU with 7701 * the maximum spare capacity in a performance domain is guaranteed to be among 7702 * the best candidates of the performance domain. 7703 * 7704 * In practice, it could be preferable from an energy standpoint to pack 7705 * small tasks on a CPU in order to let other CPUs go in deeper idle states, 7706 * but that could also hurt our chances to go cluster idle, and we have no 7707 * ways to tell with the current Energy Model if this is actually a good 7708 * idea or not. So, find_energy_efficient_cpu() basically favors 7709 * cluster-packing, and spreading inside a cluster. That should at least be 7710 * a good thing for latency, and this is consistent with the idea that most 7711 * of the energy savings of EAS come from the asymmetry of the system, and 7712 * not so much from breaking the tie between identical CPUs. That's also the 7713 * reason why EAS is enabled in the topology code only for systems where 7714 * SD_ASYM_CPUCAPACITY is set. 7715 * 7716 * NOTE: Forkees are not accepted in the energy-aware wake-up path because 7717 * they don't have any useful utilization data yet and it's not possible to 7718 * forecast their impact on energy consumption. Consequently, they will be 7719 * placed by find_idlest_cpu() on the least loaded CPU, which might turn out 7720 * to be energy-inefficient in some use-cases. The alternative would be to 7721 * bias new tasks towards specific types of CPUs first, or to try to infer 7722 * their util_avg from the parent task, but those heuristics could hurt 7723 * other use-cases too. So, until someone finds a better way to solve this, 7724 * let's keep things simple by re-using the existing slow path. 7725 */ 7726 static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu) 7727 { 7728 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 7729 unsigned long prev_delta = ULONG_MAX, best_delta = ULONG_MAX; 7730 unsigned long p_util_min = uclamp_is_used() ? uclamp_eff_value(p, UCLAMP_MIN) : 0; 7731 unsigned long p_util_max = uclamp_is_used() ? uclamp_eff_value(p, UCLAMP_MAX) : 1024; 7732 struct root_domain *rd = this_rq()->rd; 7733 int cpu, best_energy_cpu, target = -1; 7734 int prev_fits = -1, best_fits = -1; 7735 unsigned long best_thermal_cap = 0; 7736 unsigned long prev_thermal_cap = 0; 7737 struct sched_domain *sd; 7738 struct perf_domain *pd; 7739 struct energy_env eenv; 7740 7741 rcu_read_lock(); 7742 pd = rcu_dereference(rd->pd); 7743 if (!pd || READ_ONCE(rd->overutilized)) 7744 goto unlock; 7745 7746 /* 7747 * Energy-aware wake-up happens on the lowest sched_domain starting 7748 * from sd_asym_cpucapacity spanning over this_cpu and prev_cpu. 7749 */ 7750 sd = rcu_dereference(*this_cpu_ptr(&sd_asym_cpucapacity)); 7751 while (sd && !cpumask_test_cpu(prev_cpu, sched_domain_span(sd))) 7752 sd = sd->parent; 7753 if (!sd) 7754 goto unlock; 7755 7756 target = prev_cpu; 7757 7758 sync_entity_load_avg(&p->se); 7759 if (!uclamp_task_util(p, p_util_min, p_util_max)) 7760 goto unlock; 7761 7762 eenv_task_busy_time(&eenv, p, prev_cpu); 7763 7764 for (; pd; pd = pd->next) { 7765 unsigned long util_min = p_util_min, util_max = p_util_max; 7766 unsigned long cpu_cap, cpu_thermal_cap, util; 7767 unsigned long cur_delta, max_spare_cap = 0; 7768 unsigned long rq_util_min, rq_util_max; 7769 unsigned long prev_spare_cap = 0; 7770 int max_spare_cap_cpu = -1; 7771 unsigned long base_energy; 7772 int fits, max_fits = -1; 7773 7774 cpumask_and(cpus, perf_domain_span(pd), cpu_online_mask); 7775 7776 if (cpumask_empty(cpus)) 7777 continue; 7778 7779 /* Account thermal pressure for the energy estimation */ 7780 cpu = cpumask_first(cpus); 7781 cpu_thermal_cap = arch_scale_cpu_capacity(cpu); 7782 cpu_thermal_cap -= arch_scale_thermal_pressure(cpu); 7783 7784 eenv.cpu_cap = cpu_thermal_cap; 7785 eenv.pd_cap = 0; 7786 7787 for_each_cpu(cpu, cpus) { 7788 struct rq *rq = cpu_rq(cpu); 7789 7790 eenv.pd_cap += cpu_thermal_cap; 7791 7792 if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) 7793 continue; 7794 7795 if (!cpumask_test_cpu(cpu, p->cpus_ptr)) 7796 continue; 7797 7798 util = cpu_util(cpu, p, cpu, 0); 7799 cpu_cap = capacity_of(cpu); 7800 7801 /* 7802 * Skip CPUs that cannot satisfy the capacity request. 7803 * IOW, placing the task there would make the CPU 7804 * overutilized. Take uclamp into account to see how 7805 * much capacity we can get out of the CPU; this is 7806 * aligned with sched_cpu_util(). 7807 */ 7808 if (uclamp_is_used() && !uclamp_rq_is_idle(rq)) { 7809 /* 7810 * Open code uclamp_rq_util_with() except for 7811 * the clamp() part. Ie: apply max aggregation 7812 * only. util_fits_cpu() logic requires to 7813 * operate on non clamped util but must use the 7814 * max-aggregated uclamp_{min, max}. 7815 */ 7816 rq_util_min = uclamp_rq_get(rq, UCLAMP_MIN); 7817 rq_util_max = uclamp_rq_get(rq, UCLAMP_MAX); 7818 7819 util_min = max(rq_util_min, p_util_min); 7820 util_max = max(rq_util_max, p_util_max); 7821 } 7822 7823 fits = util_fits_cpu(util, util_min, util_max, cpu); 7824 if (!fits) 7825 continue; 7826 7827 lsub_positive(&cpu_cap, util); 7828 7829 if (cpu == prev_cpu) { 7830 /* Always use prev_cpu as a candidate. */ 7831 prev_spare_cap = cpu_cap; 7832 prev_fits = fits; 7833 } else if ((fits > max_fits) || 7834 ((fits == max_fits) && (cpu_cap > max_spare_cap))) { 7835 /* 7836 * Find the CPU with the maximum spare capacity 7837 * among the remaining CPUs in the performance 7838 * domain. 7839 */ 7840 max_spare_cap = cpu_cap; 7841 max_spare_cap_cpu = cpu; 7842 max_fits = fits; 7843 } 7844 } 7845 7846 if (max_spare_cap_cpu < 0 && prev_spare_cap == 0) 7847 continue; 7848 7849 eenv_pd_busy_time(&eenv, cpus, p); 7850 /* Compute the 'base' energy of the pd, without @p */ 7851 base_energy = compute_energy(&eenv, pd, cpus, p, -1); 7852 7853 /* Evaluate the energy impact of using prev_cpu. */ 7854 if (prev_spare_cap > 0) { 7855 prev_delta = compute_energy(&eenv, pd, cpus, p, 7856 prev_cpu); 7857 /* CPU utilization has changed */ 7858 if (prev_delta < base_energy) 7859 goto unlock; 7860 prev_delta -= base_energy; 7861 prev_thermal_cap = cpu_thermal_cap; 7862 best_delta = min(best_delta, prev_delta); 7863 } 7864 7865 /* Evaluate the energy impact of using max_spare_cap_cpu. */ 7866 if (max_spare_cap_cpu >= 0 && max_spare_cap > prev_spare_cap) { 7867 /* Current best energy cpu fits better */ 7868 if (max_fits < best_fits) 7869 continue; 7870 7871 /* 7872 * Both don't fit performance hint (i.e. uclamp_min) 7873 * but best energy cpu has better capacity. 7874 */ 7875 if ((max_fits < 0) && 7876 (cpu_thermal_cap <= best_thermal_cap)) 7877 continue; 7878 7879 cur_delta = compute_energy(&eenv, pd, cpus, p, 7880 max_spare_cap_cpu); 7881 /* CPU utilization has changed */ 7882 if (cur_delta < base_energy) 7883 goto unlock; 7884 cur_delta -= base_energy; 7885 7886 /* 7887 * Both fit for the task but best energy cpu has lower 7888 * energy impact. 7889 */ 7890 if ((max_fits > 0) && (best_fits > 0) && 7891 (cur_delta >= best_delta)) 7892 continue; 7893 7894 best_delta = cur_delta; 7895 best_energy_cpu = max_spare_cap_cpu; 7896 best_fits = max_fits; 7897 best_thermal_cap = cpu_thermal_cap; 7898 } 7899 } 7900 rcu_read_unlock(); 7901 7902 if ((best_fits > prev_fits) || 7903 ((best_fits > 0) && (best_delta < prev_delta)) || 7904 ((best_fits < 0) && (best_thermal_cap > prev_thermal_cap))) 7905 target = best_energy_cpu; 7906 7907 return target; 7908 7909 unlock: 7910 rcu_read_unlock(); 7911 7912 return target; 7913 } 7914 7915 /* 7916 * select_task_rq_fair: Select target runqueue for the waking task in domains 7917 * that have the relevant SD flag set. In practice, this is SD_BALANCE_WAKE, 7918 * SD_BALANCE_FORK, or SD_BALANCE_EXEC. 7919 * 7920 * Balances load by selecting the idlest CPU in the idlest group, or under 7921 * certain conditions an idle sibling CPU if the domain has SD_WAKE_AFFINE set. 7922 * 7923 * Returns the target CPU number. 7924 */ 7925 static int 7926 select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags) 7927 { 7928 int sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING); 7929 struct sched_domain *tmp, *sd = NULL; 7930 int cpu = smp_processor_id(); 7931 int new_cpu = prev_cpu; 7932 int want_affine = 0; 7933 /* SD_flags and WF_flags share the first nibble */ 7934 int sd_flag = wake_flags & 0xF; 7935 7936 /* 7937 * required for stable ->cpus_allowed 7938 */ 7939 lockdep_assert_held(&p->pi_lock); 7940 if (wake_flags & WF_TTWU) { 7941 record_wakee(p); 7942 7943 if ((wake_flags & WF_CURRENT_CPU) && 7944 cpumask_test_cpu(cpu, p->cpus_ptr)) 7945 return cpu; 7946 7947 if (sched_energy_enabled()) { 7948 new_cpu = find_energy_efficient_cpu(p, prev_cpu); 7949 if (new_cpu >= 0) 7950 return new_cpu; 7951 new_cpu = prev_cpu; 7952 } 7953 7954 want_affine = !wake_wide(p) && cpumask_test_cpu(cpu, p->cpus_ptr); 7955 } 7956 7957 rcu_read_lock(); 7958 for_each_domain(cpu, tmp) { 7959 /* 7960 * If both 'cpu' and 'prev_cpu' are part of this domain, 7961 * cpu is a valid SD_WAKE_AFFINE target. 7962 */ 7963 if (want_affine && (tmp->flags & SD_WAKE_AFFINE) && 7964 cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) { 7965 if (cpu != prev_cpu) 7966 new_cpu = wake_affine(tmp, p, cpu, prev_cpu, sync); 7967 7968 sd = NULL; /* Prefer wake_affine over balance flags */ 7969 break; 7970 } 7971 7972 /* 7973 * Usually only true for WF_EXEC and WF_FORK, as sched_domains 7974 * usually do not have SD_BALANCE_WAKE set. That means wakeup 7975 * will usually go to the fast path. 7976 */ 7977 if (tmp->flags & sd_flag) 7978 sd = tmp; 7979 else if (!want_affine) 7980 break; 7981 } 7982 7983 if (unlikely(sd)) { 7984 /* Slow path */ 7985 new_cpu = find_idlest_cpu(sd, p, cpu, prev_cpu, sd_flag); 7986 } else if (wake_flags & WF_TTWU) { /* XXX always ? */ 7987 /* Fast path */ 7988 new_cpu = select_idle_sibling(p, prev_cpu, new_cpu); 7989 } 7990 rcu_read_unlock(); 7991 7992 return new_cpu; 7993 } 7994 7995 /* 7996 * Called immediately before a task is migrated to a new CPU; task_cpu(p) and 7997 * cfs_rq_of(p) references at time of call are still valid and identify the 7998 * previous CPU. The caller guarantees p->pi_lock or task_rq(p)->lock is held. 7999 */ 8000 static void migrate_task_rq_fair(struct task_struct *p, int new_cpu) 8001 { 8002 struct sched_entity *se = &p->se; 8003 8004 if (!task_on_rq_migrating(p)) { 8005 remove_entity_load_avg(se); 8006 8007 /* 8008 * Here, the task's PELT values have been updated according to 8009 * the current rq's clock. But if that clock hasn't been 8010 * updated in a while, a substantial idle time will be missed, 8011 * leading to an inflation after wake-up on the new rq. 8012 * 8013 * Estimate the missing time from the cfs_rq last_update_time 8014 * and update sched_avg to improve the PELT continuity after 8015 * migration. 8016 */ 8017 migrate_se_pelt_lag(se); 8018 } 8019 8020 /* Tell new CPU we are migrated */ 8021 se->avg.last_update_time = 0; 8022 8023 update_scan_period(p, new_cpu); 8024 } 8025 8026 static void task_dead_fair(struct task_struct *p) 8027 { 8028 remove_entity_load_avg(&p->se); 8029 } 8030 8031 static int 8032 balance_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 8033 { 8034 if (rq->nr_running) 8035 return 1; 8036 8037 return newidle_balance(rq, rf) != 0; 8038 } 8039 #endif /* CONFIG_SMP */ 8040 8041 static void set_next_buddy(struct sched_entity *se) 8042 { 8043 for_each_sched_entity(se) { 8044 if (SCHED_WARN_ON(!se->on_rq)) 8045 return; 8046 if (se_is_idle(se)) 8047 return; 8048 cfs_rq_of(se)->next = se; 8049 } 8050 } 8051 8052 /* 8053 * Preempt the current task with a newly woken task if needed: 8054 */ 8055 static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_flags) 8056 { 8057 struct task_struct *curr = rq->curr; 8058 struct sched_entity *se = &curr->se, *pse = &p->se; 8059 struct cfs_rq *cfs_rq = task_cfs_rq(curr); 8060 int next_buddy_marked = 0; 8061 int cse_is_idle, pse_is_idle; 8062 8063 if (unlikely(se == pse)) 8064 return; 8065 8066 /* 8067 * This is possible from callers such as attach_tasks(), in which we 8068 * unconditionally check_preempt_curr() after an enqueue (which may have 8069 * lead to a throttle). This both saves work and prevents false 8070 * next-buddy nomination below. 8071 */ 8072 if (unlikely(throttled_hierarchy(cfs_rq_of(pse)))) 8073 return; 8074 8075 if (sched_feat(NEXT_BUDDY) && !(wake_flags & WF_FORK)) { 8076 set_next_buddy(pse); 8077 next_buddy_marked = 1; 8078 } 8079 8080 /* 8081 * We can come here with TIF_NEED_RESCHED already set from new task 8082 * wake up path. 8083 * 8084 * Note: this also catches the edge-case of curr being in a throttled 8085 * group (e.g. via set_curr_task), since update_curr() (in the 8086 * enqueue of curr) will have resulted in resched being set. This 8087 * prevents us from potentially nominating it as a false LAST_BUDDY 8088 * below. 8089 */ 8090 if (test_tsk_need_resched(curr)) 8091 return; 8092 8093 /* Idle tasks are by definition preempted by non-idle tasks. */ 8094 if (unlikely(task_has_idle_policy(curr)) && 8095 likely(!task_has_idle_policy(p))) 8096 goto preempt; 8097 8098 /* 8099 * Batch and idle tasks do not preempt non-idle tasks (their preemption 8100 * is driven by the tick): 8101 */ 8102 if (unlikely(p->policy != SCHED_NORMAL) || !sched_feat(WAKEUP_PREEMPTION)) 8103 return; 8104 8105 find_matching_se(&se, &pse); 8106 WARN_ON_ONCE(!pse); 8107 8108 cse_is_idle = se_is_idle(se); 8109 pse_is_idle = se_is_idle(pse); 8110 8111 /* 8112 * Preempt an idle group in favor of a non-idle group (and don't preempt 8113 * in the inverse case). 8114 */ 8115 if (cse_is_idle && !pse_is_idle) 8116 goto preempt; 8117 if (cse_is_idle != pse_is_idle) 8118 return; 8119 8120 cfs_rq = cfs_rq_of(se); 8121 update_curr(cfs_rq); 8122 8123 /* 8124 * XXX pick_eevdf(cfs_rq) != se ? 8125 */ 8126 if (pick_eevdf(cfs_rq) == pse) 8127 goto preempt; 8128 8129 return; 8130 8131 preempt: 8132 resched_curr(rq); 8133 } 8134 8135 #ifdef CONFIG_SMP 8136 static struct task_struct *pick_task_fair(struct rq *rq) 8137 { 8138 struct sched_entity *se; 8139 struct cfs_rq *cfs_rq; 8140 8141 again: 8142 cfs_rq = &rq->cfs; 8143 if (!cfs_rq->nr_running) 8144 return NULL; 8145 8146 do { 8147 struct sched_entity *curr = cfs_rq->curr; 8148 8149 /* When we pick for a remote RQ, we'll not have done put_prev_entity() */ 8150 if (curr) { 8151 if (curr->on_rq) 8152 update_curr(cfs_rq); 8153 else 8154 curr = NULL; 8155 8156 if (unlikely(check_cfs_rq_runtime(cfs_rq))) 8157 goto again; 8158 } 8159 8160 se = pick_next_entity(cfs_rq, curr); 8161 cfs_rq = group_cfs_rq(se); 8162 } while (cfs_rq); 8163 8164 return task_of(se); 8165 } 8166 #endif 8167 8168 struct task_struct * 8169 pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 8170 { 8171 struct cfs_rq *cfs_rq = &rq->cfs; 8172 struct sched_entity *se; 8173 struct task_struct *p; 8174 int new_tasks; 8175 8176 again: 8177 if (!sched_fair_runnable(rq)) 8178 goto idle; 8179 8180 #ifdef CONFIG_FAIR_GROUP_SCHED 8181 if (!prev || prev->sched_class != &fair_sched_class) 8182 goto simple; 8183 8184 /* 8185 * Because of the set_next_buddy() in dequeue_task_fair() it is rather 8186 * likely that a next task is from the same cgroup as the current. 8187 * 8188 * Therefore attempt to avoid putting and setting the entire cgroup 8189 * hierarchy, only change the part that actually changes. 8190 */ 8191 8192 do { 8193 struct sched_entity *curr = cfs_rq->curr; 8194 8195 /* 8196 * Since we got here without doing put_prev_entity() we also 8197 * have to consider cfs_rq->curr. If it is still a runnable 8198 * entity, update_curr() will update its vruntime, otherwise 8199 * forget we've ever seen it. 8200 */ 8201 if (curr) { 8202 if (curr->on_rq) 8203 update_curr(cfs_rq); 8204 else 8205 curr = NULL; 8206 8207 /* 8208 * This call to check_cfs_rq_runtime() will do the 8209 * throttle and dequeue its entity in the parent(s). 8210 * Therefore the nr_running test will indeed 8211 * be correct. 8212 */ 8213 if (unlikely(check_cfs_rq_runtime(cfs_rq))) { 8214 cfs_rq = &rq->cfs; 8215 8216 if (!cfs_rq->nr_running) 8217 goto idle; 8218 8219 goto simple; 8220 } 8221 } 8222 8223 se = pick_next_entity(cfs_rq, curr); 8224 cfs_rq = group_cfs_rq(se); 8225 } while (cfs_rq); 8226 8227 p = task_of(se); 8228 8229 /* 8230 * Since we haven't yet done put_prev_entity and if the selected task 8231 * is a different task than we started out with, try and touch the 8232 * least amount of cfs_rqs. 8233 */ 8234 if (prev != p) { 8235 struct sched_entity *pse = &prev->se; 8236 8237 while (!(cfs_rq = is_same_group(se, pse))) { 8238 int se_depth = se->depth; 8239 int pse_depth = pse->depth; 8240 8241 if (se_depth <= pse_depth) { 8242 put_prev_entity(cfs_rq_of(pse), pse); 8243 pse = parent_entity(pse); 8244 } 8245 if (se_depth >= pse_depth) { 8246 set_next_entity(cfs_rq_of(se), se); 8247 se = parent_entity(se); 8248 } 8249 } 8250 8251 put_prev_entity(cfs_rq, pse); 8252 set_next_entity(cfs_rq, se); 8253 } 8254 8255 goto done; 8256 simple: 8257 #endif 8258 if (prev) 8259 put_prev_task(rq, prev); 8260 8261 do { 8262 se = pick_next_entity(cfs_rq, NULL); 8263 set_next_entity(cfs_rq, se); 8264 cfs_rq = group_cfs_rq(se); 8265 } while (cfs_rq); 8266 8267 p = task_of(se); 8268 8269 done: __maybe_unused; 8270 #ifdef CONFIG_SMP 8271 /* 8272 * Move the next running task to the front of 8273 * the list, so our cfs_tasks list becomes MRU 8274 * one. 8275 */ 8276 list_move(&p->se.group_node, &rq->cfs_tasks); 8277 #endif 8278 8279 if (hrtick_enabled_fair(rq)) 8280 hrtick_start_fair(rq, p); 8281 8282 update_misfit_status(p, rq); 8283 sched_fair_update_stop_tick(rq, p); 8284 8285 return p; 8286 8287 idle: 8288 if (!rf) 8289 return NULL; 8290 8291 new_tasks = newidle_balance(rq, rf); 8292 8293 /* 8294 * Because newidle_balance() releases (and re-acquires) rq->lock, it is 8295 * possible for any higher priority task to appear. In that case we 8296 * must re-start the pick_next_entity() loop. 8297 */ 8298 if (new_tasks < 0) 8299 return RETRY_TASK; 8300 8301 if (new_tasks > 0) 8302 goto again; 8303 8304 /* 8305 * rq is about to be idle, check if we need to update the 8306 * lost_idle_time of clock_pelt 8307 */ 8308 update_idle_rq_clock_pelt(rq); 8309 8310 return NULL; 8311 } 8312 8313 static struct task_struct *__pick_next_task_fair(struct rq *rq) 8314 { 8315 return pick_next_task_fair(rq, NULL, NULL); 8316 } 8317 8318 /* 8319 * Account for a descheduled task: 8320 */ 8321 static void put_prev_task_fair(struct rq *rq, struct task_struct *prev) 8322 { 8323 struct sched_entity *se = &prev->se; 8324 struct cfs_rq *cfs_rq; 8325 8326 for_each_sched_entity(se) { 8327 cfs_rq = cfs_rq_of(se); 8328 put_prev_entity(cfs_rq, se); 8329 } 8330 } 8331 8332 /* 8333 * sched_yield() is very simple 8334 */ 8335 static void yield_task_fair(struct rq *rq) 8336 { 8337 struct task_struct *curr = rq->curr; 8338 struct cfs_rq *cfs_rq = task_cfs_rq(curr); 8339 struct sched_entity *se = &curr->se; 8340 8341 /* 8342 * Are we the only task in the tree? 8343 */ 8344 if (unlikely(rq->nr_running == 1)) 8345 return; 8346 8347 clear_buddies(cfs_rq, se); 8348 8349 update_rq_clock(rq); 8350 /* 8351 * Update run-time statistics of the 'current'. 8352 */ 8353 update_curr(cfs_rq); 8354 /* 8355 * Tell update_rq_clock() that we've just updated, 8356 * so we don't do microscopic update in schedule() 8357 * and double the fastpath cost. 8358 */ 8359 rq_clock_skip_update(rq); 8360 8361 se->deadline += calc_delta_fair(se->slice, se); 8362 } 8363 8364 static bool yield_to_task_fair(struct rq *rq, struct task_struct *p) 8365 { 8366 struct sched_entity *se = &p->se; 8367 8368 /* throttled hierarchies are not runnable */ 8369 if (!se->on_rq || throttled_hierarchy(cfs_rq_of(se))) 8370 return false; 8371 8372 /* Tell the scheduler that we'd really like pse to run next. */ 8373 set_next_buddy(se); 8374 8375 yield_task_fair(rq); 8376 8377 return true; 8378 } 8379 8380 #ifdef CONFIG_SMP 8381 /************************************************** 8382 * Fair scheduling class load-balancing methods. 8383 * 8384 * BASICS 8385 * 8386 * The purpose of load-balancing is to achieve the same basic fairness the 8387 * per-CPU scheduler provides, namely provide a proportional amount of compute 8388 * time to each task. This is expressed in the following equation: 8389 * 8390 * W_i,n/P_i == W_j,n/P_j for all i,j (1) 8391 * 8392 * Where W_i,n is the n-th weight average for CPU i. The instantaneous weight 8393 * W_i,0 is defined as: 8394 * 8395 * W_i,0 = \Sum_j w_i,j (2) 8396 * 8397 * Where w_i,j is the weight of the j-th runnable task on CPU i. This weight 8398 * is derived from the nice value as per sched_prio_to_weight[]. 8399 * 8400 * The weight average is an exponential decay average of the instantaneous 8401 * weight: 8402 * 8403 * W'_i,n = (2^n - 1) / 2^n * W_i,n + 1 / 2^n * W_i,0 (3) 8404 * 8405 * C_i is the compute capacity of CPU i, typically it is the 8406 * fraction of 'recent' time available for SCHED_OTHER task execution. But it 8407 * can also include other factors [XXX]. 8408 * 8409 * To achieve this balance we define a measure of imbalance which follows 8410 * directly from (1): 8411 * 8412 * imb_i,j = max{ avg(W/C), W_i/C_i } - min{ avg(W/C), W_j/C_j } (4) 8413 * 8414 * We them move tasks around to minimize the imbalance. In the continuous 8415 * function space it is obvious this converges, in the discrete case we get 8416 * a few fun cases generally called infeasible weight scenarios. 8417 * 8418 * [XXX expand on: 8419 * - infeasible weights; 8420 * - local vs global optima in the discrete case. ] 8421 * 8422 * 8423 * SCHED DOMAINS 8424 * 8425 * In order to solve the imbalance equation (4), and avoid the obvious O(n^2) 8426 * for all i,j solution, we create a tree of CPUs that follows the hardware 8427 * topology where each level pairs two lower groups (or better). This results 8428 * in O(log n) layers. Furthermore we reduce the number of CPUs going up the 8429 * tree to only the first of the previous level and we decrease the frequency 8430 * of load-balance at each level inv. proportional to the number of CPUs in 8431 * the groups. 8432 * 8433 * This yields: 8434 * 8435 * log_2 n 1 n 8436 * \Sum { --- * --- * 2^i } = O(n) (5) 8437 * i = 0 2^i 2^i 8438 * `- size of each group 8439 * | | `- number of CPUs doing load-balance 8440 * | `- freq 8441 * `- sum over all levels 8442 * 8443 * Coupled with a limit on how many tasks we can migrate every balance pass, 8444 * this makes (5) the runtime complexity of the balancer. 8445 * 8446 * An important property here is that each CPU is still (indirectly) connected 8447 * to every other CPU in at most O(log n) steps: 8448 * 8449 * The adjacency matrix of the resulting graph is given by: 8450 * 8451 * log_2 n 8452 * A_i,j = \Union (i % 2^k == 0) && i / 2^(k+1) == j / 2^(k+1) (6) 8453 * k = 0 8454 * 8455 * And you'll find that: 8456 * 8457 * A^(log_2 n)_i,j != 0 for all i,j (7) 8458 * 8459 * Showing there's indeed a path between every CPU in at most O(log n) steps. 8460 * The task movement gives a factor of O(m), giving a convergence complexity 8461 * of: 8462 * 8463 * O(nm log n), n := nr_cpus, m := nr_tasks (8) 8464 * 8465 * 8466 * WORK CONSERVING 8467 * 8468 * In order to avoid CPUs going idle while there's still work to do, new idle 8469 * balancing is more aggressive and has the newly idle CPU iterate up the domain 8470 * tree itself instead of relying on other CPUs to bring it work. 8471 * 8472 * This adds some complexity to both (5) and (8) but it reduces the total idle 8473 * time. 8474 * 8475 * [XXX more?] 8476 * 8477 * 8478 * CGROUPS 8479 * 8480 * Cgroups make a horror show out of (2), instead of a simple sum we get: 8481 * 8482 * s_k,i 8483 * W_i,0 = \Sum_j \Prod_k w_k * ----- (9) 8484 * S_k 8485 * 8486 * Where 8487 * 8488 * s_k,i = \Sum_j w_i,j,k and S_k = \Sum_i s_k,i (10) 8489 * 8490 * w_i,j,k is the weight of the j-th runnable task in the k-th cgroup on CPU i. 8491 * 8492 * The big problem is S_k, its a global sum needed to compute a local (W_i) 8493 * property. 8494 * 8495 * [XXX write more on how we solve this.. _after_ merging pjt's patches that 8496 * rewrite all of this once again.] 8497 */ 8498 8499 static unsigned long __read_mostly max_load_balance_interval = HZ/10; 8500 8501 enum fbq_type { regular, remote, all }; 8502 8503 /* 8504 * 'group_type' describes the group of CPUs at the moment of load balancing. 8505 * 8506 * The enum is ordered by pulling priority, with the group with lowest priority 8507 * first so the group_type can simply be compared when selecting the busiest 8508 * group. See update_sd_pick_busiest(). 8509 */ 8510 enum group_type { 8511 /* The group has spare capacity that can be used to run more tasks. */ 8512 group_has_spare = 0, 8513 /* 8514 * The group is fully used and the tasks don't compete for more CPU 8515 * cycles. Nevertheless, some tasks might wait before running. 8516 */ 8517 group_fully_busy, 8518 /* 8519 * One task doesn't fit with CPU's capacity and must be migrated to a 8520 * more powerful CPU. 8521 */ 8522 group_misfit_task, 8523 /* 8524 * Balance SMT group that's fully busy. Can benefit from migration 8525 * a task on SMT with busy sibling to another CPU on idle core. 8526 */ 8527 group_smt_balance, 8528 /* 8529 * SD_ASYM_PACKING only: One local CPU with higher capacity is available, 8530 * and the task should be migrated to it instead of running on the 8531 * current CPU. 8532 */ 8533 group_asym_packing, 8534 /* 8535 * The tasks' affinity constraints previously prevented the scheduler 8536 * from balancing the load across the system. 8537 */ 8538 group_imbalanced, 8539 /* 8540 * The CPU is overloaded and can't provide expected CPU cycles to all 8541 * tasks. 8542 */ 8543 group_overloaded 8544 }; 8545 8546 enum migration_type { 8547 migrate_load = 0, 8548 migrate_util, 8549 migrate_task, 8550 migrate_misfit 8551 }; 8552 8553 #define LBF_ALL_PINNED 0x01 8554 #define LBF_NEED_BREAK 0x02 8555 #define LBF_DST_PINNED 0x04 8556 #define LBF_SOME_PINNED 0x08 8557 #define LBF_ACTIVE_LB 0x10 8558 8559 struct lb_env { 8560 struct sched_domain *sd; 8561 8562 struct rq *src_rq; 8563 int src_cpu; 8564 8565 int dst_cpu; 8566 struct rq *dst_rq; 8567 8568 struct cpumask *dst_grpmask; 8569 int new_dst_cpu; 8570 enum cpu_idle_type idle; 8571 long imbalance; 8572 /* The set of CPUs under consideration for load-balancing */ 8573 struct cpumask *cpus; 8574 8575 unsigned int flags; 8576 8577 unsigned int loop; 8578 unsigned int loop_break; 8579 unsigned int loop_max; 8580 8581 enum fbq_type fbq_type; 8582 enum migration_type migration_type; 8583 struct list_head tasks; 8584 }; 8585 8586 /* 8587 * Is this task likely cache-hot: 8588 */ 8589 static int task_hot(struct task_struct *p, struct lb_env *env) 8590 { 8591 s64 delta; 8592 8593 lockdep_assert_rq_held(env->src_rq); 8594 8595 if (p->sched_class != &fair_sched_class) 8596 return 0; 8597 8598 if (unlikely(task_has_idle_policy(p))) 8599 return 0; 8600 8601 /* SMT siblings share cache */ 8602 if (env->sd->flags & SD_SHARE_CPUCAPACITY) 8603 return 0; 8604 8605 /* 8606 * Buddy candidates are cache hot: 8607 */ 8608 if (sched_feat(CACHE_HOT_BUDDY) && env->dst_rq->nr_running && 8609 (&p->se == cfs_rq_of(&p->se)->next)) 8610 return 1; 8611 8612 if (sysctl_sched_migration_cost == -1) 8613 return 1; 8614 8615 /* 8616 * Don't migrate task if the task's cookie does not match 8617 * with the destination CPU's core cookie. 8618 */ 8619 if (!sched_core_cookie_match(cpu_rq(env->dst_cpu), p)) 8620 return 1; 8621 8622 if (sysctl_sched_migration_cost == 0) 8623 return 0; 8624 8625 delta = rq_clock_task(env->src_rq) - p->se.exec_start; 8626 8627 return delta < (s64)sysctl_sched_migration_cost; 8628 } 8629 8630 #ifdef CONFIG_NUMA_BALANCING 8631 /* 8632 * Returns 1, if task migration degrades locality 8633 * Returns 0, if task migration improves locality i.e migration preferred. 8634 * Returns -1, if task migration is not affected by locality. 8635 */ 8636 static int migrate_degrades_locality(struct task_struct *p, struct lb_env *env) 8637 { 8638 struct numa_group *numa_group = rcu_dereference(p->numa_group); 8639 unsigned long src_weight, dst_weight; 8640 int src_nid, dst_nid, dist; 8641 8642 if (!static_branch_likely(&sched_numa_balancing)) 8643 return -1; 8644 8645 if (!p->numa_faults || !(env->sd->flags & SD_NUMA)) 8646 return -1; 8647 8648 src_nid = cpu_to_node(env->src_cpu); 8649 dst_nid = cpu_to_node(env->dst_cpu); 8650 8651 if (src_nid == dst_nid) 8652 return -1; 8653 8654 /* Migrating away from the preferred node is always bad. */ 8655 if (src_nid == p->numa_preferred_nid) { 8656 if (env->src_rq->nr_running > env->src_rq->nr_preferred_running) 8657 return 1; 8658 else 8659 return -1; 8660 } 8661 8662 /* Encourage migration to the preferred node. */ 8663 if (dst_nid == p->numa_preferred_nid) 8664 return 0; 8665 8666 /* Leaving a core idle is often worse than degrading locality. */ 8667 if (env->idle == CPU_IDLE) 8668 return -1; 8669 8670 dist = node_distance(src_nid, dst_nid); 8671 if (numa_group) { 8672 src_weight = group_weight(p, src_nid, dist); 8673 dst_weight = group_weight(p, dst_nid, dist); 8674 } else { 8675 src_weight = task_weight(p, src_nid, dist); 8676 dst_weight = task_weight(p, dst_nid, dist); 8677 } 8678 8679 return dst_weight < src_weight; 8680 } 8681 8682 #else 8683 static inline int migrate_degrades_locality(struct task_struct *p, 8684 struct lb_env *env) 8685 { 8686 return -1; 8687 } 8688 #endif 8689 8690 /* 8691 * can_migrate_task - may task p from runqueue rq be migrated to this_cpu? 8692 */ 8693 static 8694 int can_migrate_task(struct task_struct *p, struct lb_env *env) 8695 { 8696 int tsk_cache_hot; 8697 8698 lockdep_assert_rq_held(env->src_rq); 8699 8700 /* 8701 * We do not migrate tasks that are: 8702 * 1) throttled_lb_pair, or 8703 * 2) cannot be migrated to this CPU due to cpus_ptr, or 8704 * 3) running (obviously), or 8705 * 4) are cache-hot on their current CPU. 8706 */ 8707 if (throttled_lb_pair(task_group(p), env->src_cpu, env->dst_cpu)) 8708 return 0; 8709 8710 /* Disregard pcpu kthreads; they are where they need to be. */ 8711 if (kthread_is_per_cpu(p)) 8712 return 0; 8713 8714 if (!cpumask_test_cpu(env->dst_cpu, p->cpus_ptr)) { 8715 int cpu; 8716 8717 schedstat_inc(p->stats.nr_failed_migrations_affine); 8718 8719 env->flags |= LBF_SOME_PINNED; 8720 8721 /* 8722 * Remember if this task can be migrated to any other CPU in 8723 * our sched_group. We may want to revisit it if we couldn't 8724 * meet load balance goals by pulling other tasks on src_cpu. 8725 * 8726 * Avoid computing new_dst_cpu 8727 * - for NEWLY_IDLE 8728 * - if we have already computed one in current iteration 8729 * - if it's an active balance 8730 */ 8731 if (env->idle == CPU_NEWLY_IDLE || 8732 env->flags & (LBF_DST_PINNED | LBF_ACTIVE_LB)) 8733 return 0; 8734 8735 /* Prevent to re-select dst_cpu via env's CPUs: */ 8736 for_each_cpu_and(cpu, env->dst_grpmask, env->cpus) { 8737 if (cpumask_test_cpu(cpu, p->cpus_ptr)) { 8738 env->flags |= LBF_DST_PINNED; 8739 env->new_dst_cpu = cpu; 8740 break; 8741 } 8742 } 8743 8744 return 0; 8745 } 8746 8747 /* Record that we found at least one task that could run on dst_cpu */ 8748 env->flags &= ~LBF_ALL_PINNED; 8749 8750 if (task_on_cpu(env->src_rq, p)) { 8751 schedstat_inc(p->stats.nr_failed_migrations_running); 8752 return 0; 8753 } 8754 8755 /* 8756 * Aggressive migration if: 8757 * 1) active balance 8758 * 2) destination numa is preferred 8759 * 3) task is cache cold, or 8760 * 4) too many balance attempts have failed. 8761 */ 8762 if (env->flags & LBF_ACTIVE_LB) 8763 return 1; 8764 8765 tsk_cache_hot = migrate_degrades_locality(p, env); 8766 if (tsk_cache_hot == -1) 8767 tsk_cache_hot = task_hot(p, env); 8768 8769 if (tsk_cache_hot <= 0 || 8770 env->sd->nr_balance_failed > env->sd->cache_nice_tries) { 8771 if (tsk_cache_hot == 1) { 8772 schedstat_inc(env->sd->lb_hot_gained[env->idle]); 8773 schedstat_inc(p->stats.nr_forced_migrations); 8774 } 8775 return 1; 8776 } 8777 8778 schedstat_inc(p->stats.nr_failed_migrations_hot); 8779 return 0; 8780 } 8781 8782 /* 8783 * detach_task() -- detach the task for the migration specified in env 8784 */ 8785 static void detach_task(struct task_struct *p, struct lb_env *env) 8786 { 8787 lockdep_assert_rq_held(env->src_rq); 8788 8789 deactivate_task(env->src_rq, p, DEQUEUE_NOCLOCK); 8790 set_task_cpu(p, env->dst_cpu); 8791 } 8792 8793 /* 8794 * detach_one_task() -- tries to dequeue exactly one task from env->src_rq, as 8795 * part of active balancing operations within "domain". 8796 * 8797 * Returns a task if successful and NULL otherwise. 8798 */ 8799 static struct task_struct *detach_one_task(struct lb_env *env) 8800 { 8801 struct task_struct *p; 8802 8803 lockdep_assert_rq_held(env->src_rq); 8804 8805 list_for_each_entry_reverse(p, 8806 &env->src_rq->cfs_tasks, se.group_node) { 8807 if (!can_migrate_task(p, env)) 8808 continue; 8809 8810 detach_task(p, env); 8811 8812 /* 8813 * Right now, this is only the second place where 8814 * lb_gained[env->idle] is updated (other is detach_tasks) 8815 * so we can safely collect stats here rather than 8816 * inside detach_tasks(). 8817 */ 8818 schedstat_inc(env->sd->lb_gained[env->idle]); 8819 return p; 8820 } 8821 return NULL; 8822 } 8823 8824 /* 8825 * detach_tasks() -- tries to detach up to imbalance load/util/tasks from 8826 * busiest_rq, as part of a balancing operation within domain "sd". 8827 * 8828 * Returns number of detached tasks if successful and 0 otherwise. 8829 */ 8830 static int detach_tasks(struct lb_env *env) 8831 { 8832 struct list_head *tasks = &env->src_rq->cfs_tasks; 8833 unsigned long util, load; 8834 struct task_struct *p; 8835 int detached = 0; 8836 8837 lockdep_assert_rq_held(env->src_rq); 8838 8839 /* 8840 * Source run queue has been emptied by another CPU, clear 8841 * LBF_ALL_PINNED flag as we will not test any task. 8842 */ 8843 if (env->src_rq->nr_running <= 1) { 8844 env->flags &= ~LBF_ALL_PINNED; 8845 return 0; 8846 } 8847 8848 if (env->imbalance <= 0) 8849 return 0; 8850 8851 while (!list_empty(tasks)) { 8852 /* 8853 * We don't want to steal all, otherwise we may be treated likewise, 8854 * which could at worst lead to a livelock crash. 8855 */ 8856 if (env->idle != CPU_NOT_IDLE && env->src_rq->nr_running <= 1) 8857 break; 8858 8859 env->loop++; 8860 /* 8861 * We've more or less seen every task there is, call it quits 8862 * unless we haven't found any movable task yet. 8863 */ 8864 if (env->loop > env->loop_max && 8865 !(env->flags & LBF_ALL_PINNED)) 8866 break; 8867 8868 /* take a breather every nr_migrate tasks */ 8869 if (env->loop > env->loop_break) { 8870 env->loop_break += SCHED_NR_MIGRATE_BREAK; 8871 env->flags |= LBF_NEED_BREAK; 8872 break; 8873 } 8874 8875 p = list_last_entry(tasks, struct task_struct, se.group_node); 8876 8877 if (!can_migrate_task(p, env)) 8878 goto next; 8879 8880 switch (env->migration_type) { 8881 case migrate_load: 8882 /* 8883 * Depending of the number of CPUs and tasks and the 8884 * cgroup hierarchy, task_h_load() can return a null 8885 * value. Make sure that env->imbalance decreases 8886 * otherwise detach_tasks() will stop only after 8887 * detaching up to loop_max tasks. 8888 */ 8889 load = max_t(unsigned long, task_h_load(p), 1); 8890 8891 if (sched_feat(LB_MIN) && 8892 load < 16 && !env->sd->nr_balance_failed) 8893 goto next; 8894 8895 /* 8896 * Make sure that we don't migrate too much load. 8897 * Nevertheless, let relax the constraint if 8898 * scheduler fails to find a good waiting task to 8899 * migrate. 8900 */ 8901 if (shr_bound(load, env->sd->nr_balance_failed) > env->imbalance) 8902 goto next; 8903 8904 env->imbalance -= load; 8905 break; 8906 8907 case migrate_util: 8908 util = task_util_est(p); 8909 8910 if (util > env->imbalance) 8911 goto next; 8912 8913 env->imbalance -= util; 8914 break; 8915 8916 case migrate_task: 8917 env->imbalance--; 8918 break; 8919 8920 case migrate_misfit: 8921 /* This is not a misfit task */ 8922 if (task_fits_cpu(p, env->src_cpu)) 8923 goto next; 8924 8925 env->imbalance = 0; 8926 break; 8927 } 8928 8929 detach_task(p, env); 8930 list_add(&p->se.group_node, &env->tasks); 8931 8932 detached++; 8933 8934 #ifdef CONFIG_PREEMPTION 8935 /* 8936 * NEWIDLE balancing is a source of latency, so preemptible 8937 * kernels will stop after the first task is detached to minimize 8938 * the critical section. 8939 */ 8940 if (env->idle == CPU_NEWLY_IDLE) 8941 break; 8942 #endif 8943 8944 /* 8945 * We only want to steal up to the prescribed amount of 8946 * load/util/tasks. 8947 */ 8948 if (env->imbalance <= 0) 8949 break; 8950 8951 continue; 8952 next: 8953 list_move(&p->se.group_node, tasks); 8954 } 8955 8956 /* 8957 * Right now, this is one of only two places we collect this stat 8958 * so we can safely collect detach_one_task() stats here rather 8959 * than inside detach_one_task(). 8960 */ 8961 schedstat_add(env->sd->lb_gained[env->idle], detached); 8962 8963 return detached; 8964 } 8965 8966 /* 8967 * attach_task() -- attach the task detached by detach_task() to its new rq. 8968 */ 8969 static void attach_task(struct rq *rq, struct task_struct *p) 8970 { 8971 lockdep_assert_rq_held(rq); 8972 8973 WARN_ON_ONCE(task_rq(p) != rq); 8974 activate_task(rq, p, ENQUEUE_NOCLOCK); 8975 check_preempt_curr(rq, p, 0); 8976 } 8977 8978 /* 8979 * attach_one_task() -- attaches the task returned from detach_one_task() to 8980 * its new rq. 8981 */ 8982 static void attach_one_task(struct rq *rq, struct task_struct *p) 8983 { 8984 struct rq_flags rf; 8985 8986 rq_lock(rq, &rf); 8987 update_rq_clock(rq); 8988 attach_task(rq, p); 8989 rq_unlock(rq, &rf); 8990 } 8991 8992 /* 8993 * attach_tasks() -- attaches all tasks detached by detach_tasks() to their 8994 * new rq. 8995 */ 8996 static void attach_tasks(struct lb_env *env) 8997 { 8998 struct list_head *tasks = &env->tasks; 8999 struct task_struct *p; 9000 struct rq_flags rf; 9001 9002 rq_lock(env->dst_rq, &rf); 9003 update_rq_clock(env->dst_rq); 9004 9005 while (!list_empty(tasks)) { 9006 p = list_first_entry(tasks, struct task_struct, se.group_node); 9007 list_del_init(&p->se.group_node); 9008 9009 attach_task(env->dst_rq, p); 9010 } 9011 9012 rq_unlock(env->dst_rq, &rf); 9013 } 9014 9015 #ifdef CONFIG_NO_HZ_COMMON 9016 static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq) 9017 { 9018 if (cfs_rq->avg.load_avg) 9019 return true; 9020 9021 if (cfs_rq->avg.util_avg) 9022 return true; 9023 9024 return false; 9025 } 9026 9027 static inline bool others_have_blocked(struct rq *rq) 9028 { 9029 if (READ_ONCE(rq->avg_rt.util_avg)) 9030 return true; 9031 9032 if (READ_ONCE(rq->avg_dl.util_avg)) 9033 return true; 9034 9035 if (thermal_load_avg(rq)) 9036 return true; 9037 9038 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ 9039 if (READ_ONCE(rq->avg_irq.util_avg)) 9040 return true; 9041 #endif 9042 9043 return false; 9044 } 9045 9046 static inline void update_blocked_load_tick(struct rq *rq) 9047 { 9048 WRITE_ONCE(rq->last_blocked_load_update_tick, jiffies); 9049 } 9050 9051 static inline void update_blocked_load_status(struct rq *rq, bool has_blocked) 9052 { 9053 if (!has_blocked) 9054 rq->has_blocked_load = 0; 9055 } 9056 #else 9057 static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq) { return false; } 9058 static inline bool others_have_blocked(struct rq *rq) { return false; } 9059 static inline void update_blocked_load_tick(struct rq *rq) {} 9060 static inline void update_blocked_load_status(struct rq *rq, bool has_blocked) {} 9061 #endif 9062 9063 static bool __update_blocked_others(struct rq *rq, bool *done) 9064 { 9065 const struct sched_class *curr_class; 9066 u64 now = rq_clock_pelt(rq); 9067 unsigned long thermal_pressure; 9068 bool decayed; 9069 9070 /* 9071 * update_load_avg() can call cpufreq_update_util(). Make sure that RT, 9072 * DL and IRQ signals have been updated before updating CFS. 9073 */ 9074 curr_class = rq->curr->sched_class; 9075 9076 thermal_pressure = arch_scale_thermal_pressure(cpu_of(rq)); 9077 9078 decayed = update_rt_rq_load_avg(now, rq, curr_class == &rt_sched_class) | 9079 update_dl_rq_load_avg(now, rq, curr_class == &dl_sched_class) | 9080 update_thermal_load_avg(rq_clock_thermal(rq), rq, thermal_pressure) | 9081 update_irq_load_avg(rq, 0); 9082 9083 if (others_have_blocked(rq)) 9084 *done = false; 9085 9086 return decayed; 9087 } 9088 9089 #ifdef CONFIG_FAIR_GROUP_SCHED 9090 9091 static bool __update_blocked_fair(struct rq *rq, bool *done) 9092 { 9093 struct cfs_rq *cfs_rq, *pos; 9094 bool decayed = false; 9095 int cpu = cpu_of(rq); 9096 9097 /* 9098 * Iterates the task_group tree in a bottom up fashion, see 9099 * list_add_leaf_cfs_rq() for details. 9100 */ 9101 for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) { 9102 struct sched_entity *se; 9103 9104 if (update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq)) { 9105 update_tg_load_avg(cfs_rq); 9106 9107 if (cfs_rq->nr_running == 0) 9108 update_idle_cfs_rq_clock_pelt(cfs_rq); 9109 9110 if (cfs_rq == &rq->cfs) 9111 decayed = true; 9112 } 9113 9114 /* Propagate pending load changes to the parent, if any: */ 9115 se = cfs_rq->tg->se[cpu]; 9116 if (se && !skip_blocked_update(se)) 9117 update_load_avg(cfs_rq_of(se), se, UPDATE_TG); 9118 9119 /* 9120 * There can be a lot of idle CPU cgroups. Don't let fully 9121 * decayed cfs_rqs linger on the list. 9122 */ 9123 if (cfs_rq_is_decayed(cfs_rq)) 9124 list_del_leaf_cfs_rq(cfs_rq); 9125 9126 /* Don't need periodic decay once load/util_avg are null */ 9127 if (cfs_rq_has_blocked(cfs_rq)) 9128 *done = false; 9129 } 9130 9131 return decayed; 9132 } 9133 9134 /* 9135 * Compute the hierarchical load factor for cfs_rq and all its ascendants. 9136 * This needs to be done in a top-down fashion because the load of a child 9137 * group is a fraction of its parents load. 9138 */ 9139 static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq) 9140 { 9141 struct rq *rq = rq_of(cfs_rq); 9142 struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)]; 9143 unsigned long now = jiffies; 9144 unsigned long load; 9145 9146 if (cfs_rq->last_h_load_update == now) 9147 return; 9148 9149 WRITE_ONCE(cfs_rq->h_load_next, NULL); 9150 for_each_sched_entity(se) { 9151 cfs_rq = cfs_rq_of(se); 9152 WRITE_ONCE(cfs_rq->h_load_next, se); 9153 if (cfs_rq->last_h_load_update == now) 9154 break; 9155 } 9156 9157 if (!se) { 9158 cfs_rq->h_load = cfs_rq_load_avg(cfs_rq); 9159 cfs_rq->last_h_load_update = now; 9160 } 9161 9162 while ((se = READ_ONCE(cfs_rq->h_load_next)) != NULL) { 9163 load = cfs_rq->h_load; 9164 load = div64_ul(load * se->avg.load_avg, 9165 cfs_rq_load_avg(cfs_rq) + 1); 9166 cfs_rq = group_cfs_rq(se); 9167 cfs_rq->h_load = load; 9168 cfs_rq->last_h_load_update = now; 9169 } 9170 } 9171 9172 static unsigned long task_h_load(struct task_struct *p) 9173 { 9174 struct cfs_rq *cfs_rq = task_cfs_rq(p); 9175 9176 update_cfs_rq_h_load(cfs_rq); 9177 return div64_ul(p->se.avg.load_avg * cfs_rq->h_load, 9178 cfs_rq_load_avg(cfs_rq) + 1); 9179 } 9180 #else 9181 static bool __update_blocked_fair(struct rq *rq, bool *done) 9182 { 9183 struct cfs_rq *cfs_rq = &rq->cfs; 9184 bool decayed; 9185 9186 decayed = update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq); 9187 if (cfs_rq_has_blocked(cfs_rq)) 9188 *done = false; 9189 9190 return decayed; 9191 } 9192 9193 static unsigned long task_h_load(struct task_struct *p) 9194 { 9195 return p->se.avg.load_avg; 9196 } 9197 #endif 9198 9199 static void update_blocked_averages(int cpu) 9200 { 9201 bool decayed = false, done = true; 9202 struct rq *rq = cpu_rq(cpu); 9203 struct rq_flags rf; 9204 9205 rq_lock_irqsave(rq, &rf); 9206 update_blocked_load_tick(rq); 9207 update_rq_clock(rq); 9208 9209 decayed |= __update_blocked_others(rq, &done); 9210 decayed |= __update_blocked_fair(rq, &done); 9211 9212 update_blocked_load_status(rq, !done); 9213 if (decayed) 9214 cpufreq_update_util(rq, 0); 9215 rq_unlock_irqrestore(rq, &rf); 9216 } 9217 9218 /********** Helpers for find_busiest_group ************************/ 9219 9220 /* 9221 * sg_lb_stats - stats of a sched_group required for load_balancing 9222 */ 9223 struct sg_lb_stats { 9224 unsigned long avg_load; /*Avg load across the CPUs of the group */ 9225 unsigned long group_load; /* Total load over the CPUs of the group */ 9226 unsigned long group_capacity; 9227 unsigned long group_util; /* Total utilization over the CPUs of the group */ 9228 unsigned long group_runnable; /* Total runnable time over the CPUs of the group */ 9229 unsigned int sum_nr_running; /* Nr of tasks running in the group */ 9230 unsigned int sum_h_nr_running; /* Nr of CFS tasks running in the group */ 9231 unsigned int idle_cpus; 9232 unsigned int group_weight; 9233 enum group_type group_type; 9234 unsigned int group_asym_packing; /* Tasks should be moved to preferred CPU */ 9235 unsigned int group_smt_balance; /* Task on busy SMT be moved */ 9236 unsigned long group_misfit_task_load; /* A CPU has a task too big for its capacity */ 9237 #ifdef CONFIG_NUMA_BALANCING 9238 unsigned int nr_numa_running; 9239 unsigned int nr_preferred_running; 9240 #endif 9241 }; 9242 9243 /* 9244 * sd_lb_stats - Structure to store the statistics of a sched_domain 9245 * during load balancing. 9246 */ 9247 struct sd_lb_stats { 9248 struct sched_group *busiest; /* Busiest group in this sd */ 9249 struct sched_group *local; /* Local group in this sd */ 9250 unsigned long total_load; /* Total load of all groups in sd */ 9251 unsigned long total_capacity; /* Total capacity of all groups in sd */ 9252 unsigned long avg_load; /* Average load across all groups in sd */ 9253 unsigned int prefer_sibling; /* tasks should go to sibling first */ 9254 9255 struct sg_lb_stats busiest_stat;/* Statistics of the busiest group */ 9256 struct sg_lb_stats local_stat; /* Statistics of the local group */ 9257 }; 9258 9259 static inline void init_sd_lb_stats(struct sd_lb_stats *sds) 9260 { 9261 /* 9262 * Skimp on the clearing to avoid duplicate work. We can avoid clearing 9263 * local_stat because update_sg_lb_stats() does a full clear/assignment. 9264 * We must however set busiest_stat::group_type and 9265 * busiest_stat::idle_cpus to the worst busiest group because 9266 * update_sd_pick_busiest() reads these before assignment. 9267 */ 9268 *sds = (struct sd_lb_stats){ 9269 .busiest = NULL, 9270 .local = NULL, 9271 .total_load = 0UL, 9272 .total_capacity = 0UL, 9273 .busiest_stat = { 9274 .idle_cpus = UINT_MAX, 9275 .group_type = group_has_spare, 9276 }, 9277 }; 9278 } 9279 9280 static unsigned long scale_rt_capacity(int cpu) 9281 { 9282 struct rq *rq = cpu_rq(cpu); 9283 unsigned long max = arch_scale_cpu_capacity(cpu); 9284 unsigned long used, free; 9285 unsigned long irq; 9286 9287 irq = cpu_util_irq(rq); 9288 9289 if (unlikely(irq >= max)) 9290 return 1; 9291 9292 /* 9293 * avg_rt.util_avg and avg_dl.util_avg track binary signals 9294 * (running and not running) with weights 0 and 1024 respectively. 9295 * avg_thermal.load_avg tracks thermal pressure and the weighted 9296 * average uses the actual delta max capacity(load). 9297 */ 9298 used = READ_ONCE(rq->avg_rt.util_avg); 9299 used += READ_ONCE(rq->avg_dl.util_avg); 9300 used += thermal_load_avg(rq); 9301 9302 if (unlikely(used >= max)) 9303 return 1; 9304 9305 free = max - used; 9306 9307 return scale_irq_capacity(free, irq, max); 9308 } 9309 9310 static void update_cpu_capacity(struct sched_domain *sd, int cpu) 9311 { 9312 unsigned long capacity = scale_rt_capacity(cpu); 9313 struct sched_group *sdg = sd->groups; 9314 9315 cpu_rq(cpu)->cpu_capacity_orig = arch_scale_cpu_capacity(cpu); 9316 9317 if (!capacity) 9318 capacity = 1; 9319 9320 cpu_rq(cpu)->cpu_capacity = capacity; 9321 trace_sched_cpu_capacity_tp(cpu_rq(cpu)); 9322 9323 sdg->sgc->capacity = capacity; 9324 sdg->sgc->min_capacity = capacity; 9325 sdg->sgc->max_capacity = capacity; 9326 } 9327 9328 void update_group_capacity(struct sched_domain *sd, int cpu) 9329 { 9330 struct sched_domain *child = sd->child; 9331 struct sched_group *group, *sdg = sd->groups; 9332 unsigned long capacity, min_capacity, max_capacity; 9333 unsigned long interval; 9334 9335 interval = msecs_to_jiffies(sd->balance_interval); 9336 interval = clamp(interval, 1UL, max_load_balance_interval); 9337 sdg->sgc->next_update = jiffies + interval; 9338 9339 if (!child) { 9340 update_cpu_capacity(sd, cpu); 9341 return; 9342 } 9343 9344 capacity = 0; 9345 min_capacity = ULONG_MAX; 9346 max_capacity = 0; 9347 9348 if (child->flags & SD_OVERLAP) { 9349 /* 9350 * SD_OVERLAP domains cannot assume that child groups 9351 * span the current group. 9352 */ 9353 9354 for_each_cpu(cpu, sched_group_span(sdg)) { 9355 unsigned long cpu_cap = capacity_of(cpu); 9356 9357 capacity += cpu_cap; 9358 min_capacity = min(cpu_cap, min_capacity); 9359 max_capacity = max(cpu_cap, max_capacity); 9360 } 9361 } else { 9362 /* 9363 * !SD_OVERLAP domains can assume that child groups 9364 * span the current group. 9365 */ 9366 9367 group = child->groups; 9368 do { 9369 struct sched_group_capacity *sgc = group->sgc; 9370 9371 capacity += sgc->capacity; 9372 min_capacity = min(sgc->min_capacity, min_capacity); 9373 max_capacity = max(sgc->max_capacity, max_capacity); 9374 group = group->next; 9375 } while (group != child->groups); 9376 } 9377 9378 sdg->sgc->capacity = capacity; 9379 sdg->sgc->min_capacity = min_capacity; 9380 sdg->sgc->max_capacity = max_capacity; 9381 } 9382 9383 /* 9384 * Check whether the capacity of the rq has been noticeably reduced by side 9385 * activity. The imbalance_pct is used for the threshold. 9386 * Return true is the capacity is reduced 9387 */ 9388 static inline int 9389 check_cpu_capacity(struct rq *rq, struct sched_domain *sd) 9390 { 9391 return ((rq->cpu_capacity * sd->imbalance_pct) < 9392 (rq->cpu_capacity_orig * 100)); 9393 } 9394 9395 /* 9396 * Check whether a rq has a misfit task and if it looks like we can actually 9397 * help that task: we can migrate the task to a CPU of higher capacity, or 9398 * the task's current CPU is heavily pressured. 9399 */ 9400 static inline int check_misfit_status(struct rq *rq, struct sched_domain *sd) 9401 { 9402 return rq->misfit_task_load && 9403 (rq->cpu_capacity_orig < rq->rd->max_cpu_capacity || 9404 check_cpu_capacity(rq, sd)); 9405 } 9406 9407 /* 9408 * Group imbalance indicates (and tries to solve) the problem where balancing 9409 * groups is inadequate due to ->cpus_ptr constraints. 9410 * 9411 * Imagine a situation of two groups of 4 CPUs each and 4 tasks each with a 9412 * cpumask covering 1 CPU of the first group and 3 CPUs of the second group. 9413 * Something like: 9414 * 9415 * { 0 1 2 3 } { 4 5 6 7 } 9416 * * * * * 9417 * 9418 * If we were to balance group-wise we'd place two tasks in the first group and 9419 * two tasks in the second group. Clearly this is undesired as it will overload 9420 * cpu 3 and leave one of the CPUs in the second group unused. 9421 * 9422 * The current solution to this issue is detecting the skew in the first group 9423 * by noticing the lower domain failed to reach balance and had difficulty 9424 * moving tasks due to affinity constraints. 9425 * 9426 * When this is so detected; this group becomes a candidate for busiest; see 9427 * update_sd_pick_busiest(). And calculate_imbalance() and 9428 * find_busiest_group() avoid some of the usual balance conditions to allow it 9429 * to create an effective group imbalance. 9430 * 9431 * This is a somewhat tricky proposition since the next run might not find the 9432 * group imbalance and decide the groups need to be balanced again. A most 9433 * subtle and fragile situation. 9434 */ 9435 9436 static inline int sg_imbalanced(struct sched_group *group) 9437 { 9438 return group->sgc->imbalance; 9439 } 9440 9441 /* 9442 * group_has_capacity returns true if the group has spare capacity that could 9443 * be used by some tasks. 9444 * We consider that a group has spare capacity if the number of task is 9445 * smaller than the number of CPUs or if the utilization is lower than the 9446 * available capacity for CFS tasks. 9447 * For the latter, we use a threshold to stabilize the state, to take into 9448 * account the variance of the tasks' load and to return true if the available 9449 * capacity in meaningful for the load balancer. 9450 * As an example, an available capacity of 1% can appear but it doesn't make 9451 * any benefit for the load balance. 9452 */ 9453 static inline bool 9454 group_has_capacity(unsigned int imbalance_pct, struct sg_lb_stats *sgs) 9455 { 9456 if (sgs->sum_nr_running < sgs->group_weight) 9457 return true; 9458 9459 if ((sgs->group_capacity * imbalance_pct) < 9460 (sgs->group_runnable * 100)) 9461 return false; 9462 9463 if ((sgs->group_capacity * 100) > 9464 (sgs->group_util * imbalance_pct)) 9465 return true; 9466 9467 return false; 9468 } 9469 9470 /* 9471 * group_is_overloaded returns true if the group has more tasks than it can 9472 * handle. 9473 * group_is_overloaded is not equals to !group_has_capacity because a group 9474 * with the exact right number of tasks, has no more spare capacity but is not 9475 * overloaded so both group_has_capacity and group_is_overloaded return 9476 * false. 9477 */ 9478 static inline bool 9479 group_is_overloaded(unsigned int imbalance_pct, struct sg_lb_stats *sgs) 9480 { 9481 if (sgs->sum_nr_running <= sgs->group_weight) 9482 return false; 9483 9484 if ((sgs->group_capacity * 100) < 9485 (sgs->group_util * imbalance_pct)) 9486 return true; 9487 9488 if ((sgs->group_capacity * imbalance_pct) < 9489 (sgs->group_runnable * 100)) 9490 return true; 9491 9492 return false; 9493 } 9494 9495 static inline enum 9496 group_type group_classify(unsigned int imbalance_pct, 9497 struct sched_group *group, 9498 struct sg_lb_stats *sgs) 9499 { 9500 if (group_is_overloaded(imbalance_pct, sgs)) 9501 return group_overloaded; 9502 9503 if (sg_imbalanced(group)) 9504 return group_imbalanced; 9505 9506 if (sgs->group_asym_packing) 9507 return group_asym_packing; 9508 9509 if (sgs->group_smt_balance) 9510 return group_smt_balance; 9511 9512 if (sgs->group_misfit_task_load) 9513 return group_misfit_task; 9514 9515 if (!group_has_capacity(imbalance_pct, sgs)) 9516 return group_fully_busy; 9517 9518 return group_has_spare; 9519 } 9520 9521 /** 9522 * sched_use_asym_prio - Check whether asym_packing priority must be used 9523 * @sd: The scheduling domain of the load balancing 9524 * @cpu: A CPU 9525 * 9526 * Always use CPU priority when balancing load between SMT siblings. When 9527 * balancing load between cores, it is not sufficient that @cpu is idle. Only 9528 * use CPU priority if the whole core is idle. 9529 * 9530 * Returns: True if the priority of @cpu must be followed. False otherwise. 9531 */ 9532 static bool sched_use_asym_prio(struct sched_domain *sd, int cpu) 9533 { 9534 if (!sched_smt_active()) 9535 return true; 9536 9537 return sd->flags & SD_SHARE_CPUCAPACITY || is_core_idle(cpu); 9538 } 9539 9540 /** 9541 * sched_asym - Check if the destination CPU can do asym_packing load balance 9542 * @env: The load balancing environment 9543 * @sds: Load-balancing data with statistics of the local group 9544 * @sgs: Load-balancing statistics of the candidate busiest group 9545 * @group: The candidate busiest group 9546 * 9547 * @env::dst_cpu can do asym_packing if it has higher priority than the 9548 * preferred CPU of @group. 9549 * 9550 * SMT is a special case. If we are balancing load between cores, @env::dst_cpu 9551 * can do asym_packing balance only if all its SMT siblings are idle. Also, it 9552 * can only do it if @group is an SMT group and has exactly on busy CPU. Larger 9553 * imbalances in the number of CPUS are dealt with in find_busiest_group(). 9554 * 9555 * If we are balancing load within an SMT core, or at DIE domain level, always 9556 * proceed. 9557 * 9558 * Return: true if @env::dst_cpu can do with asym_packing load balance. False 9559 * otherwise. 9560 */ 9561 static inline bool 9562 sched_asym(struct lb_env *env, struct sd_lb_stats *sds, struct sg_lb_stats *sgs, 9563 struct sched_group *group) 9564 { 9565 /* Ensure that the whole local core is idle, if applicable. */ 9566 if (!sched_use_asym_prio(env->sd, env->dst_cpu)) 9567 return false; 9568 9569 /* 9570 * CPU priorities does not make sense for SMT cores with more than one 9571 * busy sibling. 9572 */ 9573 if (group->flags & SD_SHARE_CPUCAPACITY) { 9574 if (sgs->group_weight - sgs->idle_cpus != 1) 9575 return false; 9576 } 9577 9578 return sched_asym_prefer(env->dst_cpu, group->asym_prefer_cpu); 9579 } 9580 9581 /* One group has more than one SMT CPU while the other group does not */ 9582 static inline bool smt_vs_nonsmt_groups(struct sched_group *sg1, 9583 struct sched_group *sg2) 9584 { 9585 if (!sg1 || !sg2) 9586 return false; 9587 9588 return (sg1->flags & SD_SHARE_CPUCAPACITY) != 9589 (sg2->flags & SD_SHARE_CPUCAPACITY); 9590 } 9591 9592 static inline bool smt_balance(struct lb_env *env, struct sg_lb_stats *sgs, 9593 struct sched_group *group) 9594 { 9595 if (env->idle == CPU_NOT_IDLE) 9596 return false; 9597 9598 /* 9599 * For SMT source group, it is better to move a task 9600 * to a CPU that doesn't have multiple tasks sharing its CPU capacity. 9601 * Note that if a group has a single SMT, SD_SHARE_CPUCAPACITY 9602 * will not be on. 9603 */ 9604 if (group->flags & SD_SHARE_CPUCAPACITY && 9605 sgs->sum_h_nr_running > 1) 9606 return true; 9607 9608 return false; 9609 } 9610 9611 static inline long sibling_imbalance(struct lb_env *env, 9612 struct sd_lb_stats *sds, 9613 struct sg_lb_stats *busiest, 9614 struct sg_lb_stats *local) 9615 { 9616 int ncores_busiest, ncores_local; 9617 long imbalance; 9618 9619 if (env->idle == CPU_NOT_IDLE || !busiest->sum_nr_running) 9620 return 0; 9621 9622 ncores_busiest = sds->busiest->cores; 9623 ncores_local = sds->local->cores; 9624 9625 if (ncores_busiest == ncores_local) { 9626 imbalance = busiest->sum_nr_running; 9627 lsub_positive(&imbalance, local->sum_nr_running); 9628 return imbalance; 9629 } 9630 9631 /* Balance such that nr_running/ncores ratio are same on both groups */ 9632 imbalance = ncores_local * busiest->sum_nr_running; 9633 lsub_positive(&imbalance, ncores_busiest * local->sum_nr_running); 9634 /* Normalize imbalance and do rounding on normalization */ 9635 imbalance = 2 * imbalance + ncores_local + ncores_busiest; 9636 imbalance /= ncores_local + ncores_busiest; 9637 9638 /* Take advantage of resource in an empty sched group */ 9639 if (imbalance <= 1 && local->sum_nr_running == 0 && 9640 busiest->sum_nr_running > 1) 9641 imbalance = 2; 9642 9643 return imbalance; 9644 } 9645 9646 static inline bool 9647 sched_reduced_capacity(struct rq *rq, struct sched_domain *sd) 9648 { 9649 /* 9650 * When there is more than 1 task, the group_overloaded case already 9651 * takes care of cpu with reduced capacity 9652 */ 9653 if (rq->cfs.h_nr_running != 1) 9654 return false; 9655 9656 return check_cpu_capacity(rq, sd); 9657 } 9658 9659 /** 9660 * update_sg_lb_stats - Update sched_group's statistics for load balancing. 9661 * @env: The load balancing environment. 9662 * @sds: Load-balancing data with statistics of the local group. 9663 * @group: sched_group whose statistics are to be updated. 9664 * @sgs: variable to hold the statistics for this group. 9665 * @sg_status: Holds flag indicating the status of the sched_group 9666 */ 9667 static inline void update_sg_lb_stats(struct lb_env *env, 9668 struct sd_lb_stats *sds, 9669 struct sched_group *group, 9670 struct sg_lb_stats *sgs, 9671 int *sg_status) 9672 { 9673 int i, nr_running, local_group; 9674 9675 memset(sgs, 0, sizeof(*sgs)); 9676 9677 local_group = group == sds->local; 9678 9679 for_each_cpu_and(i, sched_group_span(group), env->cpus) { 9680 struct rq *rq = cpu_rq(i); 9681 unsigned long load = cpu_load(rq); 9682 9683 sgs->group_load += load; 9684 sgs->group_util += cpu_util_cfs(i); 9685 sgs->group_runnable += cpu_runnable(rq); 9686 sgs->sum_h_nr_running += rq->cfs.h_nr_running; 9687 9688 nr_running = rq->nr_running; 9689 sgs->sum_nr_running += nr_running; 9690 9691 if (nr_running > 1) 9692 *sg_status |= SG_OVERLOAD; 9693 9694 if (cpu_overutilized(i)) 9695 *sg_status |= SG_OVERUTILIZED; 9696 9697 #ifdef CONFIG_NUMA_BALANCING 9698 sgs->nr_numa_running += rq->nr_numa_running; 9699 sgs->nr_preferred_running += rq->nr_preferred_running; 9700 #endif 9701 /* 9702 * No need to call idle_cpu() if nr_running is not 0 9703 */ 9704 if (!nr_running && idle_cpu(i)) { 9705 sgs->idle_cpus++; 9706 /* Idle cpu can't have misfit task */ 9707 continue; 9708 } 9709 9710 if (local_group) 9711 continue; 9712 9713 if (env->sd->flags & SD_ASYM_CPUCAPACITY) { 9714 /* Check for a misfit task on the cpu */ 9715 if (sgs->group_misfit_task_load < rq->misfit_task_load) { 9716 sgs->group_misfit_task_load = rq->misfit_task_load; 9717 *sg_status |= SG_OVERLOAD; 9718 } 9719 } else if ((env->idle != CPU_NOT_IDLE) && 9720 sched_reduced_capacity(rq, env->sd)) { 9721 /* Check for a task running on a CPU with reduced capacity */ 9722 if (sgs->group_misfit_task_load < load) 9723 sgs->group_misfit_task_load = load; 9724 } 9725 } 9726 9727 sgs->group_capacity = group->sgc->capacity; 9728 9729 sgs->group_weight = group->group_weight; 9730 9731 /* Check if dst CPU is idle and preferred to this group */ 9732 if (!local_group && env->sd->flags & SD_ASYM_PACKING && 9733 env->idle != CPU_NOT_IDLE && sgs->sum_h_nr_running && 9734 sched_asym(env, sds, sgs, group)) { 9735 sgs->group_asym_packing = 1; 9736 } 9737 9738 /* Check for loaded SMT group to be balanced to dst CPU */ 9739 if (!local_group && smt_balance(env, sgs, group)) 9740 sgs->group_smt_balance = 1; 9741 9742 sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs); 9743 9744 /* Computing avg_load makes sense only when group is overloaded */ 9745 if (sgs->group_type == group_overloaded) 9746 sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) / 9747 sgs->group_capacity; 9748 } 9749 9750 /** 9751 * update_sd_pick_busiest - return 1 on busiest group 9752 * @env: The load balancing environment. 9753 * @sds: sched_domain statistics 9754 * @sg: sched_group candidate to be checked for being the busiest 9755 * @sgs: sched_group statistics 9756 * 9757 * Determine if @sg is a busier group than the previously selected 9758 * busiest group. 9759 * 9760 * Return: %true if @sg is a busier group than the previously selected 9761 * busiest group. %false otherwise. 9762 */ 9763 static bool update_sd_pick_busiest(struct lb_env *env, 9764 struct sd_lb_stats *sds, 9765 struct sched_group *sg, 9766 struct sg_lb_stats *sgs) 9767 { 9768 struct sg_lb_stats *busiest = &sds->busiest_stat; 9769 9770 /* Make sure that there is at least one task to pull */ 9771 if (!sgs->sum_h_nr_running) 9772 return false; 9773 9774 /* 9775 * Don't try to pull misfit tasks we can't help. 9776 * We can use max_capacity here as reduction in capacity on some 9777 * CPUs in the group should either be possible to resolve 9778 * internally or be covered by avg_load imbalance (eventually). 9779 */ 9780 if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && 9781 (sgs->group_type == group_misfit_task) && 9782 (!capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) || 9783 sds->local_stat.group_type != group_has_spare)) 9784 return false; 9785 9786 if (sgs->group_type > busiest->group_type) 9787 return true; 9788 9789 if (sgs->group_type < busiest->group_type) 9790 return false; 9791 9792 /* 9793 * The candidate and the current busiest group are the same type of 9794 * group. Let check which one is the busiest according to the type. 9795 */ 9796 9797 switch (sgs->group_type) { 9798 case group_overloaded: 9799 /* Select the overloaded group with highest avg_load. */ 9800 if (sgs->avg_load <= busiest->avg_load) 9801 return false; 9802 break; 9803 9804 case group_imbalanced: 9805 /* 9806 * Select the 1st imbalanced group as we don't have any way to 9807 * choose one more than another. 9808 */ 9809 return false; 9810 9811 case group_asym_packing: 9812 /* Prefer to move from lowest priority CPU's work */ 9813 if (sched_asym_prefer(sg->asym_prefer_cpu, sds->busiest->asym_prefer_cpu)) 9814 return false; 9815 break; 9816 9817 case group_misfit_task: 9818 /* 9819 * If we have more than one misfit sg go with the biggest 9820 * misfit. 9821 */ 9822 if (sgs->group_misfit_task_load < busiest->group_misfit_task_load) 9823 return false; 9824 break; 9825 9826 case group_smt_balance: 9827 /* 9828 * Check if we have spare CPUs on either SMT group to 9829 * choose has spare or fully busy handling. 9830 */ 9831 if (sgs->idle_cpus != 0 || busiest->idle_cpus != 0) 9832 goto has_spare; 9833 9834 fallthrough; 9835 9836 case group_fully_busy: 9837 /* 9838 * Select the fully busy group with highest avg_load. In 9839 * theory, there is no need to pull task from such kind of 9840 * group because tasks have all compute capacity that they need 9841 * but we can still improve the overall throughput by reducing 9842 * contention when accessing shared HW resources. 9843 * 9844 * XXX for now avg_load is not computed and always 0 so we 9845 * select the 1st one, except if @sg is composed of SMT 9846 * siblings. 9847 */ 9848 9849 if (sgs->avg_load < busiest->avg_load) 9850 return false; 9851 9852 if (sgs->avg_load == busiest->avg_load) { 9853 /* 9854 * SMT sched groups need more help than non-SMT groups. 9855 * If @sg happens to also be SMT, either choice is good. 9856 */ 9857 if (sds->busiest->flags & SD_SHARE_CPUCAPACITY) 9858 return false; 9859 } 9860 9861 break; 9862 9863 case group_has_spare: 9864 /* 9865 * Do not pick sg with SMT CPUs over sg with pure CPUs, 9866 * as we do not want to pull task off SMT core with one task 9867 * and make the core idle. 9868 */ 9869 if (smt_vs_nonsmt_groups(sds->busiest, sg)) { 9870 if (sg->flags & SD_SHARE_CPUCAPACITY && sgs->sum_h_nr_running <= 1) 9871 return false; 9872 else 9873 return true; 9874 } 9875 has_spare: 9876 9877 /* 9878 * Select not overloaded group with lowest number of idle cpus 9879 * and highest number of running tasks. We could also compare 9880 * the spare capacity which is more stable but it can end up 9881 * that the group has less spare capacity but finally more idle 9882 * CPUs which means less opportunity to pull tasks. 9883 */ 9884 if (sgs->idle_cpus > busiest->idle_cpus) 9885 return false; 9886 else if ((sgs->idle_cpus == busiest->idle_cpus) && 9887 (sgs->sum_nr_running <= busiest->sum_nr_running)) 9888 return false; 9889 9890 break; 9891 } 9892 9893 /* 9894 * Candidate sg has no more than one task per CPU and has higher 9895 * per-CPU capacity. Migrating tasks to less capable CPUs may harm 9896 * throughput. Maximize throughput, power/energy consequences are not 9897 * considered. 9898 */ 9899 if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && 9900 (sgs->group_type <= group_fully_busy) && 9901 (capacity_greater(sg->sgc->min_capacity, capacity_of(env->dst_cpu)))) 9902 return false; 9903 9904 return true; 9905 } 9906 9907 #ifdef CONFIG_NUMA_BALANCING 9908 static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs) 9909 { 9910 if (sgs->sum_h_nr_running > sgs->nr_numa_running) 9911 return regular; 9912 if (sgs->sum_h_nr_running > sgs->nr_preferred_running) 9913 return remote; 9914 return all; 9915 } 9916 9917 static inline enum fbq_type fbq_classify_rq(struct rq *rq) 9918 { 9919 if (rq->nr_running > rq->nr_numa_running) 9920 return regular; 9921 if (rq->nr_running > rq->nr_preferred_running) 9922 return remote; 9923 return all; 9924 } 9925 #else 9926 static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs) 9927 { 9928 return all; 9929 } 9930 9931 static inline enum fbq_type fbq_classify_rq(struct rq *rq) 9932 { 9933 return regular; 9934 } 9935 #endif /* CONFIG_NUMA_BALANCING */ 9936 9937 9938 struct sg_lb_stats; 9939 9940 /* 9941 * task_running_on_cpu - return 1 if @p is running on @cpu. 9942 */ 9943 9944 static unsigned int task_running_on_cpu(int cpu, struct task_struct *p) 9945 { 9946 /* Task has no contribution or is new */ 9947 if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 9948 return 0; 9949 9950 if (task_on_rq_queued(p)) 9951 return 1; 9952 9953 return 0; 9954 } 9955 9956 /** 9957 * idle_cpu_without - would a given CPU be idle without p ? 9958 * @cpu: the processor on which idleness is tested. 9959 * @p: task which should be ignored. 9960 * 9961 * Return: 1 if the CPU would be idle. 0 otherwise. 9962 */ 9963 static int idle_cpu_without(int cpu, struct task_struct *p) 9964 { 9965 struct rq *rq = cpu_rq(cpu); 9966 9967 if (rq->curr != rq->idle && rq->curr != p) 9968 return 0; 9969 9970 /* 9971 * rq->nr_running can't be used but an updated version without the 9972 * impact of p on cpu must be used instead. The updated nr_running 9973 * be computed and tested before calling idle_cpu_without(). 9974 */ 9975 9976 #ifdef CONFIG_SMP 9977 if (rq->ttwu_pending) 9978 return 0; 9979 #endif 9980 9981 return 1; 9982 } 9983 9984 /* 9985 * update_sg_wakeup_stats - Update sched_group's statistics for wakeup. 9986 * @sd: The sched_domain level to look for idlest group. 9987 * @group: sched_group whose statistics are to be updated. 9988 * @sgs: variable to hold the statistics for this group. 9989 * @p: The task for which we look for the idlest group/CPU. 9990 */ 9991 static inline void update_sg_wakeup_stats(struct sched_domain *sd, 9992 struct sched_group *group, 9993 struct sg_lb_stats *sgs, 9994 struct task_struct *p) 9995 { 9996 int i, nr_running; 9997 9998 memset(sgs, 0, sizeof(*sgs)); 9999 10000 /* Assume that task can't fit any CPU of the group */ 10001 if (sd->flags & SD_ASYM_CPUCAPACITY) 10002 sgs->group_misfit_task_load = 1; 10003 10004 for_each_cpu(i, sched_group_span(group)) { 10005 struct rq *rq = cpu_rq(i); 10006 unsigned int local; 10007 10008 sgs->group_load += cpu_load_without(rq, p); 10009 sgs->group_util += cpu_util_without(i, p); 10010 sgs->group_runnable += cpu_runnable_without(rq, p); 10011 local = task_running_on_cpu(i, p); 10012 sgs->sum_h_nr_running += rq->cfs.h_nr_running - local; 10013 10014 nr_running = rq->nr_running - local; 10015 sgs->sum_nr_running += nr_running; 10016 10017 /* 10018 * No need to call idle_cpu_without() if nr_running is not 0 10019 */ 10020 if (!nr_running && idle_cpu_without(i, p)) 10021 sgs->idle_cpus++; 10022 10023 /* Check if task fits in the CPU */ 10024 if (sd->flags & SD_ASYM_CPUCAPACITY && 10025 sgs->group_misfit_task_load && 10026 task_fits_cpu(p, i)) 10027 sgs->group_misfit_task_load = 0; 10028 10029 } 10030 10031 sgs->group_capacity = group->sgc->capacity; 10032 10033 sgs->group_weight = group->group_weight; 10034 10035 sgs->group_type = group_classify(sd->imbalance_pct, group, sgs); 10036 10037 /* 10038 * Computing avg_load makes sense only when group is fully busy or 10039 * overloaded 10040 */ 10041 if (sgs->group_type == group_fully_busy || 10042 sgs->group_type == group_overloaded) 10043 sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) / 10044 sgs->group_capacity; 10045 } 10046 10047 static bool update_pick_idlest(struct sched_group *idlest, 10048 struct sg_lb_stats *idlest_sgs, 10049 struct sched_group *group, 10050 struct sg_lb_stats *sgs) 10051 { 10052 if (sgs->group_type < idlest_sgs->group_type) 10053 return true; 10054 10055 if (sgs->group_type > idlest_sgs->group_type) 10056 return false; 10057 10058 /* 10059 * The candidate and the current idlest group are the same type of 10060 * group. Let check which one is the idlest according to the type. 10061 */ 10062 10063 switch (sgs->group_type) { 10064 case group_overloaded: 10065 case group_fully_busy: 10066 /* Select the group with lowest avg_load. */ 10067 if (idlest_sgs->avg_load <= sgs->avg_load) 10068 return false; 10069 break; 10070 10071 case group_imbalanced: 10072 case group_asym_packing: 10073 case group_smt_balance: 10074 /* Those types are not used in the slow wakeup path */ 10075 return false; 10076 10077 case group_misfit_task: 10078 /* Select group with the highest max capacity */ 10079 if (idlest->sgc->max_capacity >= group->sgc->max_capacity) 10080 return false; 10081 break; 10082 10083 case group_has_spare: 10084 /* Select group with most idle CPUs */ 10085 if (idlest_sgs->idle_cpus > sgs->idle_cpus) 10086 return false; 10087 10088 /* Select group with lowest group_util */ 10089 if (idlest_sgs->idle_cpus == sgs->idle_cpus && 10090 idlest_sgs->group_util <= sgs->group_util) 10091 return false; 10092 10093 break; 10094 } 10095 10096 return true; 10097 } 10098 10099 /* 10100 * find_idlest_group() finds and returns the least busy CPU group within the 10101 * domain. 10102 * 10103 * Assumes p is allowed on at least one CPU in sd. 10104 */ 10105 static struct sched_group * 10106 find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu) 10107 { 10108 struct sched_group *idlest = NULL, *local = NULL, *group = sd->groups; 10109 struct sg_lb_stats local_sgs, tmp_sgs; 10110 struct sg_lb_stats *sgs; 10111 unsigned long imbalance; 10112 struct sg_lb_stats idlest_sgs = { 10113 .avg_load = UINT_MAX, 10114 .group_type = group_overloaded, 10115 }; 10116 10117 do { 10118 int local_group; 10119 10120 /* Skip over this group if it has no CPUs allowed */ 10121 if (!cpumask_intersects(sched_group_span(group), 10122 p->cpus_ptr)) 10123 continue; 10124 10125 /* Skip over this group if no cookie matched */ 10126 if (!sched_group_cookie_match(cpu_rq(this_cpu), p, group)) 10127 continue; 10128 10129 local_group = cpumask_test_cpu(this_cpu, 10130 sched_group_span(group)); 10131 10132 if (local_group) { 10133 sgs = &local_sgs; 10134 local = group; 10135 } else { 10136 sgs = &tmp_sgs; 10137 } 10138 10139 update_sg_wakeup_stats(sd, group, sgs, p); 10140 10141 if (!local_group && update_pick_idlest(idlest, &idlest_sgs, group, sgs)) { 10142 idlest = group; 10143 idlest_sgs = *sgs; 10144 } 10145 10146 } while (group = group->next, group != sd->groups); 10147 10148 10149 /* There is no idlest group to push tasks to */ 10150 if (!idlest) 10151 return NULL; 10152 10153 /* The local group has been skipped because of CPU affinity */ 10154 if (!local) 10155 return idlest; 10156 10157 /* 10158 * If the local group is idler than the selected idlest group 10159 * don't try and push the task. 10160 */ 10161 if (local_sgs.group_type < idlest_sgs.group_type) 10162 return NULL; 10163 10164 /* 10165 * If the local group is busier than the selected idlest group 10166 * try and push the task. 10167 */ 10168 if (local_sgs.group_type > idlest_sgs.group_type) 10169 return idlest; 10170 10171 switch (local_sgs.group_type) { 10172 case group_overloaded: 10173 case group_fully_busy: 10174 10175 /* Calculate allowed imbalance based on load */ 10176 imbalance = scale_load_down(NICE_0_LOAD) * 10177 (sd->imbalance_pct-100) / 100; 10178 10179 /* 10180 * When comparing groups across NUMA domains, it's possible for 10181 * the local domain to be very lightly loaded relative to the 10182 * remote domains but "imbalance" skews the comparison making 10183 * remote CPUs look much more favourable. When considering 10184 * cross-domain, add imbalance to the load on the remote node 10185 * and consider staying local. 10186 */ 10187 10188 if ((sd->flags & SD_NUMA) && 10189 ((idlest_sgs.avg_load + imbalance) >= local_sgs.avg_load)) 10190 return NULL; 10191 10192 /* 10193 * If the local group is less loaded than the selected 10194 * idlest group don't try and push any tasks. 10195 */ 10196 if (idlest_sgs.avg_load >= (local_sgs.avg_load + imbalance)) 10197 return NULL; 10198 10199 if (100 * local_sgs.avg_load <= sd->imbalance_pct * idlest_sgs.avg_load) 10200 return NULL; 10201 break; 10202 10203 case group_imbalanced: 10204 case group_asym_packing: 10205 case group_smt_balance: 10206 /* Those type are not used in the slow wakeup path */ 10207 return NULL; 10208 10209 case group_misfit_task: 10210 /* Select group with the highest max capacity */ 10211 if (local->sgc->max_capacity >= idlest->sgc->max_capacity) 10212 return NULL; 10213 break; 10214 10215 case group_has_spare: 10216 #ifdef CONFIG_NUMA 10217 if (sd->flags & SD_NUMA) { 10218 int imb_numa_nr = sd->imb_numa_nr; 10219 #ifdef CONFIG_NUMA_BALANCING 10220 int idlest_cpu; 10221 /* 10222 * If there is spare capacity at NUMA, try to select 10223 * the preferred node 10224 */ 10225 if (cpu_to_node(this_cpu) == p->numa_preferred_nid) 10226 return NULL; 10227 10228 idlest_cpu = cpumask_first(sched_group_span(idlest)); 10229 if (cpu_to_node(idlest_cpu) == p->numa_preferred_nid) 10230 return idlest; 10231 #endif /* CONFIG_NUMA_BALANCING */ 10232 /* 10233 * Otherwise, keep the task close to the wakeup source 10234 * and improve locality if the number of running tasks 10235 * would remain below threshold where an imbalance is 10236 * allowed while accounting for the possibility the 10237 * task is pinned to a subset of CPUs. If there is a 10238 * real need of migration, periodic load balance will 10239 * take care of it. 10240 */ 10241 if (p->nr_cpus_allowed != NR_CPUS) { 10242 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 10243 10244 cpumask_and(cpus, sched_group_span(local), p->cpus_ptr); 10245 imb_numa_nr = min(cpumask_weight(cpus), sd->imb_numa_nr); 10246 } 10247 10248 imbalance = abs(local_sgs.idle_cpus - idlest_sgs.idle_cpus); 10249 if (!adjust_numa_imbalance(imbalance, 10250 local_sgs.sum_nr_running + 1, 10251 imb_numa_nr)) { 10252 return NULL; 10253 } 10254 } 10255 #endif /* CONFIG_NUMA */ 10256 10257 /* 10258 * Select group with highest number of idle CPUs. We could also 10259 * compare the utilization which is more stable but it can end 10260 * up that the group has less spare capacity but finally more 10261 * idle CPUs which means more opportunity to run task. 10262 */ 10263 if (local_sgs.idle_cpus >= idlest_sgs.idle_cpus) 10264 return NULL; 10265 break; 10266 } 10267 10268 return idlest; 10269 } 10270 10271 static void update_idle_cpu_scan(struct lb_env *env, 10272 unsigned long sum_util) 10273 { 10274 struct sched_domain_shared *sd_share; 10275 int llc_weight, pct; 10276 u64 x, y, tmp; 10277 /* 10278 * Update the number of CPUs to scan in LLC domain, which could 10279 * be used as a hint in select_idle_cpu(). The update of sd_share 10280 * could be expensive because it is within a shared cache line. 10281 * So the write of this hint only occurs during periodic load 10282 * balancing, rather than CPU_NEWLY_IDLE, because the latter 10283 * can fire way more frequently than the former. 10284 */ 10285 if (!sched_feat(SIS_UTIL) || env->idle == CPU_NEWLY_IDLE) 10286 return; 10287 10288 llc_weight = per_cpu(sd_llc_size, env->dst_cpu); 10289 if (env->sd->span_weight != llc_weight) 10290 return; 10291 10292 sd_share = rcu_dereference(per_cpu(sd_llc_shared, env->dst_cpu)); 10293 if (!sd_share) 10294 return; 10295 10296 /* 10297 * The number of CPUs to search drops as sum_util increases, when 10298 * sum_util hits 85% or above, the scan stops. 10299 * The reason to choose 85% as the threshold is because this is the 10300 * imbalance_pct(117) when a LLC sched group is overloaded. 10301 * 10302 * let y = SCHED_CAPACITY_SCALE - p * x^2 [1] 10303 * and y'= y / SCHED_CAPACITY_SCALE 10304 * 10305 * x is the ratio of sum_util compared to the CPU capacity: 10306 * x = sum_util / (llc_weight * SCHED_CAPACITY_SCALE) 10307 * y' is the ratio of CPUs to be scanned in the LLC domain, 10308 * and the number of CPUs to scan is calculated by: 10309 * 10310 * nr_scan = llc_weight * y' [2] 10311 * 10312 * When x hits the threshold of overloaded, AKA, when 10313 * x = 100 / pct, y drops to 0. According to [1], 10314 * p should be SCHED_CAPACITY_SCALE * pct^2 / 10000 10315 * 10316 * Scale x by SCHED_CAPACITY_SCALE: 10317 * x' = sum_util / llc_weight; [3] 10318 * 10319 * and finally [1] becomes: 10320 * y = SCHED_CAPACITY_SCALE - 10321 * x'^2 * pct^2 / (10000 * SCHED_CAPACITY_SCALE) [4] 10322 * 10323 */ 10324 /* equation [3] */ 10325 x = sum_util; 10326 do_div(x, llc_weight); 10327 10328 /* equation [4] */ 10329 pct = env->sd->imbalance_pct; 10330 tmp = x * x * pct * pct; 10331 do_div(tmp, 10000 * SCHED_CAPACITY_SCALE); 10332 tmp = min_t(long, tmp, SCHED_CAPACITY_SCALE); 10333 y = SCHED_CAPACITY_SCALE - tmp; 10334 10335 /* equation [2] */ 10336 y *= llc_weight; 10337 do_div(y, SCHED_CAPACITY_SCALE); 10338 if ((int)y != sd_share->nr_idle_scan) 10339 WRITE_ONCE(sd_share->nr_idle_scan, (int)y); 10340 } 10341 10342 /** 10343 * update_sd_lb_stats - Update sched_domain's statistics for load balancing. 10344 * @env: The load balancing environment. 10345 * @sds: variable to hold the statistics for this sched_domain. 10346 */ 10347 10348 static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sds) 10349 { 10350 struct sched_group *sg = env->sd->groups; 10351 struct sg_lb_stats *local = &sds->local_stat; 10352 struct sg_lb_stats tmp_sgs; 10353 unsigned long sum_util = 0; 10354 int sg_status = 0; 10355 10356 do { 10357 struct sg_lb_stats *sgs = &tmp_sgs; 10358 int local_group; 10359 10360 local_group = cpumask_test_cpu(env->dst_cpu, sched_group_span(sg)); 10361 if (local_group) { 10362 sds->local = sg; 10363 sgs = local; 10364 10365 if (env->idle != CPU_NEWLY_IDLE || 10366 time_after_eq(jiffies, sg->sgc->next_update)) 10367 update_group_capacity(env->sd, env->dst_cpu); 10368 } 10369 10370 update_sg_lb_stats(env, sds, sg, sgs, &sg_status); 10371 10372 if (local_group) 10373 goto next_group; 10374 10375 10376 if (update_sd_pick_busiest(env, sds, sg, sgs)) { 10377 sds->busiest = sg; 10378 sds->busiest_stat = *sgs; 10379 } 10380 10381 next_group: 10382 /* Now, start updating sd_lb_stats */ 10383 sds->total_load += sgs->group_load; 10384 sds->total_capacity += sgs->group_capacity; 10385 10386 sum_util += sgs->group_util; 10387 sg = sg->next; 10388 } while (sg != env->sd->groups); 10389 10390 /* 10391 * Indicate that the child domain of the busiest group prefers tasks 10392 * go to a child's sibling domains first. NB the flags of a sched group 10393 * are those of the child domain. 10394 */ 10395 if (sds->busiest) 10396 sds->prefer_sibling = !!(sds->busiest->flags & SD_PREFER_SIBLING); 10397 10398 10399 if (env->sd->flags & SD_NUMA) 10400 env->fbq_type = fbq_classify_group(&sds->busiest_stat); 10401 10402 if (!env->sd->parent) { 10403 struct root_domain *rd = env->dst_rq->rd; 10404 10405 /* update overload indicator if we are at root domain */ 10406 WRITE_ONCE(rd->overload, sg_status & SG_OVERLOAD); 10407 10408 /* Update over-utilization (tipping point, U >= 0) indicator */ 10409 WRITE_ONCE(rd->overutilized, sg_status & SG_OVERUTILIZED); 10410 trace_sched_overutilized_tp(rd, sg_status & SG_OVERUTILIZED); 10411 } else if (sg_status & SG_OVERUTILIZED) { 10412 struct root_domain *rd = env->dst_rq->rd; 10413 10414 WRITE_ONCE(rd->overutilized, SG_OVERUTILIZED); 10415 trace_sched_overutilized_tp(rd, SG_OVERUTILIZED); 10416 } 10417 10418 update_idle_cpu_scan(env, sum_util); 10419 } 10420 10421 /** 10422 * calculate_imbalance - Calculate the amount of imbalance present within the 10423 * groups of a given sched_domain during load balance. 10424 * @env: load balance environment 10425 * @sds: statistics of the sched_domain whose imbalance is to be calculated. 10426 */ 10427 static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds) 10428 { 10429 struct sg_lb_stats *local, *busiest; 10430 10431 local = &sds->local_stat; 10432 busiest = &sds->busiest_stat; 10433 10434 if (busiest->group_type == group_misfit_task) { 10435 if (env->sd->flags & SD_ASYM_CPUCAPACITY) { 10436 /* Set imbalance to allow misfit tasks to be balanced. */ 10437 env->migration_type = migrate_misfit; 10438 env->imbalance = 1; 10439 } else { 10440 /* 10441 * Set load imbalance to allow moving task from cpu 10442 * with reduced capacity. 10443 */ 10444 env->migration_type = migrate_load; 10445 env->imbalance = busiest->group_misfit_task_load; 10446 } 10447 return; 10448 } 10449 10450 if (busiest->group_type == group_asym_packing) { 10451 /* 10452 * In case of asym capacity, we will try to migrate all load to 10453 * the preferred CPU. 10454 */ 10455 env->migration_type = migrate_task; 10456 env->imbalance = busiest->sum_h_nr_running; 10457 return; 10458 } 10459 10460 if (busiest->group_type == group_smt_balance) { 10461 /* Reduce number of tasks sharing CPU capacity */ 10462 env->migration_type = migrate_task; 10463 env->imbalance = 1; 10464 return; 10465 } 10466 10467 if (busiest->group_type == group_imbalanced) { 10468 /* 10469 * In the group_imb case we cannot rely on group-wide averages 10470 * to ensure CPU-load equilibrium, try to move any task to fix 10471 * the imbalance. The next load balance will take care of 10472 * balancing back the system. 10473 */ 10474 env->migration_type = migrate_task; 10475 env->imbalance = 1; 10476 return; 10477 } 10478 10479 /* 10480 * Try to use spare capacity of local group without overloading it or 10481 * emptying busiest. 10482 */ 10483 if (local->group_type == group_has_spare) { 10484 if ((busiest->group_type > group_fully_busy) && 10485 !(env->sd->flags & SD_SHARE_PKG_RESOURCES)) { 10486 /* 10487 * If busiest is overloaded, try to fill spare 10488 * capacity. This might end up creating spare capacity 10489 * in busiest or busiest still being overloaded but 10490 * there is no simple way to directly compute the 10491 * amount of load to migrate in order to balance the 10492 * system. 10493 */ 10494 env->migration_type = migrate_util; 10495 env->imbalance = max(local->group_capacity, local->group_util) - 10496 local->group_util; 10497 10498 /* 10499 * In some cases, the group's utilization is max or even 10500 * higher than capacity because of migrations but the 10501 * local CPU is (newly) idle. There is at least one 10502 * waiting task in this overloaded busiest group. Let's 10503 * try to pull it. 10504 */ 10505 if (env->idle != CPU_NOT_IDLE && env->imbalance == 0) { 10506 env->migration_type = migrate_task; 10507 env->imbalance = 1; 10508 } 10509 10510 return; 10511 } 10512 10513 if (busiest->group_weight == 1 || sds->prefer_sibling) { 10514 /* 10515 * When prefer sibling, evenly spread running tasks on 10516 * groups. 10517 */ 10518 env->migration_type = migrate_task; 10519 env->imbalance = sibling_imbalance(env, sds, busiest, local); 10520 } else { 10521 10522 /* 10523 * If there is no overload, we just want to even the number of 10524 * idle cpus. 10525 */ 10526 env->migration_type = migrate_task; 10527 env->imbalance = max_t(long, 0, 10528 (local->idle_cpus - busiest->idle_cpus)); 10529 } 10530 10531 #ifdef CONFIG_NUMA 10532 /* Consider allowing a small imbalance between NUMA groups */ 10533 if (env->sd->flags & SD_NUMA) { 10534 env->imbalance = adjust_numa_imbalance(env->imbalance, 10535 local->sum_nr_running + 1, 10536 env->sd->imb_numa_nr); 10537 } 10538 #endif 10539 10540 /* Number of tasks to move to restore balance */ 10541 env->imbalance >>= 1; 10542 10543 return; 10544 } 10545 10546 /* 10547 * Local is fully busy but has to take more load to relieve the 10548 * busiest group 10549 */ 10550 if (local->group_type < group_overloaded) { 10551 /* 10552 * Local will become overloaded so the avg_load metrics are 10553 * finally needed. 10554 */ 10555 10556 local->avg_load = (local->group_load * SCHED_CAPACITY_SCALE) / 10557 local->group_capacity; 10558 10559 /* 10560 * If the local group is more loaded than the selected 10561 * busiest group don't try to pull any tasks. 10562 */ 10563 if (local->avg_load >= busiest->avg_load) { 10564 env->imbalance = 0; 10565 return; 10566 } 10567 10568 sds->avg_load = (sds->total_load * SCHED_CAPACITY_SCALE) / 10569 sds->total_capacity; 10570 10571 /* 10572 * If the local group is more loaded than the average system 10573 * load, don't try to pull any tasks. 10574 */ 10575 if (local->avg_load >= sds->avg_load) { 10576 env->imbalance = 0; 10577 return; 10578 } 10579 10580 } 10581 10582 /* 10583 * Both group are or will become overloaded and we're trying to get all 10584 * the CPUs to the average_load, so we don't want to push ourselves 10585 * above the average load, nor do we wish to reduce the max loaded CPU 10586 * below the average load. At the same time, we also don't want to 10587 * reduce the group load below the group capacity. Thus we look for 10588 * the minimum possible imbalance. 10589 */ 10590 env->migration_type = migrate_load; 10591 env->imbalance = min( 10592 (busiest->avg_load - sds->avg_load) * busiest->group_capacity, 10593 (sds->avg_load - local->avg_load) * local->group_capacity 10594 ) / SCHED_CAPACITY_SCALE; 10595 } 10596 10597 /******* find_busiest_group() helpers end here *********************/ 10598 10599 /* 10600 * Decision matrix according to the local and busiest group type: 10601 * 10602 * busiest \ local has_spare fully_busy misfit asym imbalanced overloaded 10603 * has_spare nr_idle balanced N/A N/A balanced balanced 10604 * fully_busy nr_idle nr_idle N/A N/A balanced balanced 10605 * misfit_task force N/A N/A N/A N/A N/A 10606 * asym_packing force force N/A N/A force force 10607 * imbalanced force force N/A N/A force force 10608 * overloaded force force N/A N/A force avg_load 10609 * 10610 * N/A : Not Applicable because already filtered while updating 10611 * statistics. 10612 * balanced : The system is balanced for these 2 groups. 10613 * force : Calculate the imbalance as load migration is probably needed. 10614 * avg_load : Only if imbalance is significant enough. 10615 * nr_idle : dst_cpu is not busy and the number of idle CPUs is quite 10616 * different in groups. 10617 */ 10618 10619 /** 10620 * find_busiest_group - Returns the busiest group within the sched_domain 10621 * if there is an imbalance. 10622 * @env: The load balancing environment. 10623 * 10624 * Also calculates the amount of runnable load which should be moved 10625 * to restore balance. 10626 * 10627 * Return: - The busiest group if imbalance exists. 10628 */ 10629 static struct sched_group *find_busiest_group(struct lb_env *env) 10630 { 10631 struct sg_lb_stats *local, *busiest; 10632 struct sd_lb_stats sds; 10633 10634 init_sd_lb_stats(&sds); 10635 10636 /* 10637 * Compute the various statistics relevant for load balancing at 10638 * this level. 10639 */ 10640 update_sd_lb_stats(env, &sds); 10641 10642 /* There is no busy sibling group to pull tasks from */ 10643 if (!sds.busiest) 10644 goto out_balanced; 10645 10646 busiest = &sds.busiest_stat; 10647 10648 /* Misfit tasks should be dealt with regardless of the avg load */ 10649 if (busiest->group_type == group_misfit_task) 10650 goto force_balance; 10651 10652 if (sched_energy_enabled()) { 10653 struct root_domain *rd = env->dst_rq->rd; 10654 10655 if (rcu_dereference(rd->pd) && !READ_ONCE(rd->overutilized)) 10656 goto out_balanced; 10657 } 10658 10659 /* ASYM feature bypasses nice load balance check */ 10660 if (busiest->group_type == group_asym_packing) 10661 goto force_balance; 10662 10663 /* 10664 * If the busiest group is imbalanced the below checks don't 10665 * work because they assume all things are equal, which typically 10666 * isn't true due to cpus_ptr constraints and the like. 10667 */ 10668 if (busiest->group_type == group_imbalanced) 10669 goto force_balance; 10670 10671 local = &sds.local_stat; 10672 /* 10673 * If the local group is busier than the selected busiest group 10674 * don't try and pull any tasks. 10675 */ 10676 if (local->group_type > busiest->group_type) 10677 goto out_balanced; 10678 10679 /* 10680 * When groups are overloaded, use the avg_load to ensure fairness 10681 * between tasks. 10682 */ 10683 if (local->group_type == group_overloaded) { 10684 /* 10685 * If the local group is more loaded than the selected 10686 * busiest group don't try to pull any tasks. 10687 */ 10688 if (local->avg_load >= busiest->avg_load) 10689 goto out_balanced; 10690 10691 /* XXX broken for overlapping NUMA groups */ 10692 sds.avg_load = (sds.total_load * SCHED_CAPACITY_SCALE) / 10693 sds.total_capacity; 10694 10695 /* 10696 * Don't pull any tasks if this group is already above the 10697 * domain average load. 10698 */ 10699 if (local->avg_load >= sds.avg_load) 10700 goto out_balanced; 10701 10702 /* 10703 * If the busiest group is more loaded, use imbalance_pct to be 10704 * conservative. 10705 */ 10706 if (100 * busiest->avg_load <= 10707 env->sd->imbalance_pct * local->avg_load) 10708 goto out_balanced; 10709 } 10710 10711 /* 10712 * Try to move all excess tasks to a sibling domain of the busiest 10713 * group's child domain. 10714 */ 10715 if (sds.prefer_sibling && local->group_type == group_has_spare && 10716 sibling_imbalance(env, &sds, busiest, local) > 1) 10717 goto force_balance; 10718 10719 if (busiest->group_type != group_overloaded) { 10720 if (env->idle == CPU_NOT_IDLE) { 10721 /* 10722 * If the busiest group is not overloaded (and as a 10723 * result the local one too) but this CPU is already 10724 * busy, let another idle CPU try to pull task. 10725 */ 10726 goto out_balanced; 10727 } 10728 10729 if (busiest->group_type == group_smt_balance && 10730 smt_vs_nonsmt_groups(sds.local, sds.busiest)) { 10731 /* Let non SMT CPU pull from SMT CPU sharing with sibling */ 10732 goto force_balance; 10733 } 10734 10735 if (busiest->group_weight > 1 && 10736 local->idle_cpus <= (busiest->idle_cpus + 1)) { 10737 /* 10738 * If the busiest group is not overloaded 10739 * and there is no imbalance between this and busiest 10740 * group wrt idle CPUs, it is balanced. The imbalance 10741 * becomes significant if the diff is greater than 1 10742 * otherwise we might end up to just move the imbalance 10743 * on another group. Of course this applies only if 10744 * there is more than 1 CPU per group. 10745 */ 10746 goto out_balanced; 10747 } 10748 10749 if (busiest->sum_h_nr_running == 1) { 10750 /* 10751 * busiest doesn't have any tasks waiting to run 10752 */ 10753 goto out_balanced; 10754 } 10755 } 10756 10757 force_balance: 10758 /* Looks like there is an imbalance. Compute it */ 10759 calculate_imbalance(env, &sds); 10760 return env->imbalance ? sds.busiest : NULL; 10761 10762 out_balanced: 10763 env->imbalance = 0; 10764 return NULL; 10765 } 10766 10767 /* 10768 * find_busiest_queue - find the busiest runqueue among the CPUs in the group. 10769 */ 10770 static struct rq *find_busiest_queue(struct lb_env *env, 10771 struct sched_group *group) 10772 { 10773 struct rq *busiest = NULL, *rq; 10774 unsigned long busiest_util = 0, busiest_load = 0, busiest_capacity = 1; 10775 unsigned int busiest_nr = 0; 10776 int i; 10777 10778 for_each_cpu_and(i, sched_group_span(group), env->cpus) { 10779 unsigned long capacity, load, util; 10780 unsigned int nr_running; 10781 enum fbq_type rt; 10782 10783 rq = cpu_rq(i); 10784 rt = fbq_classify_rq(rq); 10785 10786 /* 10787 * We classify groups/runqueues into three groups: 10788 * - regular: there are !numa tasks 10789 * - remote: there are numa tasks that run on the 'wrong' node 10790 * - all: there is no distinction 10791 * 10792 * In order to avoid migrating ideally placed numa tasks, 10793 * ignore those when there's better options. 10794 * 10795 * If we ignore the actual busiest queue to migrate another 10796 * task, the next balance pass can still reduce the busiest 10797 * queue by moving tasks around inside the node. 10798 * 10799 * If we cannot move enough load due to this classification 10800 * the next pass will adjust the group classification and 10801 * allow migration of more tasks. 10802 * 10803 * Both cases only affect the total convergence complexity. 10804 */ 10805 if (rt > env->fbq_type) 10806 continue; 10807 10808 nr_running = rq->cfs.h_nr_running; 10809 if (!nr_running) 10810 continue; 10811 10812 capacity = capacity_of(i); 10813 10814 /* 10815 * For ASYM_CPUCAPACITY domains, don't pick a CPU that could 10816 * eventually lead to active_balancing high->low capacity. 10817 * Higher per-CPU capacity is considered better than balancing 10818 * average load. 10819 */ 10820 if (env->sd->flags & SD_ASYM_CPUCAPACITY && 10821 !capacity_greater(capacity_of(env->dst_cpu), capacity) && 10822 nr_running == 1) 10823 continue; 10824 10825 /* 10826 * Make sure we only pull tasks from a CPU of lower priority 10827 * when balancing between SMT siblings. 10828 * 10829 * If balancing between cores, let lower priority CPUs help 10830 * SMT cores with more than one busy sibling. 10831 */ 10832 if ((env->sd->flags & SD_ASYM_PACKING) && 10833 sched_use_asym_prio(env->sd, i) && 10834 sched_asym_prefer(i, env->dst_cpu) && 10835 nr_running == 1) 10836 continue; 10837 10838 switch (env->migration_type) { 10839 case migrate_load: 10840 /* 10841 * When comparing with load imbalance, use cpu_load() 10842 * which is not scaled with the CPU capacity. 10843 */ 10844 load = cpu_load(rq); 10845 10846 if (nr_running == 1 && load > env->imbalance && 10847 !check_cpu_capacity(rq, env->sd)) 10848 break; 10849 10850 /* 10851 * For the load comparisons with the other CPUs, 10852 * consider the cpu_load() scaled with the CPU 10853 * capacity, so that the load can be moved away 10854 * from the CPU that is potentially running at a 10855 * lower capacity. 10856 * 10857 * Thus we're looking for max(load_i / capacity_i), 10858 * crosswise multiplication to rid ourselves of the 10859 * division works out to: 10860 * load_i * capacity_j > load_j * capacity_i; 10861 * where j is our previous maximum. 10862 */ 10863 if (load * busiest_capacity > busiest_load * capacity) { 10864 busiest_load = load; 10865 busiest_capacity = capacity; 10866 busiest = rq; 10867 } 10868 break; 10869 10870 case migrate_util: 10871 util = cpu_util_cfs_boost(i); 10872 10873 /* 10874 * Don't try to pull utilization from a CPU with one 10875 * running task. Whatever its utilization, we will fail 10876 * detach the task. 10877 */ 10878 if (nr_running <= 1) 10879 continue; 10880 10881 if (busiest_util < util) { 10882 busiest_util = util; 10883 busiest = rq; 10884 } 10885 break; 10886 10887 case migrate_task: 10888 if (busiest_nr < nr_running) { 10889 busiest_nr = nr_running; 10890 busiest = rq; 10891 } 10892 break; 10893 10894 case migrate_misfit: 10895 /* 10896 * For ASYM_CPUCAPACITY domains with misfit tasks we 10897 * simply seek the "biggest" misfit task. 10898 */ 10899 if (rq->misfit_task_load > busiest_load) { 10900 busiest_load = rq->misfit_task_load; 10901 busiest = rq; 10902 } 10903 10904 break; 10905 10906 } 10907 } 10908 10909 return busiest; 10910 } 10911 10912 /* 10913 * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but 10914 * so long as it is large enough. 10915 */ 10916 #define MAX_PINNED_INTERVAL 512 10917 10918 static inline bool 10919 asym_active_balance(struct lb_env *env) 10920 { 10921 /* 10922 * ASYM_PACKING needs to force migrate tasks from busy but lower 10923 * priority CPUs in order to pack all tasks in the highest priority 10924 * CPUs. When done between cores, do it only if the whole core if the 10925 * whole core is idle. 10926 * 10927 * If @env::src_cpu is an SMT core with busy siblings, let 10928 * the lower priority @env::dst_cpu help it. Do not follow 10929 * CPU priority. 10930 */ 10931 return env->idle != CPU_NOT_IDLE && (env->sd->flags & SD_ASYM_PACKING) && 10932 sched_use_asym_prio(env->sd, env->dst_cpu) && 10933 (sched_asym_prefer(env->dst_cpu, env->src_cpu) || 10934 !sched_use_asym_prio(env->sd, env->src_cpu)); 10935 } 10936 10937 static inline bool 10938 imbalanced_active_balance(struct lb_env *env) 10939 { 10940 struct sched_domain *sd = env->sd; 10941 10942 /* 10943 * The imbalanced case includes the case of pinned tasks preventing a fair 10944 * distribution of the load on the system but also the even distribution of the 10945 * threads on a system with spare capacity 10946 */ 10947 if ((env->migration_type == migrate_task) && 10948 (sd->nr_balance_failed > sd->cache_nice_tries+2)) 10949 return 1; 10950 10951 return 0; 10952 } 10953 10954 static int need_active_balance(struct lb_env *env) 10955 { 10956 struct sched_domain *sd = env->sd; 10957 10958 if (asym_active_balance(env)) 10959 return 1; 10960 10961 if (imbalanced_active_balance(env)) 10962 return 1; 10963 10964 /* 10965 * The dst_cpu is idle and the src_cpu CPU has only 1 CFS task. 10966 * It's worth migrating the task if the src_cpu's capacity is reduced 10967 * because of other sched_class or IRQs if more capacity stays 10968 * available on dst_cpu. 10969 */ 10970 if ((env->idle != CPU_NOT_IDLE) && 10971 (env->src_rq->cfs.h_nr_running == 1)) { 10972 if ((check_cpu_capacity(env->src_rq, sd)) && 10973 (capacity_of(env->src_cpu)*sd->imbalance_pct < capacity_of(env->dst_cpu)*100)) 10974 return 1; 10975 } 10976 10977 if (env->migration_type == migrate_misfit) 10978 return 1; 10979 10980 return 0; 10981 } 10982 10983 static int active_load_balance_cpu_stop(void *data); 10984 10985 static int should_we_balance(struct lb_env *env) 10986 { 10987 struct cpumask *swb_cpus = this_cpu_cpumask_var_ptr(should_we_balance_tmpmask); 10988 struct sched_group *sg = env->sd->groups; 10989 int cpu, idle_smt = -1; 10990 10991 /* 10992 * Ensure the balancing environment is consistent; can happen 10993 * when the softirq triggers 'during' hotplug. 10994 */ 10995 if (!cpumask_test_cpu(env->dst_cpu, env->cpus)) 10996 return 0; 10997 10998 /* 10999 * In the newly idle case, we will allow all the CPUs 11000 * to do the newly idle load balance. 11001 * 11002 * However, we bail out if we already have tasks or a wakeup pending, 11003 * to optimize wakeup latency. 11004 */ 11005 if (env->idle == CPU_NEWLY_IDLE) { 11006 if (env->dst_rq->nr_running > 0 || env->dst_rq->ttwu_pending) 11007 return 0; 11008 return 1; 11009 } 11010 11011 cpumask_copy(swb_cpus, group_balance_mask(sg)); 11012 /* Try to find first idle CPU */ 11013 for_each_cpu_and(cpu, swb_cpus, env->cpus) { 11014 if (!idle_cpu(cpu)) 11015 continue; 11016 11017 /* 11018 * Don't balance to idle SMT in busy core right away when 11019 * balancing cores, but remember the first idle SMT CPU for 11020 * later consideration. Find CPU on an idle core first. 11021 */ 11022 if (!(env->sd->flags & SD_SHARE_CPUCAPACITY) && !is_core_idle(cpu)) { 11023 if (idle_smt == -1) 11024 idle_smt = cpu; 11025 /* 11026 * If the core is not idle, and first SMT sibling which is 11027 * idle has been found, then its not needed to check other 11028 * SMT siblings for idleness: 11029 */ 11030 #ifdef CONFIG_SCHED_SMT 11031 cpumask_andnot(swb_cpus, swb_cpus, cpu_smt_mask(cpu)); 11032 #endif 11033 continue; 11034 } 11035 11036 /* Are we the first idle CPU? */ 11037 return cpu == env->dst_cpu; 11038 } 11039 11040 if (idle_smt == env->dst_cpu) 11041 return true; 11042 11043 /* Are we the first CPU of this group ? */ 11044 return group_balance_cpu(sg) == env->dst_cpu; 11045 } 11046 11047 /* 11048 * Check this_cpu to ensure it is balanced within domain. Attempt to move 11049 * tasks if there is an imbalance. 11050 */ 11051 static int load_balance(int this_cpu, struct rq *this_rq, 11052 struct sched_domain *sd, enum cpu_idle_type idle, 11053 int *continue_balancing) 11054 { 11055 int ld_moved, cur_ld_moved, active_balance = 0; 11056 struct sched_domain *sd_parent = sd->parent; 11057 struct sched_group *group; 11058 struct rq *busiest; 11059 struct rq_flags rf; 11060 struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask); 11061 struct lb_env env = { 11062 .sd = sd, 11063 .dst_cpu = this_cpu, 11064 .dst_rq = this_rq, 11065 .dst_grpmask = group_balance_mask(sd->groups), 11066 .idle = idle, 11067 .loop_break = SCHED_NR_MIGRATE_BREAK, 11068 .cpus = cpus, 11069 .fbq_type = all, 11070 .tasks = LIST_HEAD_INIT(env.tasks), 11071 }; 11072 11073 cpumask_and(cpus, sched_domain_span(sd), cpu_active_mask); 11074 11075 schedstat_inc(sd->lb_count[idle]); 11076 11077 redo: 11078 if (!should_we_balance(&env)) { 11079 *continue_balancing = 0; 11080 goto out_balanced; 11081 } 11082 11083 group = find_busiest_group(&env); 11084 if (!group) { 11085 schedstat_inc(sd->lb_nobusyg[idle]); 11086 goto out_balanced; 11087 } 11088 11089 busiest = find_busiest_queue(&env, group); 11090 if (!busiest) { 11091 schedstat_inc(sd->lb_nobusyq[idle]); 11092 goto out_balanced; 11093 } 11094 11095 WARN_ON_ONCE(busiest == env.dst_rq); 11096 11097 schedstat_add(sd->lb_imbalance[idle], env.imbalance); 11098 11099 env.src_cpu = busiest->cpu; 11100 env.src_rq = busiest; 11101 11102 ld_moved = 0; 11103 /* Clear this flag as soon as we find a pullable task */ 11104 env.flags |= LBF_ALL_PINNED; 11105 if (busiest->nr_running > 1) { 11106 /* 11107 * Attempt to move tasks. If find_busiest_group has found 11108 * an imbalance but busiest->nr_running <= 1, the group is 11109 * still unbalanced. ld_moved simply stays zero, so it is 11110 * correctly treated as an imbalance. 11111 */ 11112 env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running); 11113 11114 more_balance: 11115 rq_lock_irqsave(busiest, &rf); 11116 update_rq_clock(busiest); 11117 11118 /* 11119 * cur_ld_moved - load moved in current iteration 11120 * ld_moved - cumulative load moved across iterations 11121 */ 11122 cur_ld_moved = detach_tasks(&env); 11123 11124 /* 11125 * We've detached some tasks from busiest_rq. Every 11126 * task is masked "TASK_ON_RQ_MIGRATING", so we can safely 11127 * unlock busiest->lock, and we are able to be sure 11128 * that nobody can manipulate the tasks in parallel. 11129 * See task_rq_lock() family for the details. 11130 */ 11131 11132 rq_unlock(busiest, &rf); 11133 11134 if (cur_ld_moved) { 11135 attach_tasks(&env); 11136 ld_moved += cur_ld_moved; 11137 } 11138 11139 local_irq_restore(rf.flags); 11140 11141 if (env.flags & LBF_NEED_BREAK) { 11142 env.flags &= ~LBF_NEED_BREAK; 11143 /* Stop if we tried all running tasks */ 11144 if (env.loop < busiest->nr_running) 11145 goto more_balance; 11146 } 11147 11148 /* 11149 * Revisit (affine) tasks on src_cpu that couldn't be moved to 11150 * us and move them to an alternate dst_cpu in our sched_group 11151 * where they can run. The upper limit on how many times we 11152 * iterate on same src_cpu is dependent on number of CPUs in our 11153 * sched_group. 11154 * 11155 * This changes load balance semantics a bit on who can move 11156 * load to a given_cpu. In addition to the given_cpu itself 11157 * (or a ilb_cpu acting on its behalf where given_cpu is 11158 * nohz-idle), we now have balance_cpu in a position to move 11159 * load to given_cpu. In rare situations, this may cause 11160 * conflicts (balance_cpu and given_cpu/ilb_cpu deciding 11161 * _independently_ and at _same_ time to move some load to 11162 * given_cpu) causing excess load to be moved to given_cpu. 11163 * This however should not happen so much in practice and 11164 * moreover subsequent load balance cycles should correct the 11165 * excess load moved. 11166 */ 11167 if ((env.flags & LBF_DST_PINNED) && env.imbalance > 0) { 11168 11169 /* Prevent to re-select dst_cpu via env's CPUs */ 11170 __cpumask_clear_cpu(env.dst_cpu, env.cpus); 11171 11172 env.dst_rq = cpu_rq(env.new_dst_cpu); 11173 env.dst_cpu = env.new_dst_cpu; 11174 env.flags &= ~LBF_DST_PINNED; 11175 env.loop = 0; 11176 env.loop_break = SCHED_NR_MIGRATE_BREAK; 11177 11178 /* 11179 * Go back to "more_balance" rather than "redo" since we 11180 * need to continue with same src_cpu. 11181 */ 11182 goto more_balance; 11183 } 11184 11185 /* 11186 * We failed to reach balance because of affinity. 11187 */ 11188 if (sd_parent) { 11189 int *group_imbalance = &sd_parent->groups->sgc->imbalance; 11190 11191 if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0) 11192 *group_imbalance = 1; 11193 } 11194 11195 /* All tasks on this runqueue were pinned by CPU affinity */ 11196 if (unlikely(env.flags & LBF_ALL_PINNED)) { 11197 __cpumask_clear_cpu(cpu_of(busiest), cpus); 11198 /* 11199 * Attempting to continue load balancing at the current 11200 * sched_domain level only makes sense if there are 11201 * active CPUs remaining as possible busiest CPUs to 11202 * pull load from which are not contained within the 11203 * destination group that is receiving any migrated 11204 * load. 11205 */ 11206 if (!cpumask_subset(cpus, env.dst_grpmask)) { 11207 env.loop = 0; 11208 env.loop_break = SCHED_NR_MIGRATE_BREAK; 11209 goto redo; 11210 } 11211 goto out_all_pinned; 11212 } 11213 } 11214 11215 if (!ld_moved) { 11216 schedstat_inc(sd->lb_failed[idle]); 11217 /* 11218 * Increment the failure counter only on periodic balance. 11219 * We do not want newidle balance, which can be very 11220 * frequent, pollute the failure counter causing 11221 * excessive cache_hot migrations and active balances. 11222 */ 11223 if (idle != CPU_NEWLY_IDLE) 11224 sd->nr_balance_failed++; 11225 11226 if (need_active_balance(&env)) { 11227 unsigned long flags; 11228 11229 raw_spin_rq_lock_irqsave(busiest, flags); 11230 11231 /* 11232 * Don't kick the active_load_balance_cpu_stop, 11233 * if the curr task on busiest CPU can't be 11234 * moved to this_cpu: 11235 */ 11236 if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr)) { 11237 raw_spin_rq_unlock_irqrestore(busiest, flags); 11238 goto out_one_pinned; 11239 } 11240 11241 /* Record that we found at least one task that could run on this_cpu */ 11242 env.flags &= ~LBF_ALL_PINNED; 11243 11244 /* 11245 * ->active_balance synchronizes accesses to 11246 * ->active_balance_work. Once set, it's cleared 11247 * only after active load balance is finished. 11248 */ 11249 if (!busiest->active_balance) { 11250 busiest->active_balance = 1; 11251 busiest->push_cpu = this_cpu; 11252 active_balance = 1; 11253 } 11254 raw_spin_rq_unlock_irqrestore(busiest, flags); 11255 11256 if (active_balance) { 11257 stop_one_cpu_nowait(cpu_of(busiest), 11258 active_load_balance_cpu_stop, busiest, 11259 &busiest->active_balance_work); 11260 } 11261 } 11262 } else { 11263 sd->nr_balance_failed = 0; 11264 } 11265 11266 if (likely(!active_balance) || need_active_balance(&env)) { 11267 /* We were unbalanced, so reset the balancing interval */ 11268 sd->balance_interval = sd->min_interval; 11269 } 11270 11271 goto out; 11272 11273 out_balanced: 11274 /* 11275 * We reach balance although we may have faced some affinity 11276 * constraints. Clear the imbalance flag only if other tasks got 11277 * a chance to move and fix the imbalance. 11278 */ 11279 if (sd_parent && !(env.flags & LBF_ALL_PINNED)) { 11280 int *group_imbalance = &sd_parent->groups->sgc->imbalance; 11281 11282 if (*group_imbalance) 11283 *group_imbalance = 0; 11284 } 11285 11286 out_all_pinned: 11287 /* 11288 * We reach balance because all tasks are pinned at this level so 11289 * we can't migrate them. Let the imbalance flag set so parent level 11290 * can try to migrate them. 11291 */ 11292 schedstat_inc(sd->lb_balanced[idle]); 11293 11294 sd->nr_balance_failed = 0; 11295 11296 out_one_pinned: 11297 ld_moved = 0; 11298 11299 /* 11300 * newidle_balance() disregards balance intervals, so we could 11301 * repeatedly reach this code, which would lead to balance_interval 11302 * skyrocketing in a short amount of time. Skip the balance_interval 11303 * increase logic to avoid that. 11304 */ 11305 if (env.idle == CPU_NEWLY_IDLE) 11306 goto out; 11307 11308 /* tune up the balancing interval */ 11309 if ((env.flags & LBF_ALL_PINNED && 11310 sd->balance_interval < MAX_PINNED_INTERVAL) || 11311 sd->balance_interval < sd->max_interval) 11312 sd->balance_interval *= 2; 11313 out: 11314 return ld_moved; 11315 } 11316 11317 static inline unsigned long 11318 get_sd_balance_interval(struct sched_domain *sd, int cpu_busy) 11319 { 11320 unsigned long interval = sd->balance_interval; 11321 11322 if (cpu_busy) 11323 interval *= sd->busy_factor; 11324 11325 /* scale ms to jiffies */ 11326 interval = msecs_to_jiffies(interval); 11327 11328 /* 11329 * Reduce likelihood of busy balancing at higher domains racing with 11330 * balancing at lower domains by preventing their balancing periods 11331 * from being multiples of each other. 11332 */ 11333 if (cpu_busy) 11334 interval -= 1; 11335 11336 interval = clamp(interval, 1UL, max_load_balance_interval); 11337 11338 return interval; 11339 } 11340 11341 static inline void 11342 update_next_balance(struct sched_domain *sd, unsigned long *next_balance) 11343 { 11344 unsigned long interval, next; 11345 11346 /* used by idle balance, so cpu_busy = 0 */ 11347 interval = get_sd_balance_interval(sd, 0); 11348 next = sd->last_balance + interval; 11349 11350 if (time_after(*next_balance, next)) 11351 *next_balance = next; 11352 } 11353 11354 /* 11355 * active_load_balance_cpu_stop is run by the CPU stopper. It pushes 11356 * running tasks off the busiest CPU onto idle CPUs. It requires at 11357 * least 1 task to be running on each physical CPU where possible, and 11358 * avoids physical / logical imbalances. 11359 */ 11360 static int active_load_balance_cpu_stop(void *data) 11361 { 11362 struct rq *busiest_rq = data; 11363 int busiest_cpu = cpu_of(busiest_rq); 11364 int target_cpu = busiest_rq->push_cpu; 11365 struct rq *target_rq = cpu_rq(target_cpu); 11366 struct sched_domain *sd; 11367 struct task_struct *p = NULL; 11368 struct rq_flags rf; 11369 11370 rq_lock_irq(busiest_rq, &rf); 11371 /* 11372 * Between queueing the stop-work and running it is a hole in which 11373 * CPUs can become inactive. We should not move tasks from or to 11374 * inactive CPUs. 11375 */ 11376 if (!cpu_active(busiest_cpu) || !cpu_active(target_cpu)) 11377 goto out_unlock; 11378 11379 /* Make sure the requested CPU hasn't gone down in the meantime: */ 11380 if (unlikely(busiest_cpu != smp_processor_id() || 11381 !busiest_rq->active_balance)) 11382 goto out_unlock; 11383 11384 /* Is there any task to move? */ 11385 if (busiest_rq->nr_running <= 1) 11386 goto out_unlock; 11387 11388 /* 11389 * This condition is "impossible", if it occurs 11390 * we need to fix it. Originally reported by 11391 * Bjorn Helgaas on a 128-CPU setup. 11392 */ 11393 WARN_ON_ONCE(busiest_rq == target_rq); 11394 11395 /* Search for an sd spanning us and the target CPU. */ 11396 rcu_read_lock(); 11397 for_each_domain(target_cpu, sd) { 11398 if (cpumask_test_cpu(busiest_cpu, sched_domain_span(sd))) 11399 break; 11400 } 11401 11402 if (likely(sd)) { 11403 struct lb_env env = { 11404 .sd = sd, 11405 .dst_cpu = target_cpu, 11406 .dst_rq = target_rq, 11407 .src_cpu = busiest_rq->cpu, 11408 .src_rq = busiest_rq, 11409 .idle = CPU_IDLE, 11410 .flags = LBF_ACTIVE_LB, 11411 }; 11412 11413 schedstat_inc(sd->alb_count); 11414 update_rq_clock(busiest_rq); 11415 11416 p = detach_one_task(&env); 11417 if (p) { 11418 schedstat_inc(sd->alb_pushed); 11419 /* Active balancing done, reset the failure counter. */ 11420 sd->nr_balance_failed = 0; 11421 } else { 11422 schedstat_inc(sd->alb_failed); 11423 } 11424 } 11425 rcu_read_unlock(); 11426 out_unlock: 11427 busiest_rq->active_balance = 0; 11428 rq_unlock(busiest_rq, &rf); 11429 11430 if (p) 11431 attach_one_task(target_rq, p); 11432 11433 local_irq_enable(); 11434 11435 return 0; 11436 } 11437 11438 static DEFINE_SPINLOCK(balancing); 11439 11440 /* 11441 * Scale the max load_balance interval with the number of CPUs in the system. 11442 * This trades load-balance latency on larger machines for less cross talk. 11443 */ 11444 void update_max_interval(void) 11445 { 11446 max_load_balance_interval = HZ*num_online_cpus()/10; 11447 } 11448 11449 static inline bool update_newidle_cost(struct sched_domain *sd, u64 cost) 11450 { 11451 if (cost > sd->max_newidle_lb_cost) { 11452 /* 11453 * Track max cost of a domain to make sure to not delay the 11454 * next wakeup on the CPU. 11455 */ 11456 sd->max_newidle_lb_cost = cost; 11457 sd->last_decay_max_lb_cost = jiffies; 11458 } else if (time_after(jiffies, sd->last_decay_max_lb_cost + HZ)) { 11459 /* 11460 * Decay the newidle max times by ~1% per second to ensure that 11461 * it is not outdated and the current max cost is actually 11462 * shorter. 11463 */ 11464 sd->max_newidle_lb_cost = (sd->max_newidle_lb_cost * 253) / 256; 11465 sd->last_decay_max_lb_cost = jiffies; 11466 11467 return true; 11468 } 11469 11470 return false; 11471 } 11472 11473 /* 11474 * It checks each scheduling domain to see if it is due to be balanced, 11475 * and initiates a balancing operation if so. 11476 * 11477 * Balancing parameters are set up in init_sched_domains. 11478 */ 11479 static void rebalance_domains(struct rq *rq, enum cpu_idle_type idle) 11480 { 11481 int continue_balancing = 1; 11482 int cpu = rq->cpu; 11483 int busy = idle != CPU_IDLE && !sched_idle_cpu(cpu); 11484 unsigned long interval; 11485 struct sched_domain *sd; 11486 /* Earliest time when we have to do rebalance again */ 11487 unsigned long next_balance = jiffies + 60*HZ; 11488 int update_next_balance = 0; 11489 int need_serialize, need_decay = 0; 11490 u64 max_cost = 0; 11491 11492 rcu_read_lock(); 11493 for_each_domain(cpu, sd) { 11494 /* 11495 * Decay the newidle max times here because this is a regular 11496 * visit to all the domains. 11497 */ 11498 need_decay = update_newidle_cost(sd, 0); 11499 max_cost += sd->max_newidle_lb_cost; 11500 11501 /* 11502 * Stop the load balance at this level. There is another 11503 * CPU in our sched group which is doing load balancing more 11504 * actively. 11505 */ 11506 if (!continue_balancing) { 11507 if (need_decay) 11508 continue; 11509 break; 11510 } 11511 11512 interval = get_sd_balance_interval(sd, busy); 11513 11514 need_serialize = sd->flags & SD_SERIALIZE; 11515 if (need_serialize) { 11516 if (!spin_trylock(&balancing)) 11517 goto out; 11518 } 11519 11520 if (time_after_eq(jiffies, sd->last_balance + interval)) { 11521 if (load_balance(cpu, rq, sd, idle, &continue_balancing)) { 11522 /* 11523 * The LBF_DST_PINNED logic could have changed 11524 * env->dst_cpu, so we can't know our idle 11525 * state even if we migrated tasks. Update it. 11526 */ 11527 idle = idle_cpu(cpu) ? CPU_IDLE : CPU_NOT_IDLE; 11528 busy = idle != CPU_IDLE && !sched_idle_cpu(cpu); 11529 } 11530 sd->last_balance = jiffies; 11531 interval = get_sd_balance_interval(sd, busy); 11532 } 11533 if (need_serialize) 11534 spin_unlock(&balancing); 11535 out: 11536 if (time_after(next_balance, sd->last_balance + interval)) { 11537 next_balance = sd->last_balance + interval; 11538 update_next_balance = 1; 11539 } 11540 } 11541 if (need_decay) { 11542 /* 11543 * Ensure the rq-wide value also decays but keep it at a 11544 * reasonable floor to avoid funnies with rq->avg_idle. 11545 */ 11546 rq->max_idle_balance_cost = 11547 max((u64)sysctl_sched_migration_cost, max_cost); 11548 } 11549 rcu_read_unlock(); 11550 11551 /* 11552 * next_balance will be updated only when there is a need. 11553 * When the cpu is attached to null domain for ex, it will not be 11554 * updated. 11555 */ 11556 if (likely(update_next_balance)) 11557 rq->next_balance = next_balance; 11558 11559 } 11560 11561 static inline int on_null_domain(struct rq *rq) 11562 { 11563 return unlikely(!rcu_dereference_sched(rq->sd)); 11564 } 11565 11566 #ifdef CONFIG_NO_HZ_COMMON 11567 /* 11568 * idle load balancing details 11569 * - When one of the busy CPUs notice that there may be an idle rebalancing 11570 * needed, they will kick the idle load balancer, which then does idle 11571 * load balancing for all the idle CPUs. 11572 * - HK_TYPE_MISC CPUs are used for this task, because HK_TYPE_SCHED not set 11573 * anywhere yet. 11574 */ 11575 11576 static inline int find_new_ilb(void) 11577 { 11578 int ilb; 11579 const struct cpumask *hk_mask; 11580 11581 hk_mask = housekeeping_cpumask(HK_TYPE_MISC); 11582 11583 for_each_cpu_and(ilb, nohz.idle_cpus_mask, hk_mask) { 11584 11585 if (ilb == smp_processor_id()) 11586 continue; 11587 11588 if (idle_cpu(ilb)) 11589 return ilb; 11590 } 11591 11592 return nr_cpu_ids; 11593 } 11594 11595 /* 11596 * Kick a CPU to do the nohz balancing, if it is time for it. We pick any 11597 * idle CPU in the HK_TYPE_MISC housekeeping set (if there is one). 11598 */ 11599 static void kick_ilb(unsigned int flags) 11600 { 11601 int ilb_cpu; 11602 11603 /* 11604 * Increase nohz.next_balance only when if full ilb is triggered but 11605 * not if we only update stats. 11606 */ 11607 if (flags & NOHZ_BALANCE_KICK) 11608 nohz.next_balance = jiffies+1; 11609 11610 ilb_cpu = find_new_ilb(); 11611 11612 if (ilb_cpu >= nr_cpu_ids) 11613 return; 11614 11615 /* 11616 * Access to rq::nohz_csd is serialized by NOHZ_KICK_MASK; he who sets 11617 * the first flag owns it; cleared by nohz_csd_func(). 11618 */ 11619 flags = atomic_fetch_or(flags, nohz_flags(ilb_cpu)); 11620 if (flags & NOHZ_KICK_MASK) 11621 return; 11622 11623 /* 11624 * This way we generate an IPI on the target CPU which 11625 * is idle. And the softirq performing nohz idle load balance 11626 * will be run before returning from the IPI. 11627 */ 11628 smp_call_function_single_async(ilb_cpu, &cpu_rq(ilb_cpu)->nohz_csd); 11629 } 11630 11631 /* 11632 * Current decision point for kicking the idle load balancer in the presence 11633 * of idle CPUs in the system. 11634 */ 11635 static void nohz_balancer_kick(struct rq *rq) 11636 { 11637 unsigned long now = jiffies; 11638 struct sched_domain_shared *sds; 11639 struct sched_domain *sd; 11640 int nr_busy, i, cpu = rq->cpu; 11641 unsigned int flags = 0; 11642 11643 if (unlikely(rq->idle_balance)) 11644 return; 11645 11646 /* 11647 * We may be recently in ticked or tickless idle mode. At the first 11648 * busy tick after returning from idle, we will update the busy stats. 11649 */ 11650 nohz_balance_exit_idle(rq); 11651 11652 /* 11653 * None are in tickless mode and hence no need for NOHZ idle load 11654 * balancing. 11655 */ 11656 if (likely(!atomic_read(&nohz.nr_cpus))) 11657 return; 11658 11659 if (READ_ONCE(nohz.has_blocked) && 11660 time_after(now, READ_ONCE(nohz.next_blocked))) 11661 flags = NOHZ_STATS_KICK; 11662 11663 if (time_before(now, nohz.next_balance)) 11664 goto out; 11665 11666 if (rq->nr_running >= 2) { 11667 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 11668 goto out; 11669 } 11670 11671 rcu_read_lock(); 11672 11673 sd = rcu_dereference(rq->sd); 11674 if (sd) { 11675 /* 11676 * If there's a CFS task and the current CPU has reduced 11677 * capacity; kick the ILB to see if there's a better CPU to run 11678 * on. 11679 */ 11680 if (rq->cfs.h_nr_running >= 1 && check_cpu_capacity(rq, sd)) { 11681 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 11682 goto unlock; 11683 } 11684 } 11685 11686 sd = rcu_dereference(per_cpu(sd_asym_packing, cpu)); 11687 if (sd) { 11688 /* 11689 * When ASYM_PACKING; see if there's a more preferred CPU 11690 * currently idle; in which case, kick the ILB to move tasks 11691 * around. 11692 * 11693 * When balancing betwen cores, all the SMT siblings of the 11694 * preferred CPU must be idle. 11695 */ 11696 for_each_cpu_and(i, sched_domain_span(sd), nohz.idle_cpus_mask) { 11697 if (sched_use_asym_prio(sd, i) && 11698 sched_asym_prefer(i, cpu)) { 11699 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 11700 goto unlock; 11701 } 11702 } 11703 } 11704 11705 sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, cpu)); 11706 if (sd) { 11707 /* 11708 * When ASYM_CPUCAPACITY; see if there's a higher capacity CPU 11709 * to run the misfit task on. 11710 */ 11711 if (check_misfit_status(rq, sd)) { 11712 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 11713 goto unlock; 11714 } 11715 11716 /* 11717 * For asymmetric systems, we do not want to nicely balance 11718 * cache use, instead we want to embrace asymmetry and only 11719 * ensure tasks have enough CPU capacity. 11720 * 11721 * Skip the LLC logic because it's not relevant in that case. 11722 */ 11723 goto unlock; 11724 } 11725 11726 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); 11727 if (sds) { 11728 /* 11729 * If there is an imbalance between LLC domains (IOW we could 11730 * increase the overall cache use), we need some less-loaded LLC 11731 * domain to pull some load. Likewise, we may need to spread 11732 * load within the current LLC domain (e.g. packed SMT cores but 11733 * other CPUs are idle). We can't really know from here how busy 11734 * the others are - so just get a nohz balance going if it looks 11735 * like this LLC domain has tasks we could move. 11736 */ 11737 nr_busy = atomic_read(&sds->nr_busy_cpus); 11738 if (nr_busy > 1) { 11739 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 11740 goto unlock; 11741 } 11742 } 11743 unlock: 11744 rcu_read_unlock(); 11745 out: 11746 if (READ_ONCE(nohz.needs_update)) 11747 flags |= NOHZ_NEXT_KICK; 11748 11749 if (flags) 11750 kick_ilb(flags); 11751 } 11752 11753 static void set_cpu_sd_state_busy(int cpu) 11754 { 11755 struct sched_domain *sd; 11756 11757 rcu_read_lock(); 11758 sd = rcu_dereference(per_cpu(sd_llc, cpu)); 11759 11760 if (!sd || !sd->nohz_idle) 11761 goto unlock; 11762 sd->nohz_idle = 0; 11763 11764 atomic_inc(&sd->shared->nr_busy_cpus); 11765 unlock: 11766 rcu_read_unlock(); 11767 } 11768 11769 void nohz_balance_exit_idle(struct rq *rq) 11770 { 11771 SCHED_WARN_ON(rq != this_rq()); 11772 11773 if (likely(!rq->nohz_tick_stopped)) 11774 return; 11775 11776 rq->nohz_tick_stopped = 0; 11777 cpumask_clear_cpu(rq->cpu, nohz.idle_cpus_mask); 11778 atomic_dec(&nohz.nr_cpus); 11779 11780 set_cpu_sd_state_busy(rq->cpu); 11781 } 11782 11783 static void set_cpu_sd_state_idle(int cpu) 11784 { 11785 struct sched_domain *sd; 11786 11787 rcu_read_lock(); 11788 sd = rcu_dereference(per_cpu(sd_llc, cpu)); 11789 11790 if (!sd || sd->nohz_idle) 11791 goto unlock; 11792 sd->nohz_idle = 1; 11793 11794 atomic_dec(&sd->shared->nr_busy_cpus); 11795 unlock: 11796 rcu_read_unlock(); 11797 } 11798 11799 /* 11800 * This routine will record that the CPU is going idle with tick stopped. 11801 * This info will be used in performing idle load balancing in the future. 11802 */ 11803 void nohz_balance_enter_idle(int cpu) 11804 { 11805 struct rq *rq = cpu_rq(cpu); 11806 11807 SCHED_WARN_ON(cpu != smp_processor_id()); 11808 11809 /* If this CPU is going down, then nothing needs to be done: */ 11810 if (!cpu_active(cpu)) 11811 return; 11812 11813 /* Spare idle load balancing on CPUs that don't want to be disturbed: */ 11814 if (!housekeeping_cpu(cpu, HK_TYPE_SCHED)) 11815 return; 11816 11817 /* 11818 * Can be set safely without rq->lock held 11819 * If a clear happens, it will have evaluated last additions because 11820 * rq->lock is held during the check and the clear 11821 */ 11822 rq->has_blocked_load = 1; 11823 11824 /* 11825 * The tick is still stopped but load could have been added in the 11826 * meantime. We set the nohz.has_blocked flag to trig a check of the 11827 * *_avg. The CPU is already part of nohz.idle_cpus_mask so the clear 11828 * of nohz.has_blocked can only happen after checking the new load 11829 */ 11830 if (rq->nohz_tick_stopped) 11831 goto out; 11832 11833 /* If we're a completely isolated CPU, we don't play: */ 11834 if (on_null_domain(rq)) 11835 return; 11836 11837 rq->nohz_tick_stopped = 1; 11838 11839 cpumask_set_cpu(cpu, nohz.idle_cpus_mask); 11840 atomic_inc(&nohz.nr_cpus); 11841 11842 /* 11843 * Ensures that if nohz_idle_balance() fails to observe our 11844 * @idle_cpus_mask store, it must observe the @has_blocked 11845 * and @needs_update stores. 11846 */ 11847 smp_mb__after_atomic(); 11848 11849 set_cpu_sd_state_idle(cpu); 11850 11851 WRITE_ONCE(nohz.needs_update, 1); 11852 out: 11853 /* 11854 * Each time a cpu enter idle, we assume that it has blocked load and 11855 * enable the periodic update of the load of idle cpus 11856 */ 11857 WRITE_ONCE(nohz.has_blocked, 1); 11858 } 11859 11860 static bool update_nohz_stats(struct rq *rq) 11861 { 11862 unsigned int cpu = rq->cpu; 11863 11864 if (!rq->has_blocked_load) 11865 return false; 11866 11867 if (!cpumask_test_cpu(cpu, nohz.idle_cpus_mask)) 11868 return false; 11869 11870 if (!time_after(jiffies, READ_ONCE(rq->last_blocked_load_update_tick))) 11871 return true; 11872 11873 update_blocked_averages(cpu); 11874 11875 return rq->has_blocked_load; 11876 } 11877 11878 /* 11879 * Internal function that runs load balance for all idle cpus. The load balance 11880 * can be a simple update of blocked load or a complete load balance with 11881 * tasks movement depending of flags. 11882 */ 11883 static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags) 11884 { 11885 /* Earliest time when we have to do rebalance again */ 11886 unsigned long now = jiffies; 11887 unsigned long next_balance = now + 60*HZ; 11888 bool has_blocked_load = false; 11889 int update_next_balance = 0; 11890 int this_cpu = this_rq->cpu; 11891 int balance_cpu; 11892 struct rq *rq; 11893 11894 SCHED_WARN_ON((flags & NOHZ_KICK_MASK) == NOHZ_BALANCE_KICK); 11895 11896 /* 11897 * We assume there will be no idle load after this update and clear 11898 * the has_blocked flag. If a cpu enters idle in the mean time, it will 11899 * set the has_blocked flag and trigger another update of idle load. 11900 * Because a cpu that becomes idle, is added to idle_cpus_mask before 11901 * setting the flag, we are sure to not clear the state and not 11902 * check the load of an idle cpu. 11903 * 11904 * Same applies to idle_cpus_mask vs needs_update. 11905 */ 11906 if (flags & NOHZ_STATS_KICK) 11907 WRITE_ONCE(nohz.has_blocked, 0); 11908 if (flags & NOHZ_NEXT_KICK) 11909 WRITE_ONCE(nohz.needs_update, 0); 11910 11911 /* 11912 * Ensures that if we miss the CPU, we must see the has_blocked 11913 * store from nohz_balance_enter_idle(). 11914 */ 11915 smp_mb(); 11916 11917 /* 11918 * Start with the next CPU after this_cpu so we will end with this_cpu and let a 11919 * chance for other idle cpu to pull load. 11920 */ 11921 for_each_cpu_wrap(balance_cpu, nohz.idle_cpus_mask, this_cpu+1) { 11922 if (!idle_cpu(balance_cpu)) 11923 continue; 11924 11925 /* 11926 * If this CPU gets work to do, stop the load balancing 11927 * work being done for other CPUs. Next load 11928 * balancing owner will pick it up. 11929 */ 11930 if (need_resched()) { 11931 if (flags & NOHZ_STATS_KICK) 11932 has_blocked_load = true; 11933 if (flags & NOHZ_NEXT_KICK) 11934 WRITE_ONCE(nohz.needs_update, 1); 11935 goto abort; 11936 } 11937 11938 rq = cpu_rq(balance_cpu); 11939 11940 if (flags & NOHZ_STATS_KICK) 11941 has_blocked_load |= update_nohz_stats(rq); 11942 11943 /* 11944 * If time for next balance is due, 11945 * do the balance. 11946 */ 11947 if (time_after_eq(jiffies, rq->next_balance)) { 11948 struct rq_flags rf; 11949 11950 rq_lock_irqsave(rq, &rf); 11951 update_rq_clock(rq); 11952 rq_unlock_irqrestore(rq, &rf); 11953 11954 if (flags & NOHZ_BALANCE_KICK) 11955 rebalance_domains(rq, CPU_IDLE); 11956 } 11957 11958 if (time_after(next_balance, rq->next_balance)) { 11959 next_balance = rq->next_balance; 11960 update_next_balance = 1; 11961 } 11962 } 11963 11964 /* 11965 * next_balance will be updated only when there is a need. 11966 * When the CPU is attached to null domain for ex, it will not be 11967 * updated. 11968 */ 11969 if (likely(update_next_balance)) 11970 nohz.next_balance = next_balance; 11971 11972 if (flags & NOHZ_STATS_KICK) 11973 WRITE_ONCE(nohz.next_blocked, 11974 now + msecs_to_jiffies(LOAD_AVG_PERIOD)); 11975 11976 abort: 11977 /* There is still blocked load, enable periodic update */ 11978 if (has_blocked_load) 11979 WRITE_ONCE(nohz.has_blocked, 1); 11980 } 11981 11982 /* 11983 * In CONFIG_NO_HZ_COMMON case, the idle balance kickee will do the 11984 * rebalancing for all the cpus for whom scheduler ticks are stopped. 11985 */ 11986 static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) 11987 { 11988 unsigned int flags = this_rq->nohz_idle_balance; 11989 11990 if (!flags) 11991 return false; 11992 11993 this_rq->nohz_idle_balance = 0; 11994 11995 if (idle != CPU_IDLE) 11996 return false; 11997 11998 _nohz_idle_balance(this_rq, flags); 11999 12000 return true; 12001 } 12002 12003 /* 12004 * Check if we need to run the ILB for updating blocked load before entering 12005 * idle state. 12006 */ 12007 void nohz_run_idle_balance(int cpu) 12008 { 12009 unsigned int flags; 12010 12011 flags = atomic_fetch_andnot(NOHZ_NEWILB_KICK, nohz_flags(cpu)); 12012 12013 /* 12014 * Update the blocked load only if no SCHED_SOFTIRQ is about to happen 12015 * (ie NOHZ_STATS_KICK set) and will do the same. 12016 */ 12017 if ((flags == NOHZ_NEWILB_KICK) && !need_resched()) 12018 _nohz_idle_balance(cpu_rq(cpu), NOHZ_STATS_KICK); 12019 } 12020 12021 static void nohz_newidle_balance(struct rq *this_rq) 12022 { 12023 int this_cpu = this_rq->cpu; 12024 12025 /* 12026 * This CPU doesn't want to be disturbed by scheduler 12027 * housekeeping 12028 */ 12029 if (!housekeeping_cpu(this_cpu, HK_TYPE_SCHED)) 12030 return; 12031 12032 /* Will wake up very soon. No time for doing anything else*/ 12033 if (this_rq->avg_idle < sysctl_sched_migration_cost) 12034 return; 12035 12036 /* Don't need to update blocked load of idle CPUs*/ 12037 if (!READ_ONCE(nohz.has_blocked) || 12038 time_before(jiffies, READ_ONCE(nohz.next_blocked))) 12039 return; 12040 12041 /* 12042 * Set the need to trigger ILB in order to update blocked load 12043 * before entering idle state. 12044 */ 12045 atomic_or(NOHZ_NEWILB_KICK, nohz_flags(this_cpu)); 12046 } 12047 12048 #else /* !CONFIG_NO_HZ_COMMON */ 12049 static inline void nohz_balancer_kick(struct rq *rq) { } 12050 12051 static inline bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) 12052 { 12053 return false; 12054 } 12055 12056 static inline void nohz_newidle_balance(struct rq *this_rq) { } 12057 #endif /* CONFIG_NO_HZ_COMMON */ 12058 12059 /* 12060 * newidle_balance is called by schedule() if this_cpu is about to become 12061 * idle. Attempts to pull tasks from other CPUs. 12062 * 12063 * Returns: 12064 * < 0 - we released the lock and there are !fair tasks present 12065 * 0 - failed, no new tasks 12066 * > 0 - success, new (fair) tasks present 12067 */ 12068 static int newidle_balance(struct rq *this_rq, struct rq_flags *rf) 12069 { 12070 unsigned long next_balance = jiffies + HZ; 12071 int this_cpu = this_rq->cpu; 12072 u64 t0, t1, curr_cost = 0; 12073 struct sched_domain *sd; 12074 int pulled_task = 0; 12075 12076 update_misfit_status(NULL, this_rq); 12077 12078 /* 12079 * There is a task waiting to run. No need to search for one. 12080 * Return 0; the task will be enqueued when switching to idle. 12081 */ 12082 if (this_rq->ttwu_pending) 12083 return 0; 12084 12085 /* 12086 * We must set idle_stamp _before_ calling idle_balance(), such that we 12087 * measure the duration of idle_balance() as idle time. 12088 */ 12089 this_rq->idle_stamp = rq_clock(this_rq); 12090 12091 /* 12092 * Do not pull tasks towards !active CPUs... 12093 */ 12094 if (!cpu_active(this_cpu)) 12095 return 0; 12096 12097 /* 12098 * This is OK, because current is on_cpu, which avoids it being picked 12099 * for load-balance and preemption/IRQs are still disabled avoiding 12100 * further scheduler activity on it and we're being very careful to 12101 * re-start the picking loop. 12102 */ 12103 rq_unpin_lock(this_rq, rf); 12104 12105 rcu_read_lock(); 12106 sd = rcu_dereference_check_sched_domain(this_rq->sd); 12107 12108 if (!READ_ONCE(this_rq->rd->overload) || 12109 (sd && this_rq->avg_idle < sd->max_newidle_lb_cost)) { 12110 12111 if (sd) 12112 update_next_balance(sd, &next_balance); 12113 rcu_read_unlock(); 12114 12115 goto out; 12116 } 12117 rcu_read_unlock(); 12118 12119 raw_spin_rq_unlock(this_rq); 12120 12121 t0 = sched_clock_cpu(this_cpu); 12122 update_blocked_averages(this_cpu); 12123 12124 rcu_read_lock(); 12125 for_each_domain(this_cpu, sd) { 12126 int continue_balancing = 1; 12127 u64 domain_cost; 12128 12129 update_next_balance(sd, &next_balance); 12130 12131 if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost) 12132 break; 12133 12134 if (sd->flags & SD_BALANCE_NEWIDLE) { 12135 12136 pulled_task = load_balance(this_cpu, this_rq, 12137 sd, CPU_NEWLY_IDLE, 12138 &continue_balancing); 12139 12140 t1 = sched_clock_cpu(this_cpu); 12141 domain_cost = t1 - t0; 12142 update_newidle_cost(sd, domain_cost); 12143 12144 curr_cost += domain_cost; 12145 t0 = t1; 12146 } 12147 12148 /* 12149 * Stop searching for tasks to pull if there are 12150 * now runnable tasks on this rq. 12151 */ 12152 if (pulled_task || this_rq->nr_running > 0 || 12153 this_rq->ttwu_pending) 12154 break; 12155 } 12156 rcu_read_unlock(); 12157 12158 raw_spin_rq_lock(this_rq); 12159 12160 if (curr_cost > this_rq->max_idle_balance_cost) 12161 this_rq->max_idle_balance_cost = curr_cost; 12162 12163 /* 12164 * While browsing the domains, we released the rq lock, a task could 12165 * have been enqueued in the meantime. Since we're not going idle, 12166 * pretend we pulled a task. 12167 */ 12168 if (this_rq->cfs.h_nr_running && !pulled_task) 12169 pulled_task = 1; 12170 12171 /* Is there a task of a high priority class? */ 12172 if (this_rq->nr_running != this_rq->cfs.h_nr_running) 12173 pulled_task = -1; 12174 12175 out: 12176 /* Move the next balance forward */ 12177 if (time_after(this_rq->next_balance, next_balance)) 12178 this_rq->next_balance = next_balance; 12179 12180 if (pulled_task) 12181 this_rq->idle_stamp = 0; 12182 else 12183 nohz_newidle_balance(this_rq); 12184 12185 rq_repin_lock(this_rq, rf); 12186 12187 return pulled_task; 12188 } 12189 12190 /* 12191 * run_rebalance_domains is triggered when needed from the scheduler tick. 12192 * Also triggered for nohz idle balancing (with nohz_balancing_kick set). 12193 */ 12194 static __latent_entropy void run_rebalance_domains(struct softirq_action *h) 12195 { 12196 struct rq *this_rq = this_rq(); 12197 enum cpu_idle_type idle = this_rq->idle_balance ? 12198 CPU_IDLE : CPU_NOT_IDLE; 12199 12200 /* 12201 * If this CPU has a pending nohz_balance_kick, then do the 12202 * balancing on behalf of the other idle CPUs whose ticks are 12203 * stopped. Do nohz_idle_balance *before* rebalance_domains to 12204 * give the idle CPUs a chance to load balance. Else we may 12205 * load balance only within the local sched_domain hierarchy 12206 * and abort nohz_idle_balance altogether if we pull some load. 12207 */ 12208 if (nohz_idle_balance(this_rq, idle)) 12209 return; 12210 12211 /* normal load balance */ 12212 update_blocked_averages(this_rq->cpu); 12213 rebalance_domains(this_rq, idle); 12214 } 12215 12216 /* 12217 * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing. 12218 */ 12219 void trigger_load_balance(struct rq *rq) 12220 { 12221 /* 12222 * Don't need to rebalance while attached to NULL domain or 12223 * runqueue CPU is not active 12224 */ 12225 if (unlikely(on_null_domain(rq) || !cpu_active(cpu_of(rq)))) 12226 return; 12227 12228 if (time_after_eq(jiffies, rq->next_balance)) 12229 raise_softirq(SCHED_SOFTIRQ); 12230 12231 nohz_balancer_kick(rq); 12232 } 12233 12234 static void rq_online_fair(struct rq *rq) 12235 { 12236 update_sysctl(); 12237 12238 update_runtime_enabled(rq); 12239 } 12240 12241 static void rq_offline_fair(struct rq *rq) 12242 { 12243 update_sysctl(); 12244 12245 /* Ensure any throttled groups are reachable by pick_next_task */ 12246 unthrottle_offline_cfs_rqs(rq); 12247 } 12248 12249 #endif /* CONFIG_SMP */ 12250 12251 #ifdef CONFIG_SCHED_CORE 12252 static inline bool 12253 __entity_slice_used(struct sched_entity *se, int min_nr_tasks) 12254 { 12255 u64 rtime = se->sum_exec_runtime - se->prev_sum_exec_runtime; 12256 u64 slice = se->slice; 12257 12258 return (rtime * min_nr_tasks > slice); 12259 } 12260 12261 #define MIN_NR_TASKS_DURING_FORCEIDLE 2 12262 static inline void task_tick_core(struct rq *rq, struct task_struct *curr) 12263 { 12264 if (!sched_core_enabled(rq)) 12265 return; 12266 12267 /* 12268 * If runqueue has only one task which used up its slice and 12269 * if the sibling is forced idle, then trigger schedule to 12270 * give forced idle task a chance. 12271 * 12272 * sched_slice() considers only this active rq and it gets the 12273 * whole slice. But during force idle, we have siblings acting 12274 * like a single runqueue and hence we need to consider runnable 12275 * tasks on this CPU and the forced idle CPU. Ideally, we should 12276 * go through the forced idle rq, but that would be a perf hit. 12277 * We can assume that the forced idle CPU has at least 12278 * MIN_NR_TASKS_DURING_FORCEIDLE - 1 tasks and use that to check 12279 * if we need to give up the CPU. 12280 */ 12281 if (rq->core->core_forceidle_count && rq->cfs.nr_running == 1 && 12282 __entity_slice_used(&curr->se, MIN_NR_TASKS_DURING_FORCEIDLE)) 12283 resched_curr(rq); 12284 } 12285 12286 /* 12287 * se_fi_update - Update the cfs_rq->min_vruntime_fi in a CFS hierarchy if needed. 12288 */ 12289 static void se_fi_update(const struct sched_entity *se, unsigned int fi_seq, 12290 bool forceidle) 12291 { 12292 for_each_sched_entity(se) { 12293 struct cfs_rq *cfs_rq = cfs_rq_of(se); 12294 12295 if (forceidle) { 12296 if (cfs_rq->forceidle_seq == fi_seq) 12297 break; 12298 cfs_rq->forceidle_seq = fi_seq; 12299 } 12300 12301 cfs_rq->min_vruntime_fi = cfs_rq->min_vruntime; 12302 } 12303 } 12304 12305 void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi) 12306 { 12307 struct sched_entity *se = &p->se; 12308 12309 if (p->sched_class != &fair_sched_class) 12310 return; 12311 12312 se_fi_update(se, rq->core->core_forceidle_seq, in_fi); 12313 } 12314 12315 bool cfs_prio_less(const struct task_struct *a, const struct task_struct *b, 12316 bool in_fi) 12317 { 12318 struct rq *rq = task_rq(a); 12319 const struct sched_entity *sea = &a->se; 12320 const struct sched_entity *seb = &b->se; 12321 struct cfs_rq *cfs_rqa; 12322 struct cfs_rq *cfs_rqb; 12323 s64 delta; 12324 12325 SCHED_WARN_ON(task_rq(b)->core != rq->core); 12326 12327 #ifdef CONFIG_FAIR_GROUP_SCHED 12328 /* 12329 * Find an se in the hierarchy for tasks a and b, such that the se's 12330 * are immediate siblings. 12331 */ 12332 while (sea->cfs_rq->tg != seb->cfs_rq->tg) { 12333 int sea_depth = sea->depth; 12334 int seb_depth = seb->depth; 12335 12336 if (sea_depth >= seb_depth) 12337 sea = parent_entity(sea); 12338 if (sea_depth <= seb_depth) 12339 seb = parent_entity(seb); 12340 } 12341 12342 se_fi_update(sea, rq->core->core_forceidle_seq, in_fi); 12343 se_fi_update(seb, rq->core->core_forceidle_seq, in_fi); 12344 12345 cfs_rqa = sea->cfs_rq; 12346 cfs_rqb = seb->cfs_rq; 12347 #else 12348 cfs_rqa = &task_rq(a)->cfs; 12349 cfs_rqb = &task_rq(b)->cfs; 12350 #endif 12351 12352 /* 12353 * Find delta after normalizing se's vruntime with its cfs_rq's 12354 * min_vruntime_fi, which would have been updated in prior calls 12355 * to se_fi_update(). 12356 */ 12357 delta = (s64)(sea->vruntime - seb->vruntime) + 12358 (s64)(cfs_rqb->min_vruntime_fi - cfs_rqa->min_vruntime_fi); 12359 12360 return delta > 0; 12361 } 12362 12363 static int task_is_throttled_fair(struct task_struct *p, int cpu) 12364 { 12365 struct cfs_rq *cfs_rq; 12366 12367 #ifdef CONFIG_FAIR_GROUP_SCHED 12368 cfs_rq = task_group(p)->cfs_rq[cpu]; 12369 #else 12370 cfs_rq = &cpu_rq(cpu)->cfs; 12371 #endif 12372 return throttled_hierarchy(cfs_rq); 12373 } 12374 #else 12375 static inline void task_tick_core(struct rq *rq, struct task_struct *curr) {} 12376 #endif 12377 12378 /* 12379 * scheduler tick hitting a task of our scheduling class. 12380 * 12381 * NOTE: This function can be called remotely by the tick offload that 12382 * goes along full dynticks. Therefore no local assumption can be made 12383 * and everything must be accessed through the @rq and @curr passed in 12384 * parameters. 12385 */ 12386 static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) 12387 { 12388 struct cfs_rq *cfs_rq; 12389 struct sched_entity *se = &curr->se; 12390 12391 for_each_sched_entity(se) { 12392 cfs_rq = cfs_rq_of(se); 12393 entity_tick(cfs_rq, se, queued); 12394 } 12395 12396 if (static_branch_unlikely(&sched_numa_balancing)) 12397 task_tick_numa(rq, curr); 12398 12399 update_misfit_status(curr, rq); 12400 update_overutilized_status(task_rq(curr)); 12401 12402 task_tick_core(rq, curr); 12403 } 12404 12405 /* 12406 * called on fork with the child task as argument from the parent's context 12407 * - child not yet on the tasklist 12408 * - preemption disabled 12409 */ 12410 static void task_fork_fair(struct task_struct *p) 12411 { 12412 struct sched_entity *se = &p->se, *curr; 12413 struct cfs_rq *cfs_rq; 12414 struct rq *rq = this_rq(); 12415 struct rq_flags rf; 12416 12417 rq_lock(rq, &rf); 12418 update_rq_clock(rq); 12419 12420 cfs_rq = task_cfs_rq(current); 12421 curr = cfs_rq->curr; 12422 if (curr) 12423 update_curr(cfs_rq); 12424 place_entity(cfs_rq, se, ENQUEUE_INITIAL); 12425 rq_unlock(rq, &rf); 12426 } 12427 12428 /* 12429 * Priority of the task has changed. Check to see if we preempt 12430 * the current task. 12431 */ 12432 static void 12433 prio_changed_fair(struct rq *rq, struct task_struct *p, int oldprio) 12434 { 12435 if (!task_on_rq_queued(p)) 12436 return; 12437 12438 if (rq->cfs.nr_running == 1) 12439 return; 12440 12441 /* 12442 * Reschedule if we are currently running on this runqueue and 12443 * our priority decreased, or if we are not currently running on 12444 * this runqueue and our priority is higher than the current's 12445 */ 12446 if (task_current(rq, p)) { 12447 if (p->prio > oldprio) 12448 resched_curr(rq); 12449 } else 12450 check_preempt_curr(rq, p, 0); 12451 } 12452 12453 #ifdef CONFIG_FAIR_GROUP_SCHED 12454 /* 12455 * Propagate the changes of the sched_entity across the tg tree to make it 12456 * visible to the root 12457 */ 12458 static void propagate_entity_cfs_rq(struct sched_entity *se) 12459 { 12460 struct cfs_rq *cfs_rq = cfs_rq_of(se); 12461 12462 if (cfs_rq_throttled(cfs_rq)) 12463 return; 12464 12465 if (!throttled_hierarchy(cfs_rq)) 12466 list_add_leaf_cfs_rq(cfs_rq); 12467 12468 /* Start to propagate at parent */ 12469 se = se->parent; 12470 12471 for_each_sched_entity(se) { 12472 cfs_rq = cfs_rq_of(se); 12473 12474 update_load_avg(cfs_rq, se, UPDATE_TG); 12475 12476 if (cfs_rq_throttled(cfs_rq)) 12477 break; 12478 12479 if (!throttled_hierarchy(cfs_rq)) 12480 list_add_leaf_cfs_rq(cfs_rq); 12481 } 12482 } 12483 #else 12484 static void propagate_entity_cfs_rq(struct sched_entity *se) { } 12485 #endif 12486 12487 static void detach_entity_cfs_rq(struct sched_entity *se) 12488 { 12489 struct cfs_rq *cfs_rq = cfs_rq_of(se); 12490 12491 #ifdef CONFIG_SMP 12492 /* 12493 * In case the task sched_avg hasn't been attached: 12494 * - A forked task which hasn't been woken up by wake_up_new_task(). 12495 * - A task which has been woken up by try_to_wake_up() but is 12496 * waiting for actually being woken up by sched_ttwu_pending(). 12497 */ 12498 if (!se->avg.last_update_time) 12499 return; 12500 #endif 12501 12502 /* Catch up with the cfs_rq and remove our load when we leave */ 12503 update_load_avg(cfs_rq, se, 0); 12504 detach_entity_load_avg(cfs_rq, se); 12505 update_tg_load_avg(cfs_rq); 12506 propagate_entity_cfs_rq(se); 12507 } 12508 12509 static void attach_entity_cfs_rq(struct sched_entity *se) 12510 { 12511 struct cfs_rq *cfs_rq = cfs_rq_of(se); 12512 12513 /* Synchronize entity with its cfs_rq */ 12514 update_load_avg(cfs_rq, se, sched_feat(ATTACH_AGE_LOAD) ? 0 : SKIP_AGE_LOAD); 12515 attach_entity_load_avg(cfs_rq, se); 12516 update_tg_load_avg(cfs_rq); 12517 propagate_entity_cfs_rq(se); 12518 } 12519 12520 static void detach_task_cfs_rq(struct task_struct *p) 12521 { 12522 struct sched_entity *se = &p->se; 12523 12524 detach_entity_cfs_rq(se); 12525 } 12526 12527 static void attach_task_cfs_rq(struct task_struct *p) 12528 { 12529 struct sched_entity *se = &p->se; 12530 12531 attach_entity_cfs_rq(se); 12532 } 12533 12534 static void switched_from_fair(struct rq *rq, struct task_struct *p) 12535 { 12536 detach_task_cfs_rq(p); 12537 } 12538 12539 static void switched_to_fair(struct rq *rq, struct task_struct *p) 12540 { 12541 attach_task_cfs_rq(p); 12542 12543 if (task_on_rq_queued(p)) { 12544 /* 12545 * We were most likely switched from sched_rt, so 12546 * kick off the schedule if running, otherwise just see 12547 * if we can still preempt the current task. 12548 */ 12549 if (task_current(rq, p)) 12550 resched_curr(rq); 12551 else 12552 check_preempt_curr(rq, p, 0); 12553 } 12554 } 12555 12556 /* Account for a task changing its policy or group. 12557 * 12558 * This routine is mostly called to set cfs_rq->curr field when a task 12559 * migrates between groups/classes. 12560 */ 12561 static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) 12562 { 12563 struct sched_entity *se = &p->se; 12564 12565 #ifdef CONFIG_SMP 12566 if (task_on_rq_queued(p)) { 12567 /* 12568 * Move the next running task to the front of the list, so our 12569 * cfs_tasks list becomes MRU one. 12570 */ 12571 list_move(&se->group_node, &rq->cfs_tasks); 12572 } 12573 #endif 12574 12575 for_each_sched_entity(se) { 12576 struct cfs_rq *cfs_rq = cfs_rq_of(se); 12577 12578 set_next_entity(cfs_rq, se); 12579 /* ensure bandwidth has been allocated on our new cfs_rq */ 12580 account_cfs_rq_runtime(cfs_rq, 0); 12581 } 12582 } 12583 12584 void init_cfs_rq(struct cfs_rq *cfs_rq) 12585 { 12586 cfs_rq->tasks_timeline = RB_ROOT_CACHED; 12587 u64_u32_store(cfs_rq->min_vruntime, (u64)(-(1LL << 20))); 12588 #ifdef CONFIG_SMP 12589 raw_spin_lock_init(&cfs_rq->removed.lock); 12590 #endif 12591 } 12592 12593 #ifdef CONFIG_FAIR_GROUP_SCHED 12594 static void task_change_group_fair(struct task_struct *p) 12595 { 12596 /* 12597 * We couldn't detach or attach a forked task which 12598 * hasn't been woken up by wake_up_new_task(). 12599 */ 12600 if (READ_ONCE(p->__state) == TASK_NEW) 12601 return; 12602 12603 detach_task_cfs_rq(p); 12604 12605 #ifdef CONFIG_SMP 12606 /* Tell se's cfs_rq has been changed -- migrated */ 12607 p->se.avg.last_update_time = 0; 12608 #endif 12609 set_task_rq(p, task_cpu(p)); 12610 attach_task_cfs_rq(p); 12611 } 12612 12613 void free_fair_sched_group(struct task_group *tg) 12614 { 12615 int i; 12616 12617 for_each_possible_cpu(i) { 12618 if (tg->cfs_rq) 12619 kfree(tg->cfs_rq[i]); 12620 if (tg->se) 12621 kfree(tg->se[i]); 12622 } 12623 12624 kfree(tg->cfs_rq); 12625 kfree(tg->se); 12626 } 12627 12628 int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) 12629 { 12630 struct sched_entity *se; 12631 struct cfs_rq *cfs_rq; 12632 int i; 12633 12634 tg->cfs_rq = kcalloc(nr_cpu_ids, sizeof(cfs_rq), GFP_KERNEL); 12635 if (!tg->cfs_rq) 12636 goto err; 12637 tg->se = kcalloc(nr_cpu_ids, sizeof(se), GFP_KERNEL); 12638 if (!tg->se) 12639 goto err; 12640 12641 tg->shares = NICE_0_LOAD; 12642 12643 init_cfs_bandwidth(tg_cfs_bandwidth(tg), tg_cfs_bandwidth(parent)); 12644 12645 for_each_possible_cpu(i) { 12646 cfs_rq = kzalloc_node(sizeof(struct cfs_rq), 12647 GFP_KERNEL, cpu_to_node(i)); 12648 if (!cfs_rq) 12649 goto err; 12650 12651 se = kzalloc_node(sizeof(struct sched_entity_stats), 12652 GFP_KERNEL, cpu_to_node(i)); 12653 if (!se) 12654 goto err_free_rq; 12655 12656 init_cfs_rq(cfs_rq); 12657 init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]); 12658 init_entity_runnable_average(se); 12659 } 12660 12661 return 1; 12662 12663 err_free_rq: 12664 kfree(cfs_rq); 12665 err: 12666 return 0; 12667 } 12668 12669 void online_fair_sched_group(struct task_group *tg) 12670 { 12671 struct sched_entity *se; 12672 struct rq_flags rf; 12673 struct rq *rq; 12674 int i; 12675 12676 for_each_possible_cpu(i) { 12677 rq = cpu_rq(i); 12678 se = tg->se[i]; 12679 rq_lock_irq(rq, &rf); 12680 update_rq_clock(rq); 12681 attach_entity_cfs_rq(se); 12682 sync_throttle(tg, i); 12683 rq_unlock_irq(rq, &rf); 12684 } 12685 } 12686 12687 void unregister_fair_sched_group(struct task_group *tg) 12688 { 12689 unsigned long flags; 12690 struct rq *rq; 12691 int cpu; 12692 12693 destroy_cfs_bandwidth(tg_cfs_bandwidth(tg)); 12694 12695 for_each_possible_cpu(cpu) { 12696 if (tg->se[cpu]) 12697 remove_entity_load_avg(tg->se[cpu]); 12698 12699 /* 12700 * Only empty task groups can be destroyed; so we can speculatively 12701 * check on_list without danger of it being re-added. 12702 */ 12703 if (!tg->cfs_rq[cpu]->on_list) 12704 continue; 12705 12706 rq = cpu_rq(cpu); 12707 12708 raw_spin_rq_lock_irqsave(rq, flags); 12709 list_del_leaf_cfs_rq(tg->cfs_rq[cpu]); 12710 raw_spin_rq_unlock_irqrestore(rq, flags); 12711 } 12712 } 12713 12714 void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq, 12715 struct sched_entity *se, int cpu, 12716 struct sched_entity *parent) 12717 { 12718 struct rq *rq = cpu_rq(cpu); 12719 12720 cfs_rq->tg = tg; 12721 cfs_rq->rq = rq; 12722 init_cfs_rq_runtime(cfs_rq); 12723 12724 tg->cfs_rq[cpu] = cfs_rq; 12725 tg->se[cpu] = se; 12726 12727 /* se could be NULL for root_task_group */ 12728 if (!se) 12729 return; 12730 12731 if (!parent) { 12732 se->cfs_rq = &rq->cfs; 12733 se->depth = 0; 12734 } else { 12735 se->cfs_rq = parent->my_q; 12736 se->depth = parent->depth + 1; 12737 } 12738 12739 se->my_q = cfs_rq; 12740 /* guarantee group entities always have weight */ 12741 update_load_set(&se->load, NICE_0_LOAD); 12742 se->parent = parent; 12743 } 12744 12745 static DEFINE_MUTEX(shares_mutex); 12746 12747 static int __sched_group_set_shares(struct task_group *tg, unsigned long shares) 12748 { 12749 int i; 12750 12751 lockdep_assert_held(&shares_mutex); 12752 12753 /* 12754 * We can't change the weight of the root cgroup. 12755 */ 12756 if (!tg->se[0]) 12757 return -EINVAL; 12758 12759 shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES)); 12760 12761 if (tg->shares == shares) 12762 return 0; 12763 12764 tg->shares = shares; 12765 for_each_possible_cpu(i) { 12766 struct rq *rq = cpu_rq(i); 12767 struct sched_entity *se = tg->se[i]; 12768 struct rq_flags rf; 12769 12770 /* Propagate contribution to hierarchy */ 12771 rq_lock_irqsave(rq, &rf); 12772 update_rq_clock(rq); 12773 for_each_sched_entity(se) { 12774 update_load_avg(cfs_rq_of(se), se, UPDATE_TG); 12775 update_cfs_group(se); 12776 } 12777 rq_unlock_irqrestore(rq, &rf); 12778 } 12779 12780 return 0; 12781 } 12782 12783 int sched_group_set_shares(struct task_group *tg, unsigned long shares) 12784 { 12785 int ret; 12786 12787 mutex_lock(&shares_mutex); 12788 if (tg_is_idle(tg)) 12789 ret = -EINVAL; 12790 else 12791 ret = __sched_group_set_shares(tg, shares); 12792 mutex_unlock(&shares_mutex); 12793 12794 return ret; 12795 } 12796 12797 int sched_group_set_idle(struct task_group *tg, long idle) 12798 { 12799 int i; 12800 12801 if (tg == &root_task_group) 12802 return -EINVAL; 12803 12804 if (idle < 0 || idle > 1) 12805 return -EINVAL; 12806 12807 mutex_lock(&shares_mutex); 12808 12809 if (tg->idle == idle) { 12810 mutex_unlock(&shares_mutex); 12811 return 0; 12812 } 12813 12814 tg->idle = idle; 12815 12816 for_each_possible_cpu(i) { 12817 struct rq *rq = cpu_rq(i); 12818 struct sched_entity *se = tg->se[i]; 12819 struct cfs_rq *parent_cfs_rq, *grp_cfs_rq = tg->cfs_rq[i]; 12820 bool was_idle = cfs_rq_is_idle(grp_cfs_rq); 12821 long idle_task_delta; 12822 struct rq_flags rf; 12823 12824 rq_lock_irqsave(rq, &rf); 12825 12826 grp_cfs_rq->idle = idle; 12827 if (WARN_ON_ONCE(was_idle == cfs_rq_is_idle(grp_cfs_rq))) 12828 goto next_cpu; 12829 12830 if (se->on_rq) { 12831 parent_cfs_rq = cfs_rq_of(se); 12832 if (cfs_rq_is_idle(grp_cfs_rq)) 12833 parent_cfs_rq->idle_nr_running++; 12834 else 12835 parent_cfs_rq->idle_nr_running--; 12836 } 12837 12838 idle_task_delta = grp_cfs_rq->h_nr_running - 12839 grp_cfs_rq->idle_h_nr_running; 12840 if (!cfs_rq_is_idle(grp_cfs_rq)) 12841 idle_task_delta *= -1; 12842 12843 for_each_sched_entity(se) { 12844 struct cfs_rq *cfs_rq = cfs_rq_of(se); 12845 12846 if (!se->on_rq) 12847 break; 12848 12849 cfs_rq->idle_h_nr_running += idle_task_delta; 12850 12851 /* Already accounted at parent level and above. */ 12852 if (cfs_rq_is_idle(cfs_rq)) 12853 break; 12854 } 12855 12856 next_cpu: 12857 rq_unlock_irqrestore(rq, &rf); 12858 } 12859 12860 /* Idle groups have minimum weight. */ 12861 if (tg_is_idle(tg)) 12862 __sched_group_set_shares(tg, scale_load(WEIGHT_IDLEPRIO)); 12863 else 12864 __sched_group_set_shares(tg, NICE_0_LOAD); 12865 12866 mutex_unlock(&shares_mutex); 12867 return 0; 12868 } 12869 12870 #else /* CONFIG_FAIR_GROUP_SCHED */ 12871 12872 void free_fair_sched_group(struct task_group *tg) { } 12873 12874 int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) 12875 { 12876 return 1; 12877 } 12878 12879 void online_fair_sched_group(struct task_group *tg) { } 12880 12881 void unregister_fair_sched_group(struct task_group *tg) { } 12882 12883 #endif /* CONFIG_FAIR_GROUP_SCHED */ 12884 12885 12886 static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task) 12887 { 12888 struct sched_entity *se = &task->se; 12889 unsigned int rr_interval = 0; 12890 12891 /* 12892 * Time slice is 0 for SCHED_OTHER tasks that are on an otherwise 12893 * idle runqueue: 12894 */ 12895 if (rq->cfs.load.weight) 12896 rr_interval = NS_TO_JIFFIES(se->slice); 12897 12898 return rr_interval; 12899 } 12900 12901 /* 12902 * All the scheduling class methods: 12903 */ 12904 DEFINE_SCHED_CLASS(fair) = { 12905 12906 .enqueue_task = enqueue_task_fair, 12907 .dequeue_task = dequeue_task_fair, 12908 .yield_task = yield_task_fair, 12909 .yield_to_task = yield_to_task_fair, 12910 12911 .check_preempt_curr = check_preempt_wakeup, 12912 12913 .pick_next_task = __pick_next_task_fair, 12914 .put_prev_task = put_prev_task_fair, 12915 .set_next_task = set_next_task_fair, 12916 12917 #ifdef CONFIG_SMP 12918 .balance = balance_fair, 12919 .pick_task = pick_task_fair, 12920 .select_task_rq = select_task_rq_fair, 12921 .migrate_task_rq = migrate_task_rq_fair, 12922 12923 .rq_online = rq_online_fair, 12924 .rq_offline = rq_offline_fair, 12925 12926 .task_dead = task_dead_fair, 12927 .set_cpus_allowed = set_cpus_allowed_common, 12928 #endif 12929 12930 .task_tick = task_tick_fair, 12931 .task_fork = task_fork_fair, 12932 12933 .prio_changed = prio_changed_fair, 12934 .switched_from = switched_from_fair, 12935 .switched_to = switched_to_fair, 12936 12937 .get_rr_interval = get_rr_interval_fair, 12938 12939 .update_curr = update_curr_fair, 12940 12941 #ifdef CONFIG_FAIR_GROUP_SCHED 12942 .task_change_group = task_change_group_fair, 12943 #endif 12944 12945 #ifdef CONFIG_SCHED_CORE 12946 .task_is_throttled = task_is_throttled_fair, 12947 #endif 12948 12949 #ifdef CONFIG_UCLAMP_TASK 12950 .uclamp_enabled = 1, 12951 #endif 12952 }; 12953 12954 #ifdef CONFIG_SCHED_DEBUG 12955 void print_cfs_stats(struct seq_file *m, int cpu) 12956 { 12957 struct cfs_rq *cfs_rq, *pos; 12958 12959 rcu_read_lock(); 12960 for_each_leaf_cfs_rq_safe(cpu_rq(cpu), cfs_rq, pos) 12961 print_cfs_rq(m, cpu, cfs_rq); 12962 rcu_read_unlock(); 12963 } 12964 12965 #ifdef CONFIG_NUMA_BALANCING 12966 void show_numa_stats(struct task_struct *p, struct seq_file *m) 12967 { 12968 int node; 12969 unsigned long tsf = 0, tpf = 0, gsf = 0, gpf = 0; 12970 struct numa_group *ng; 12971 12972 rcu_read_lock(); 12973 ng = rcu_dereference(p->numa_group); 12974 for_each_online_node(node) { 12975 if (p->numa_faults) { 12976 tsf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 0)]; 12977 tpf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 1)]; 12978 } 12979 if (ng) { 12980 gsf = ng->faults[task_faults_idx(NUMA_MEM, node, 0)], 12981 gpf = ng->faults[task_faults_idx(NUMA_MEM, node, 1)]; 12982 } 12983 print_numa_stats(m, node, tsf, tpf, gsf, gpf); 12984 } 12985 rcu_read_unlock(); 12986 } 12987 #endif /* CONFIG_NUMA_BALANCING */ 12988 #endif /* CONFIG_SCHED_DEBUG */ 12989 12990 __init void init_sched_fair_class(void) 12991 { 12992 #ifdef CONFIG_SMP 12993 int i; 12994 12995 for_each_possible_cpu(i) { 12996 zalloc_cpumask_var_node(&per_cpu(load_balance_mask, i), GFP_KERNEL, cpu_to_node(i)); 12997 zalloc_cpumask_var_node(&per_cpu(select_rq_mask, i), GFP_KERNEL, cpu_to_node(i)); 12998 zalloc_cpumask_var_node(&per_cpu(should_we_balance_tmpmask, i), 12999 GFP_KERNEL, cpu_to_node(i)); 13000 13001 #ifdef CONFIG_CFS_BANDWIDTH 13002 INIT_CSD(&cpu_rq(i)->cfsb_csd, __cfsb_csd_unthrottle, cpu_rq(i)); 13003 INIT_LIST_HEAD(&cpu_rq(i)->cfsb_csd_list); 13004 #endif 13005 } 13006 13007 open_softirq(SCHED_SOFTIRQ, run_rebalance_domains); 13008 13009 #ifdef CONFIG_NO_HZ_COMMON 13010 nohz.next_balance = jiffies; 13011 nohz.next_blocked = jiffies; 13012 zalloc_cpumask_var(&nohz.idle_cpus_mask, GFP_NOWAIT); 13013 #endif 13014 #endif /* SMP */ 13015 13016 }