개념 설명 전체 · v6.18.37 / mm/page_alloc.c
1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/mm/page_alloc.c 4 * 5 * Manages the free list, the system allocates free pages here. 6 * Note that kmalloc() lives in slab.c 7 * 8 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 9 * Swap reorganised 29.12.95, Stephen Tweedie 10 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 11 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 12 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 13 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 14 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 15 * (lots of bits borrowed from Ingo Molnar & Andrew Morton) 16 */ 17 18 #include <linux/stddef.h> 19 #include <linux/mm.h> 20 #include <linux/highmem.h> 21 #include <linux/interrupt.h> 22 #include <linux/jiffies.h> 23 #include <linux/compiler.h> 24 #include <linux/kernel.h> 25 #include <linux/kasan.h> 26 #include <linux/kmsan.h> 27 #include <linux/module.h> 28 #include <linux/suspend.h> 29 #include <linux/ratelimit.h> 30 #include <linux/oom.h> 31 #include <linux/topology.h> 32 #include <linux/sysctl.h> 33 #include <linux/cpu.h> 34 #include <linux/cpuset.h> 35 #include <linux/pagevec.h> 36 #include <linux/memory_hotplug.h> 37 #include <linux/nodemask.h> 38 #include <linux/vmstat.h> 39 #include <linux/fault-inject.h> 40 #include <linux/compaction.h> 41 #include <trace/events/kmem.h> 42 #include <trace/events/oom.h> 43 #include <linux/prefetch.h> 44 #include <linux/mm_inline.h> 45 #include <linux/mmu_notifier.h> 46 #include <linux/migrate.h> 47 #include <linux/sched/mm.h> 48 #include <linux/page_owner.h> 49 #include <linux/page_table_check.h> 50 #include <linux/memcontrol.h> 51 #include <linux/ftrace.h> 52 #include <linux/lockdep.h> 53 #include <linux/psi.h> 54 #include <linux/khugepaged.h> 55 #include <linux/delayacct.h> 56 #include <linux/cacheinfo.h> 57 #include <linux/pgalloc_tag.h> 58 #include <asm/div64.h> 59 #include "internal.h" 60 #include "shuffle.h" 61 #include "page_reporting.h" 62 63 /* Free Page Internal flags: for internal, non-pcp variants of free_pages(). */ 64 typedef int __bitwise fpi_t; 65 66 /* No special request */ 67 #define FPI_NONE ((__force fpi_t)0) 68 69 /* 70 * Skip free page reporting notification for the (possibly merged) page. 71 * This does not hinder free page reporting from grabbing the page, 72 * reporting it and marking it "reported" - it only skips notifying 73 * the free page reporting infrastructure about a newly freed page. For 74 * example, used when temporarily pulling a page from a freelist and 75 * putting it back unmodified. 76 */ 77 #define FPI_SKIP_REPORT_NOTIFY ((__force fpi_t)BIT(0)) 78 79 /* 80 * Place the (possibly merged) page to the tail of the freelist. Will ignore 81 * page shuffling (relevant code - e.g., memory onlining - is expected to 82 * shuffle the whole zone). 83 * 84 * Note: No code should rely on this flag for correctness - it's purely 85 * to allow for optimizations when handing back either fresh pages 86 * (memory onlining) or untouched pages (page isolation, free page 87 * reporting). 88 */ 89 #define FPI_TO_TAIL ((__force fpi_t)BIT(1)) 90 91 /* Free the page without taking locks. Rely on trylock only. */ 92 #define FPI_TRYLOCK ((__force fpi_t)BIT(2)) 93 94 /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */ 95 static DEFINE_MUTEX(pcp_batch_high_lock); 96 #define MIN_PERCPU_PAGELIST_HIGH_FRACTION (8) 97 98 #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT) 99 /* 100 * On SMP, spin_trylock is sufficient protection. 101 * On PREEMPT_RT, spin_trylock is equivalent on both SMP and UP. 102 */ 103 #define pcp_trylock_prepare(flags) do { } while (0) 104 #define pcp_trylock_finish(flag) do { } while (0) 105 #else 106 107 /* UP spin_trylock always succeeds so disable IRQs to prevent re-entrancy. */ 108 #define pcp_trylock_prepare(flags) local_irq_save(flags) 109 #define pcp_trylock_finish(flags) local_irq_restore(flags) 110 #endif 111 112 /* 113 * Locking a pcp requires a PCP lookup followed by a spinlock. To avoid 114 * a migration causing the wrong PCP to be locked and remote memory being 115 * potentially allocated, pin the task to the CPU for the lookup+lock. 116 * preempt_disable is used on !RT because it is faster than migrate_disable. 117 * migrate_disable is used on RT because otherwise RT spinlock usage is 118 * interfered with and a high priority task cannot preempt the allocator. 119 */ 120 #ifndef CONFIG_PREEMPT_RT 121 #define pcpu_task_pin() preempt_disable() 122 #define pcpu_task_unpin() preempt_enable() 123 #else 124 #define pcpu_task_pin() migrate_disable() 125 #define pcpu_task_unpin() migrate_enable() 126 #endif 127 128 /* 129 * Generic helper to lookup and a per-cpu variable with an embedded spinlock. 130 * Return value should be used with equivalent unlock helper. 131 */ 132 #define pcpu_spin_lock(type, member, ptr) \ 133 ({ \ 134 type *_ret; \ 135 pcpu_task_pin(); \ 136 _ret = this_cpu_ptr(ptr); \ 137 spin_lock(&_ret->member); \ 138 _ret; \ 139 }) 140 141 #define pcpu_spin_trylock(type, member, ptr) \ 142 ({ \ 143 type *_ret; \ 144 pcpu_task_pin(); \ 145 _ret = this_cpu_ptr(ptr); \ 146 if (!spin_trylock(&_ret->member)) { \ 147 pcpu_task_unpin(); \ 148 _ret = NULL; \ 149 } \ 150 _ret; \ 151 }) 152 153 #define pcpu_spin_unlock(member, ptr) \ 154 ({ \ 155 spin_unlock(&ptr->member); \ 156 pcpu_task_unpin(); \ 157 }) 158 159 /* struct per_cpu_pages specific helpers. */ 160 #define pcp_spin_lock(ptr) \ 161 pcpu_spin_lock(struct per_cpu_pages, lock, ptr) 162 163 #define pcp_spin_trylock(ptr) \ 164 pcpu_spin_trylock(struct per_cpu_pages, lock, ptr) 165 166 #define pcp_spin_unlock(ptr) \ 167 pcpu_spin_unlock(lock, ptr) 168 169 /* 170 * With the UP spinlock implementation, when we spin_lock(&pcp->lock) (for i.e. 171 * a potentially remote cpu drain) and get interrupted by an operation that 172 * attempts pcp_spin_trylock(), we can't rely on the trylock failure due to UP 173 * spinlock assumptions making the trylock a no-op. So we have to turn that 174 * spin_lock() to a spin_lock_irqsave(). This works because on UP there are no 175 * remote cpu's so we can only be locking the only existing local one. 176 */ 177 #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT) 178 static inline void __flags_noop(unsigned long *flags) { } 179 #define pcp_spin_lock_maybe_irqsave(ptr, flags) \ 180 ({ \ 181 __flags_noop(&(flags)); \ 182 spin_lock(&(ptr)->lock); \ 183 }) 184 #define pcp_spin_unlock_maybe_irqrestore(ptr, flags) \ 185 ({ \ 186 spin_unlock(&(ptr)->lock); \ 187 __flags_noop(&(flags)); \ 188 }) 189 #else 190 #define pcp_spin_lock_maybe_irqsave(ptr, flags) \ 191 spin_lock_irqsave(&(ptr)->lock, flags) 192 #define pcp_spin_unlock_maybe_irqrestore(ptr, flags) \ 193 spin_unlock_irqrestore(&(ptr)->lock, flags) 194 #endif 195 196 #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID 197 DEFINE_PER_CPU(int, numa_node); 198 EXPORT_PER_CPU_SYMBOL(numa_node); 199 #endif 200 201 DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key); 202 203 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 204 /* 205 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. 206 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined. 207 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem() 208 * defined in <linux/topology.h>. 209 */ 210 DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ 211 EXPORT_PER_CPU_SYMBOL(_numa_mem_); 212 #endif 213 214 static DEFINE_MUTEX(pcpu_drain_mutex); 215 216 #ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY 217 volatile unsigned long latent_entropy __latent_entropy; 218 EXPORT_SYMBOL(latent_entropy); 219 #endif 220 221 /* 222 * Array of node states. 223 */ 224 nodemask_t node_states[NR_NODE_STATES] __read_mostly = { 225 [N_POSSIBLE] = NODE_MASK_ALL, 226 [N_ONLINE] = { { [0] = 1UL } }, 227 #ifndef CONFIG_NUMA 228 [N_NORMAL_MEMORY] = { { [0] = 1UL } }, 229 #ifdef CONFIG_HIGHMEM 230 [N_HIGH_MEMORY] = { { [0] = 1UL } }, 231 #endif 232 [N_MEMORY] = { { [0] = 1UL } }, 233 [N_CPU] = { { [0] = 1UL } }, 234 #endif /* NUMA */ 235 }; 236 EXPORT_SYMBOL(node_states); 237 238 gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; 239 240 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 241 unsigned int pageblock_order __read_mostly; 242 #endif 243 244 static void __free_pages_ok(struct page *page, unsigned int order, 245 fpi_t fpi_flags); 246 247 /* 248 * results with 256, 32 in the lowmem_reserve sysctl: 249 * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) 250 * 1G machine -> (16M dma, 784M normal, 224M high) 251 * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA 252 * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL 253 * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA 254 * 255 * TBD: should special case ZONE_DMA32 machines here - in those we normally 256 * don't need any ZONE_NORMAL reservation 257 */ 258 static int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = { 259 #ifdef CONFIG_ZONE_DMA 260 [ZONE_DMA] = 256, 261 #endif 262 #ifdef CONFIG_ZONE_DMA32 263 [ZONE_DMA32] = 256, 264 #endif 265 [ZONE_NORMAL] = 32, 266 #ifdef CONFIG_HIGHMEM 267 [ZONE_HIGHMEM] = 0, 268 #endif 269 [ZONE_MOVABLE] = 0, 270 }; 271 272 char * const zone_names[MAX_NR_ZONES] = { 273 #ifdef CONFIG_ZONE_DMA 274 "DMA", 275 #endif 276 #ifdef CONFIG_ZONE_DMA32 277 "DMA32", 278 #endif 279 "Normal", 280 #ifdef CONFIG_HIGHMEM 281 "HighMem", 282 #endif 283 "Movable", 284 #ifdef CONFIG_ZONE_DEVICE 285 "Device", 286 #endif 287 }; 288 289 const char * const migratetype_names[MIGRATE_TYPES] = { 290 "Unmovable", 291 "Movable", 292 "Reclaimable", 293 "HighAtomic", 294 #ifdef CONFIG_CMA 295 "CMA", 296 #endif 297 #ifdef CONFIG_MEMORY_ISOLATION 298 "Isolate", 299 #endif 300 }; 301 302 int min_free_kbytes = 1024; 303 int user_min_free_kbytes = -1; 304 static int watermark_boost_factor __read_mostly = 15000; 305 static int watermark_scale_factor = 10; 306 int defrag_mode; 307 308 /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */ 309 int movable_zone; 310 EXPORT_SYMBOL(movable_zone); 311 312 #if MAX_NUMNODES > 1 313 unsigned int nr_node_ids __read_mostly = MAX_NUMNODES; 314 unsigned int nr_online_nodes __read_mostly = 1; 315 EXPORT_SYMBOL(nr_node_ids); 316 EXPORT_SYMBOL(nr_online_nodes); 317 #endif 318 319 static bool page_contains_unaccepted(struct page *page, unsigned int order); 320 static bool cond_accept_memory(struct zone *zone, unsigned int order, 321 int alloc_flags); 322 static bool __free_unaccepted(struct page *page); 323 324 int page_group_by_mobility_disabled __read_mostly; 325 326 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 327 /* 328 * During boot we initialize deferred pages on-demand, as needed, but once 329 * page_alloc_init_late() has finished, the deferred pages are all initialized, 330 * and we can permanently disable that path. 331 */ 332 DEFINE_STATIC_KEY_TRUE(deferred_pages); 333 334 static inline bool deferred_pages_enabled(void) 335 { 336 return static_branch_unlikely(&deferred_pages); 337 } 338 339 /* 340 * deferred_grow_zone() is __init, but it is called from 341 * get_page_from_freelist() during early boot until deferred_pages permanently 342 * disables this call. This is why we have refdata wrapper to avoid warning, 343 * and to ensure that the function body gets unloaded. 344 */ 345 static bool __ref 346 _deferred_grow_zone(struct zone *zone, unsigned int order) 347 { 348 return deferred_grow_zone(zone, order); 349 } 350 #else 351 static inline bool deferred_pages_enabled(void) 352 { 353 return false; 354 } 355 356 static inline bool _deferred_grow_zone(struct zone *zone, unsigned int order) 357 { 358 return false; 359 } 360 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 361 362 /* Return a pointer to the bitmap storing bits affecting a block of pages */ 363 static inline unsigned long *get_pageblock_bitmap(const struct page *page, 364 unsigned long pfn) 365 { 366 #ifdef CONFIG_SPARSEMEM 367 return section_to_usemap(__pfn_to_section(pfn)); 368 #else 369 return page_zone(page)->pageblock_flags; 370 #endif /* CONFIG_SPARSEMEM */ 371 } 372 373 static inline int pfn_to_bitidx(const struct page *page, unsigned long pfn) 374 { 375 #ifdef CONFIG_SPARSEMEM 376 pfn &= (PAGES_PER_SECTION-1); 377 #else 378 pfn = pfn - pageblock_start_pfn(page_zone(page)->zone_start_pfn); 379 #endif /* CONFIG_SPARSEMEM */ 380 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 381 } 382 383 static __always_inline bool is_standalone_pb_bit(enum pageblock_bits pb_bit) 384 { 385 return pb_bit >= PB_compact_skip && pb_bit < __NR_PAGEBLOCK_BITS; 386 } 387 388 static __always_inline void 389 get_pfnblock_bitmap_bitidx(const struct page *page, unsigned long pfn, 390 unsigned long **bitmap_word, unsigned long *bitidx) 391 { 392 unsigned long *bitmap; 393 unsigned long word_bitidx; 394 395 #ifdef CONFIG_MEMORY_ISOLATION 396 BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 8); 397 #else 398 BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4); 399 #endif 400 BUILD_BUG_ON(__MIGRATE_TYPE_END > MIGRATETYPE_MASK); 401 VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page); 402 403 bitmap = get_pageblock_bitmap(page, pfn); 404 *bitidx = pfn_to_bitidx(page, pfn); 405 word_bitidx = *bitidx / BITS_PER_LONG; 406 *bitidx &= (BITS_PER_LONG - 1); 407 *bitmap_word = &bitmap[word_bitidx]; 408 } 409 410 411 /** 412 * __get_pfnblock_flags_mask - Return the requested group of flags for 413 * a pageblock_nr_pages block of pages 414 * @page: The page within the block of interest 415 * @pfn: The target page frame number 416 * @mask: mask of bits that the caller is interested in 417 * 418 * Return: pageblock_bits flags 419 */ 420 static unsigned long __get_pfnblock_flags_mask(const struct page *page, 421 unsigned long pfn, 422 unsigned long mask) 423 { 424 unsigned long *bitmap_word; 425 unsigned long bitidx; 426 unsigned long word; 427 428 get_pfnblock_bitmap_bitidx(page, pfn, &bitmap_word, &bitidx); 429 /* 430 * This races, without locks, with set_pfnblock_migratetype(). Ensure 431 * a consistent read of the memory array, so that results, even though 432 * racy, are not corrupted. 433 */ 434 word = READ_ONCE(*bitmap_word); 435 return (word >> bitidx) & mask; 436 } 437 438 /** 439 * get_pfnblock_bit - Check if a standalone bit of a pageblock is set 440 * @page: The page within the block of interest 441 * @pfn: The target page frame number 442 * @pb_bit: pageblock bit to check 443 * 444 * Return: true if the bit is set, otherwise false 445 */ 446 bool get_pfnblock_bit(const struct page *page, unsigned long pfn, 447 enum pageblock_bits pb_bit) 448 { 449 unsigned long *bitmap_word; 450 unsigned long bitidx; 451 452 if (WARN_ON_ONCE(!is_standalone_pb_bit(pb_bit))) 453 return false; 454 455 get_pfnblock_bitmap_bitidx(page, pfn, &bitmap_word, &bitidx); 456 457 return test_bit(bitidx + pb_bit, bitmap_word); 458 } 459 460 /** 461 * get_pfnblock_migratetype - Return the migratetype of a pageblock 462 * @page: The page within the block of interest 463 * @pfn: The target page frame number 464 * 465 * Return: The migratetype of the pageblock 466 * 467 * Use get_pfnblock_migratetype() if caller already has both @page and @pfn 468 * to save a call to page_to_pfn(). 469 */ 470 __always_inline enum migratetype 471 get_pfnblock_migratetype(const struct page *page, unsigned long pfn) 472 { 473 unsigned long mask = MIGRATETYPE_AND_ISO_MASK; 474 unsigned long flags; 475 476 flags = __get_pfnblock_flags_mask(page, pfn, mask); 477 478 #ifdef CONFIG_MEMORY_ISOLATION 479 if (flags & BIT(PB_migrate_isolate)) 480 return MIGRATE_ISOLATE; 481 #endif 482 return flags & MIGRATETYPE_MASK; 483 } 484 485 /** 486 * __set_pfnblock_flags_mask - Set the requested group of flags for 487 * a pageblock_nr_pages block of pages 488 * @page: The page within the block of interest 489 * @pfn: The target page frame number 490 * @flags: The flags to set 491 * @mask: mask of bits that the caller is interested in 492 */ 493 static void __set_pfnblock_flags_mask(struct page *page, unsigned long pfn, 494 unsigned long flags, unsigned long mask) 495 { 496 unsigned long *bitmap_word; 497 unsigned long bitidx; 498 unsigned long word; 499 500 get_pfnblock_bitmap_bitidx(page, pfn, &bitmap_word, &bitidx); 501 502 mask <<= bitidx; 503 flags <<= bitidx; 504 505 word = READ_ONCE(*bitmap_word); 506 do { 507 } while (!try_cmpxchg(bitmap_word, &word, (word & ~mask) | flags)); 508 } 509 510 /** 511 * set_pfnblock_bit - Set a standalone bit of a pageblock 512 * @page: The page within the block of interest 513 * @pfn: The target page frame number 514 * @pb_bit: pageblock bit to set 515 */ 516 void set_pfnblock_bit(const struct page *page, unsigned long pfn, 517 enum pageblock_bits pb_bit) 518 { 519 unsigned long *bitmap_word; 520 unsigned long bitidx; 521 522 if (WARN_ON_ONCE(!is_standalone_pb_bit(pb_bit))) 523 return; 524 525 get_pfnblock_bitmap_bitidx(page, pfn, &bitmap_word, &bitidx); 526 527 set_bit(bitidx + pb_bit, bitmap_word); 528 } 529 530 /** 531 * clear_pfnblock_bit - Clear a standalone bit of a pageblock 532 * @page: The page within the block of interest 533 * @pfn: The target page frame number 534 * @pb_bit: pageblock bit to clear 535 */ 536 void clear_pfnblock_bit(const struct page *page, unsigned long pfn, 537 enum pageblock_bits pb_bit) 538 { 539 unsigned long *bitmap_word; 540 unsigned long bitidx; 541 542 if (WARN_ON_ONCE(!is_standalone_pb_bit(pb_bit))) 543 return; 544 545 get_pfnblock_bitmap_bitidx(page, pfn, &bitmap_word, &bitidx); 546 547 clear_bit(bitidx + pb_bit, bitmap_word); 548 } 549 550 /** 551 * set_pageblock_migratetype - Set the migratetype of a pageblock 552 * @page: The page within the block of interest 553 * @migratetype: migratetype to set 554 */ 555 static void set_pageblock_migratetype(struct page *page, 556 enum migratetype migratetype) 557 { 558 if (unlikely(page_group_by_mobility_disabled && 559 migratetype < MIGRATE_PCPTYPES)) 560 migratetype = MIGRATE_UNMOVABLE; 561 562 #ifdef CONFIG_MEMORY_ISOLATION 563 if (migratetype == MIGRATE_ISOLATE) { 564 VM_WARN_ONCE(1, 565 "Use set_pageblock_isolate() for pageblock isolation"); 566 return; 567 } 568 VM_WARN_ONCE(get_pageblock_isolate(page), 569 "Use clear_pageblock_isolate() to unisolate pageblock"); 570 /* MIGRATETYPE_AND_ISO_MASK clears PB_migrate_isolate if it is set */ 571 #endif 572 __set_pfnblock_flags_mask(page, page_to_pfn(page), 573 (unsigned long)migratetype, 574 MIGRATETYPE_AND_ISO_MASK); 575 } 576 577 void __meminit init_pageblock_migratetype(struct page *page, 578 enum migratetype migratetype, 579 bool isolate) 580 { 581 unsigned long flags; 582 583 if (unlikely(page_group_by_mobility_disabled && 584 migratetype < MIGRATE_PCPTYPES)) 585 migratetype = MIGRATE_UNMOVABLE; 586 587 flags = migratetype; 588 589 #ifdef CONFIG_MEMORY_ISOLATION 590 if (migratetype == MIGRATE_ISOLATE) { 591 VM_WARN_ONCE( 592 1, 593 "Set isolate=true to isolate pageblock with a migratetype"); 594 return; 595 } 596 if (isolate) 597 flags |= BIT(PB_migrate_isolate); 598 #endif 599 __set_pfnblock_flags_mask(page, page_to_pfn(page), flags, 600 MIGRATETYPE_AND_ISO_MASK); 601 } 602 603 #ifdef CONFIG_DEBUG_VM 604 static int page_outside_zone_boundaries(struct zone *zone, struct page *page) 605 { 606 int ret; 607 unsigned seq; 608 unsigned long pfn = page_to_pfn(page); 609 unsigned long sp, start_pfn; 610 611 do { 612 seq = zone_span_seqbegin(zone); 613 start_pfn = zone->zone_start_pfn; 614 sp = zone->spanned_pages; 615 ret = !zone_spans_pfn(zone, pfn); 616 } while (zone_span_seqretry(zone, seq)); 617 618 if (ret) 619 pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n", 620 pfn, zone_to_nid(zone), zone->name, 621 start_pfn, start_pfn + sp); 622 623 return ret; 624 } 625 626 /* 627 * Temporary debugging check for pages not lying within a given zone. 628 */ 629 static bool __maybe_unused bad_range(struct zone *zone, struct page *page) 630 { 631 if (page_outside_zone_boundaries(zone, page)) 632 return true; 633 if (zone != page_zone(page)) 634 return true; 635 636 return false; 637 } 638 #else 639 static inline bool __maybe_unused bad_range(struct zone *zone, struct page *page) 640 { 641 return false; 642 } 643 #endif 644 645 static void bad_page(struct page *page, const char *reason) 646 { 647 static unsigned long resume; 648 static unsigned long nr_shown; 649 static unsigned long nr_unshown; 650 651 /* 652 * Allow a burst of 60 reports, then keep quiet for that minute; 653 * or allow a steady drip of one report per second. 654 */ 655 if (nr_shown == 60) { 656 if (time_before(jiffies, resume)) { 657 nr_unshown++; 658 goto out; 659 } 660 if (nr_unshown) { 661 pr_alert( 662 "BUG: Bad page state: %lu messages suppressed\n", 663 nr_unshown); 664 nr_unshown = 0; 665 } 666 nr_shown = 0; 667 } 668 if (nr_shown++ == 0) 669 resume = jiffies + 60 * HZ; 670 671 pr_alert("BUG: Bad page state in process %s pfn:%05lx\n", 672 current->comm, page_to_pfn(page)); 673 dump_page(page, reason); 674 675 print_modules(); 676 dump_stack(); 677 out: 678 /* Leave bad fields for debug, except PageBuddy could make trouble */ 679 if (PageBuddy(page)) 680 __ClearPageBuddy(page); 681 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 682 } 683 684 static inline unsigned int order_to_pindex(int migratetype, int order) 685 { 686 687 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 688 bool movable; 689 if (order > PAGE_ALLOC_COSTLY_ORDER) { 690 VM_BUG_ON(order != HPAGE_PMD_ORDER); 691 692 movable = migratetype == MIGRATE_MOVABLE; 693 694 return NR_LOWORDER_PCP_LISTS + movable; 695 } 696 #else 697 VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER); 698 #endif 699 700 return (MIGRATE_PCPTYPES * order) + migratetype; 701 } 702 703 static inline int pindex_to_order(unsigned int pindex) 704 { 705 int order = pindex / MIGRATE_PCPTYPES; 706 707 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 708 if (pindex >= NR_LOWORDER_PCP_LISTS) 709 order = HPAGE_PMD_ORDER; 710 #else 711 VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER); 712 #endif 713 714 return order; 715 } 716 717 static inline bool pcp_allowed_order(unsigned int order) 718 { 719 if (order <= PAGE_ALLOC_COSTLY_ORDER) 720 return true; 721 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 722 if (order == HPAGE_PMD_ORDER) 723 return true; 724 #endif 725 return false; 726 } 727 728 /* 729 * Higher-order pages are called "compound pages". They are structured thusly: 730 * 731 * The first PAGE_SIZE page is called the "head page" and have PG_head set. 732 * 733 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded 734 * in bit 0 of page->compound_head. The rest of bits is pointer to head page. 735 * 736 * The first tail page's ->compound_order holds the order of allocation. 737 * This usage means that zero-order pages may not be compound. 738 */ 739 740 void prep_compound_page(struct page *page, unsigned int order) 741 { 742 int i; 743 int nr_pages = 1 << order; 744 745 __SetPageHead(page); 746 for (i = 1; i < nr_pages; i++) 747 prep_compound_tail(page, i); 748 749 prep_compound_head(page, order); 750 } 751 752 static inline void set_buddy_order(struct page *page, unsigned int order) 753 { 754 set_page_private(page, order); 755 __SetPageBuddy(page); 756 } 757 758 #ifdef CONFIG_COMPACTION 759 static inline struct capture_control *task_capc(struct zone *zone) 760 { 761 struct capture_control *capc = current->capture_control; 762 763 return unlikely(capc) && 764 !(current->flags & PF_KTHREAD) && 765 !capc->page && 766 capc->cc->zone == zone ? capc : NULL; 767 } 768 769 static inline bool 770 compaction_capture(struct capture_control *capc, struct page *page, 771 int order, int migratetype) 772 { 773 if (!capc || order != capc->cc->order) 774 return false; 775 776 /* Do not accidentally pollute CMA or isolated regions*/ 777 if (is_migrate_cma(migratetype) || 778 is_migrate_isolate(migratetype)) 779 return false; 780 781 /* 782 * Do not let lower order allocations pollute a movable pageblock 783 * unless compaction is also requesting movable pages. 784 * This might let an unmovable request use a reclaimable pageblock 785 * and vice-versa but no more than normal fallback logic which can 786 * have trouble finding a high-order free page. 787 */ 788 if (order < pageblock_order && migratetype == MIGRATE_MOVABLE && 789 capc->cc->migratetype != MIGRATE_MOVABLE) 790 return false; 791 792 if (migratetype != capc->cc->migratetype) 793 trace_mm_page_alloc_extfrag(page, capc->cc->order, order, 794 capc->cc->migratetype, migratetype); 795 796 capc->page = page; 797 return true; 798 } 799 800 #else 801 static inline struct capture_control *task_capc(struct zone *zone) 802 { 803 return NULL; 804 } 805 806 static inline bool 807 compaction_capture(struct capture_control *capc, struct page *page, 808 int order, int migratetype) 809 { 810 return false; 811 } 812 #endif /* CONFIG_COMPACTION */ 813 814 static inline void account_freepages(struct zone *zone, int nr_pages, 815 int migratetype) 816 { 817 lockdep_assert_held(&zone->lock); 818 819 if (is_migrate_isolate(migratetype)) 820 return; 821 822 __mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages); 823 824 if (is_migrate_cma(migratetype)) 825 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_pages); 826 else if (migratetype == MIGRATE_HIGHATOMIC) 827 WRITE_ONCE(zone->nr_free_highatomic, 828 zone->nr_free_highatomic + nr_pages); 829 } 830 831 /* Used for pages not on another list */ 832 static inline void __add_to_free_list(struct page *page, struct zone *zone, 833 unsigned int order, int migratetype, 834 bool tail) 835 { 836 struct free_area *area = &zone->free_area[order]; 837 int nr_pages = 1 << order; 838 839 VM_WARN_ONCE(get_pageblock_migratetype(page) != migratetype, 840 "page type is %d, passed migratetype is %d (nr=%d)\n", 841 get_pageblock_migratetype(page), migratetype, nr_pages); 842 843 if (tail) 844 list_add_tail(&page->buddy_list, &area->free_list[migratetype]); 845 else 846 list_add(&page->buddy_list, &area->free_list[migratetype]); 847 area->nr_free++; 848 849 if (order >= pageblock_order && !is_migrate_isolate(migratetype)) 850 __mod_zone_page_state(zone, NR_FREE_PAGES_BLOCKS, nr_pages); 851 } 852 853 /* 854 * Used for pages which are on another list. Move the pages to the tail 855 * of the list - so the moved pages won't immediately be considered for 856 * allocation again (e.g., optimization for memory onlining). 857 */ 858 static inline void move_to_free_list(struct page *page, struct zone *zone, 859 unsigned int order, int old_mt, int new_mt) 860 { 861 struct free_area *area = &zone->free_area[order]; 862 int nr_pages = 1 << order; 863 864 /* Free page moving can fail, so it happens before the type update */ 865 VM_WARN_ONCE(get_pageblock_migratetype(page) != old_mt, 866 "page type is %d, passed migratetype is %d (nr=%d)\n", 867 get_pageblock_migratetype(page), old_mt, nr_pages); 868 869 list_move_tail(&page->buddy_list, &area->free_list[new_mt]); 870 871 account_freepages(zone, -nr_pages, old_mt); 872 account_freepages(zone, nr_pages, new_mt); 873 874 if (order >= pageblock_order && 875 is_migrate_isolate(old_mt) != is_migrate_isolate(new_mt)) { 876 if (!is_migrate_isolate(old_mt)) 877 nr_pages = -nr_pages; 878 __mod_zone_page_state(zone, NR_FREE_PAGES_BLOCKS, nr_pages); 879 } 880 } 881 882 static inline void __del_page_from_free_list(struct page *page, struct zone *zone, 883 unsigned int order, int migratetype) 884 { 885 int nr_pages = 1 << order; 886 887 VM_WARN_ONCE(get_pageblock_migratetype(page) != migratetype, 888 "page type is %d, passed migratetype is %d (nr=%d)\n", 889 get_pageblock_migratetype(page), migratetype, nr_pages); 890 891 /* clear reported state and update reported page count */ 892 if (page_reported(page)) 893 __ClearPageReported(page); 894 895 list_del(&page->buddy_list); 896 __ClearPageBuddy(page); 897 set_page_private(page, 0); 898 zone->free_area[order].nr_free--; 899 900 if (order >= pageblock_order && !is_migrate_isolate(migratetype)) 901 __mod_zone_page_state(zone, NR_FREE_PAGES_BLOCKS, -nr_pages); 902 } 903 904 static inline void del_page_from_free_list(struct page *page, struct zone *zone, 905 unsigned int order, int migratetype) 906 { 907 __del_page_from_free_list(page, zone, order, migratetype); 908 account_freepages(zone, -(1 << order), migratetype); 909 } 910 911 static inline struct page *get_page_from_free_area(struct free_area *area, 912 int migratetype) 913 { 914 return list_first_entry_or_null(&area->free_list[migratetype], 915 struct page, buddy_list); 916 } 917 918 /* 919 * If this is less than the 2nd largest possible page, check if the buddy 920 * of the next-higher order is free. If it is, it's possible 921 * that pages are being freed that will coalesce soon. In case, 922 * that is happening, add the free page to the tail of the list 923 * so it's less likely to be used soon and more likely to be merged 924 * as a 2-level higher order page 925 */ 926 static inline bool 927 buddy_merge_likely(unsigned long pfn, unsigned long buddy_pfn, 928 struct page *page, unsigned int order) 929 { 930 unsigned long higher_page_pfn; 931 struct page *higher_page; 932 933 if (order >= MAX_PAGE_ORDER - 1) 934 return false; 935 936 higher_page_pfn = buddy_pfn & pfn; 937 higher_page = page + (higher_page_pfn - pfn); 938 939 return find_buddy_page_pfn(higher_page, higher_page_pfn, order + 1, 940 NULL) != NULL; 941 } 942 943 static void change_pageblock_range(struct page *pageblock_page, 944 int start_order, int migratetype) 945 { 946 int nr_pageblocks = 1 << (start_order - pageblock_order); 947 948 while (nr_pageblocks--) { 949 set_pageblock_migratetype(pageblock_page, migratetype); 950 pageblock_page += pageblock_nr_pages; 951 } 952 } 953 954 /* 955 * Freeing function for a buddy system allocator. 956 * 957 * The concept of a buddy system is to maintain direct-mapped table 958 * (containing bit values) for memory blocks of various "orders". 959 * The bottom level table contains the map for the smallest allocatable 960 * units of memory (here, pages), and each level above it describes 961 * pairs of units from the levels below, hence, "buddies". 962 * At a high level, all that happens here is marking the table entry 963 * at the bottom level available, and propagating the changes upward 964 * as necessary, plus some accounting needed to play nicely with other 965 * parts of the VM system. 966 * At each level, we keep a list of pages, which are heads of continuous 967 * free pages of length of (1 << order) and marked with PageBuddy. 968 * Page's order is recorded in page_private(page) field. 969 * So when we are allocating or freeing one, we can derive the state of the 970 * other. That is, if we allocate a small block, and both were 971 * free, the remainder of the region must be split into blocks. 972 * If a block is freed, and its buddy is also free, then this 973 * triggers coalescing into a block of larger size. 974 * 975 * -- nyc 976 */ 977 978 static inline void __free_one_page(struct page *page, 979 unsigned long pfn, 980 struct zone *zone, unsigned int order, 981 int migratetype, fpi_t fpi_flags) 982 { 983 struct capture_control *capc = task_capc(zone); 984 unsigned long buddy_pfn = 0; 985 unsigned long combined_pfn; 986 struct page *buddy; 987 bool to_tail; 988 989 VM_BUG_ON(!zone_is_initialized(zone)); 990 VM_BUG_ON_PAGE(page->flags.f & PAGE_FLAGS_CHECK_AT_PREP, page); 991 992 VM_BUG_ON(migratetype == -1); 993 VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page); 994 VM_BUG_ON_PAGE(bad_range(zone, page), page); 995 996 account_freepages(zone, 1 << order, migratetype); 997 998 while (order < MAX_PAGE_ORDER) { 999 int buddy_mt = migratetype; 1000 1001 if (compaction_capture(capc, page, order, migratetype)) { 1002 account_freepages(zone, -(1 << order), migratetype); 1003 return; 1004 } 1005 1006 buddy = find_buddy_page_pfn(page, pfn, order, &buddy_pfn); 1007 if (!buddy) 1008 goto done_merging; 1009 1010 if (unlikely(order >= pageblock_order)) { 1011 /* 1012 * We want to prevent merge between freepages on pageblock 1013 * without fallbacks and normal pageblock. Without this, 1014 * pageblock isolation could cause incorrect freepage or CMA 1015 * accounting or HIGHATOMIC accounting. 1016 */ 1017 buddy_mt = get_pfnblock_migratetype(buddy, buddy_pfn); 1018 1019 if (migratetype != buddy_mt && 1020 (!migratetype_is_mergeable(migratetype) || 1021 !migratetype_is_mergeable(buddy_mt))) 1022 goto done_merging; 1023 } 1024 1025 /* 1026 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page, 1027 * merge with it and move up one order. 1028 */ 1029 if (page_is_guard(buddy)) 1030 clear_page_guard(zone, buddy, order); 1031 else 1032 __del_page_from_free_list(buddy, zone, order, buddy_mt); 1033 1034 if (unlikely(buddy_mt != migratetype)) { 1035 /* 1036 * Match buddy type. This ensures that an 1037 * expand() down the line puts the sub-blocks 1038 * on the right freelists. 1039 */ 1040 change_pageblock_range(buddy, order, migratetype); 1041 } 1042 1043 combined_pfn = buddy_pfn & pfn; 1044 page = page + (combined_pfn - pfn); 1045 pfn = combined_pfn; 1046 order++; 1047 } 1048 1049 done_merging: 1050 set_buddy_order(page, order); 1051 1052 if (fpi_flags & FPI_TO_TAIL) 1053 to_tail = true; 1054 else if (is_shuffle_order(order)) 1055 to_tail = shuffle_pick_tail(); 1056 else 1057 to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order); 1058 1059 __add_to_free_list(page, zone, order, migratetype, to_tail); 1060 1061 /* Notify page reporting subsystem of freed page */ 1062 if (!(fpi_flags & FPI_SKIP_REPORT_NOTIFY)) 1063 page_reporting_notify_free(order); 1064 } 1065 1066 /* 1067 * A bad page could be due to a number of fields. Instead of multiple branches, 1068 * try and check multiple fields with one check. The caller must do a detailed 1069 * check if necessary. 1070 */ 1071 static inline bool page_expected_state(struct page *page, 1072 unsigned long check_flags) 1073 { 1074 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1075 return false; 1076 1077 if (unlikely((unsigned long)page->mapping | 1078 page_ref_count(page) | 1079 #ifdef CONFIG_MEMCG 1080 page->memcg_data | 1081 #endif 1082 page_pool_page_is_pp(page) | 1083 (page->flags.f & check_flags))) 1084 return false; 1085 1086 return true; 1087 } 1088 1089 static const char *page_bad_reason(struct page *page, unsigned long flags) 1090 { 1091 const char *bad_reason = NULL; 1092 1093 if (unlikely(atomic_read(&page->_mapcount) != -1)) 1094 bad_reason = "nonzero mapcount"; 1095 if (unlikely(page->mapping != NULL)) 1096 bad_reason = "non-NULL mapping"; 1097 if (unlikely(page_ref_count(page) != 0)) 1098 bad_reason = "nonzero _refcount"; 1099 if (unlikely(page->flags.f & flags)) { 1100 if (flags == PAGE_FLAGS_CHECK_AT_PREP) 1101 bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag(s) set"; 1102 else 1103 bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set"; 1104 } 1105 #ifdef CONFIG_MEMCG 1106 if (unlikely(page->memcg_data)) 1107 bad_reason = "page still charged to cgroup"; 1108 #endif 1109 if (unlikely(page_pool_page_is_pp(page))) 1110 bad_reason = "page_pool leak"; 1111 return bad_reason; 1112 } 1113 1114 static inline bool free_page_is_bad(struct page *page) 1115 { 1116 if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE))) 1117 return false; 1118 1119 /* Something has gone sideways, find it */ 1120 bad_page(page, page_bad_reason(page, PAGE_FLAGS_CHECK_AT_FREE)); 1121 return true; 1122 } 1123 1124 static inline bool is_check_pages_enabled(void) 1125 { 1126 return static_branch_unlikely(&check_pages_enabled); 1127 } 1128 1129 static int free_tail_page_prepare(struct page *head_page, struct page *page) 1130 { 1131 struct folio *folio = (struct folio *)head_page; 1132 int ret = 1; 1133 1134 /* 1135 * We rely page->lru.next never has bit 0 set, unless the page 1136 * is PageTail(). Let's make sure that's true even for poisoned ->lru. 1137 */ 1138 BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1); 1139 1140 if (!is_check_pages_enabled()) { 1141 ret = 0; 1142 goto out; 1143 } 1144 switch (page - head_page) { 1145 case 1: 1146 /* the first tail page: these may be in place of ->mapping */ 1147 if (unlikely(folio_large_mapcount(folio))) { 1148 bad_page(page, "nonzero large_mapcount"); 1149 goto out; 1150 } 1151 if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT) && 1152 unlikely(atomic_read(&folio->_nr_pages_mapped))) { 1153 bad_page(page, "nonzero nr_pages_mapped"); 1154 goto out; 1155 } 1156 if (IS_ENABLED(CONFIG_MM_ID)) { 1157 if (unlikely(folio->_mm_id_mapcount[0] != -1)) { 1158 bad_page(page, "nonzero mm mapcount 0"); 1159 goto out; 1160 } 1161 if (unlikely(folio->_mm_id_mapcount[1] != -1)) { 1162 bad_page(page, "nonzero mm mapcount 1"); 1163 goto out; 1164 } 1165 } 1166 if (IS_ENABLED(CONFIG_64BIT)) { 1167 if (unlikely(atomic_read(&folio->_entire_mapcount) + 1)) { 1168 bad_page(page, "nonzero entire_mapcount"); 1169 goto out; 1170 } 1171 if (unlikely(atomic_read(&folio->_pincount))) { 1172 bad_page(page, "nonzero pincount"); 1173 goto out; 1174 } 1175 } 1176 break; 1177 case 2: 1178 /* the second tail page: deferred_list overlaps ->mapping */ 1179 if (unlikely(!list_empty(&folio->_deferred_list))) { 1180 bad_page(page, "on deferred list"); 1181 goto out; 1182 } 1183 if (!IS_ENABLED(CONFIG_64BIT)) { 1184 if (unlikely(atomic_read(&folio->_entire_mapcount) + 1)) { 1185 bad_page(page, "nonzero entire_mapcount"); 1186 goto out; 1187 } 1188 if (unlikely(atomic_read(&folio->_pincount))) { 1189 bad_page(page, "nonzero pincount"); 1190 goto out; 1191 } 1192 } 1193 break; 1194 case 3: 1195 /* the third tail page: hugetlb specifics overlap ->mappings */ 1196 if (IS_ENABLED(CONFIG_HUGETLB_PAGE)) 1197 break; 1198 fallthrough; 1199 default: 1200 if (page->mapping != TAIL_MAPPING) { 1201 bad_page(page, "corrupted mapping in tail page"); 1202 goto out; 1203 } 1204 break; 1205 } 1206 if (unlikely(!PageTail(page))) { 1207 bad_page(page, "PageTail not set"); 1208 goto out; 1209 } 1210 if (unlikely(compound_head(page) != head_page)) { 1211 bad_page(page, "compound_head not consistent"); 1212 goto out; 1213 } 1214 ret = 0; 1215 out: 1216 page->mapping = NULL; 1217 clear_compound_head(page); 1218 return ret; 1219 } 1220 1221 /* 1222 * Skip KASAN memory poisoning when either: 1223 * 1224 * 1. For generic KASAN: deferred memory initialization has not yet completed. 1225 * Tag-based KASAN modes skip pages freed via deferred memory initialization 1226 * using page tags instead (see below). 1227 * 2. For tag-based KASAN modes: the page has a match-all KASAN tag, indicating 1228 * that error detection is disabled for accesses via the page address. 1229 * 1230 * Pages will have match-all tags in the following circumstances: 1231 * 1232 * 1. Pages are being initialized for the first time, including during deferred 1233 * memory init; see the call to page_kasan_tag_reset in __init_single_page. 1234 * 2. The allocation was not unpoisoned due to __GFP_SKIP_KASAN, with the 1235 * exception of pages unpoisoned by kasan_unpoison_vmalloc. 1236 * 3. The allocation was excluded from being checked due to sampling, 1237 * see the call to kasan_unpoison_pages. 1238 * 1239 * Poisoning pages during deferred memory init will greatly lengthen the 1240 * process and cause problem in large memory systems as the deferred pages 1241 * initialization is done with interrupt disabled. 1242 * 1243 * Assuming that there will be no reference to those newly initialized 1244 * pages before they are ever allocated, this should have no effect on 1245 * KASAN memory tracking as the poison will be properly inserted at page 1246 * allocation time. The only corner case is when pages are allocated by 1247 * on-demand allocation and then freed again before the deferred pages 1248 * initialization is done, but this is not likely to happen. 1249 */ 1250 static inline bool should_skip_kasan_poison(struct page *page) 1251 { 1252 if (IS_ENABLED(CONFIG_KASAN_GENERIC)) 1253 return deferred_pages_enabled(); 1254 1255 return page_kasan_tag(page) == KASAN_TAG_KERNEL; 1256 } 1257 1258 static void kernel_init_pages(struct page *page, int numpages) 1259 { 1260 int i; 1261 1262 /* s390's use of memset() could override KASAN redzones. */ 1263 kasan_disable_current(); 1264 for (i = 0; i < numpages; i++) 1265 clear_highpage_kasan_tagged(page + i); 1266 kasan_enable_current(); 1267 } 1268 1269 #ifdef CONFIG_MEM_ALLOC_PROFILING 1270 1271 /* Should be called only if mem_alloc_profiling_enabled() */ 1272 void __clear_page_tag_ref(struct page *page) 1273 { 1274 union pgtag_ref_handle handle; 1275 union codetag_ref ref; 1276 1277 if (get_page_tag_ref(page, &ref, &handle)) { 1278 set_codetag_empty(&ref); 1279 update_page_tag_ref(handle, &ref); 1280 put_page_tag_ref(handle); 1281 } 1282 } 1283 1284 /* Should be called only if mem_alloc_profiling_enabled() */ 1285 static noinline 1286 void __pgalloc_tag_add(struct page *page, struct task_struct *task, 1287 unsigned int nr) 1288 { 1289 union pgtag_ref_handle handle; 1290 union codetag_ref ref; 1291 1292 if (likely(get_page_tag_ref(page, &ref, &handle))) { 1293 alloc_tag_add(&ref, task->alloc_tag, PAGE_SIZE * nr); 1294 update_page_tag_ref(handle, &ref); 1295 put_page_tag_ref(handle); 1296 } else { 1297 /* 1298 * page_ext is not available yet, record the pfn so we can 1299 * clear the tag ref later when page_ext is initialized. 1300 */ 1301 alloc_tag_add_early_pfn(page_to_pfn(page)); 1302 if (task->alloc_tag) 1303 alloc_tag_set_inaccurate(task->alloc_tag); 1304 } 1305 } 1306 1307 static inline void pgalloc_tag_add(struct page *page, struct task_struct *task, 1308 unsigned int nr) 1309 { 1310 if (mem_alloc_profiling_enabled()) 1311 __pgalloc_tag_add(page, task, nr); 1312 } 1313 1314 /* Should be called only if mem_alloc_profiling_enabled() */ 1315 static noinline 1316 void __pgalloc_tag_sub(struct page *page, unsigned int nr) 1317 { 1318 union pgtag_ref_handle handle; 1319 union codetag_ref ref; 1320 1321 if (get_page_tag_ref(page, &ref, &handle)) { 1322 alloc_tag_sub(&ref, PAGE_SIZE * nr); 1323 update_page_tag_ref(handle, &ref); 1324 put_page_tag_ref(handle); 1325 } 1326 } 1327 1328 static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) 1329 { 1330 if (mem_alloc_profiling_enabled()) 1331 __pgalloc_tag_sub(page, nr); 1332 } 1333 1334 /* When tag is not NULL, assuming mem_alloc_profiling_enabled */ 1335 static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) 1336 { 1337 if (tag) 1338 this_cpu_sub(tag->counters->bytes, PAGE_SIZE * nr); 1339 } 1340 1341 #else /* CONFIG_MEM_ALLOC_PROFILING */ 1342 1343 static inline void pgalloc_tag_add(struct page *page, struct task_struct *task, 1344 unsigned int nr) {} 1345 static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) {} 1346 static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) {} 1347 1348 #endif /* CONFIG_MEM_ALLOC_PROFILING */ 1349 1350 __always_inline bool __free_pages_prepare(struct page *page, 1351 unsigned int order, fpi_t fpi_flags) 1352 { 1353 int bad = 0; 1354 bool skip_kasan_poison = should_skip_kasan_poison(page); 1355 bool init = want_init_on_free(); 1356 bool compound = PageCompound(page); 1357 struct folio *folio = page_folio(page); 1358 1359 VM_BUG_ON_PAGE(PageTail(page), page); 1360 1361 trace_mm_page_free(page, order); 1362 kmsan_free_page(page, order); 1363 1364 if (memcg_kmem_online() && PageMemcgKmem(page)) 1365 __memcg_kmem_uncharge_page(page, order); 1366 1367 /* 1368 * In rare cases, when truncation or holepunching raced with 1369 * munlock after VM_LOCKED was cleared, Mlocked may still be 1370 * found set here. This does not indicate a problem, unless 1371 * "unevictable_pgs_cleared" appears worryingly large. 1372 */ 1373 if (unlikely(folio_test_mlocked(folio))) { 1374 long nr_pages = folio_nr_pages(folio); 1375 1376 __folio_clear_mlocked(folio); 1377 zone_stat_mod_folio(folio, NR_MLOCK, -nr_pages); 1378 count_vm_events(UNEVICTABLE_PGCLEARED, nr_pages); 1379 } 1380 1381 if (unlikely(PageHWPoison(page)) && !order) { 1382 /* Do not let hwpoison pages hit pcplists/buddy */ 1383 reset_page_owner(page, order); 1384 page_table_check_free(page, order); 1385 pgalloc_tag_sub(page, 1 << order); 1386 1387 /* 1388 * The page is isolated and accounted for. 1389 * Mark the codetag as empty to avoid accounting error 1390 * when the page is freed by unpoison_memory(). 1391 */ 1392 clear_page_tag_ref(page); 1393 return false; 1394 } 1395 1396 VM_BUG_ON_PAGE(compound && compound_order(page) != order, page); 1397 1398 /* 1399 * Check tail pages before head page information is cleared to 1400 * avoid checking PageCompound for order-0 pages. 1401 */ 1402 if (unlikely(order)) { 1403 int i; 1404 1405 if (compound) { 1406 page[1].flags.f &= ~PAGE_FLAGS_SECOND; 1407 #ifdef NR_PAGES_IN_LARGE_FOLIO 1408 folio->_nr_pages = 0; 1409 #endif 1410 } 1411 for (i = 1; i < (1 << order); i++) { 1412 if (compound) 1413 bad += free_tail_page_prepare(page, page + i); 1414 if (is_check_pages_enabled()) { 1415 if (free_page_is_bad(page + i)) { 1416 bad++; 1417 continue; 1418 } 1419 } 1420 (page + i)->flags.f &= ~PAGE_FLAGS_CHECK_AT_PREP; 1421 } 1422 } 1423 if (folio_test_anon(folio)) { 1424 mod_mthp_stat(order, MTHP_STAT_NR_ANON, -1); 1425 folio->mapping = NULL; 1426 } 1427 if (unlikely(page_has_type(page))) 1428 /* Reset the page_type (which overlays _mapcount) */ 1429 page->page_type = UINT_MAX; 1430 1431 if (is_check_pages_enabled()) { 1432 if (free_page_is_bad(page)) 1433 bad++; 1434 if (bad) 1435 return false; 1436 } 1437 1438 page_cpupid_reset_last(page); 1439 page->flags.f &= ~PAGE_FLAGS_CHECK_AT_PREP; 1440 page->private = 0; 1441 reset_page_owner(page, order); 1442 page_table_check_free(page, order); 1443 pgalloc_tag_sub(page, 1 << order); 1444 1445 if (!PageHighMem(page) && !(fpi_flags & FPI_TRYLOCK)) { 1446 debug_check_no_locks_freed(page_address(page), 1447 PAGE_SIZE << order); 1448 debug_check_no_obj_freed(page_address(page), 1449 PAGE_SIZE << order); 1450 } 1451 1452 kernel_poison_pages(page, 1 << order); 1453 1454 /* 1455 * As memory initialization might be integrated into KASAN, 1456 * KASAN poisoning and memory initialization code must be 1457 * kept together to avoid discrepancies in behavior. 1458 * 1459 * With hardware tag-based KASAN, memory tags must be set before the 1460 * page becomes unavailable via debug_pagealloc or arch_free_page. 1461 */ 1462 if (!skip_kasan_poison) { 1463 kasan_poison_pages(page, order, init); 1464 1465 /* Memory is already initialized if KASAN did it internally. */ 1466 if (kasan_has_integrated_init()) 1467 init = false; 1468 } 1469 if (init) 1470 kernel_init_pages(page, 1 << order); 1471 1472 /* 1473 * arch_free_page() can make the page's contents inaccessible. s390 1474 * does this. So nothing which can access the page's contents should 1475 * happen after this. 1476 */ 1477 arch_free_page(page, order); 1478 1479 debug_pagealloc_unmap_pages(page, 1 << order); 1480 1481 return true; 1482 } 1483 1484 bool free_pages_prepare(struct page *page, unsigned int order) 1485 { 1486 return __free_pages_prepare(page, order, FPI_NONE); 1487 } 1488 1489 /* 1490 * Frees a number of pages from the PCP lists 1491 * Assumes all pages on list are in same zone. 1492 * count is the number of pages to free. 1493 */ 1494 static void free_pcppages_bulk(struct zone *zone, int count, 1495 struct per_cpu_pages *pcp, 1496 int pindex) 1497 { 1498 unsigned long flags; 1499 unsigned int order; 1500 struct page *page; 1501 1502 /* 1503 * Ensure proper count is passed which otherwise would stuck in the 1504 * below while (list_empty(list)) loop. 1505 */ 1506 count = min(pcp->count, count); 1507 1508 /* Ensure requested pindex is drained first. */ 1509 pindex = pindex - 1; 1510 1511 spin_lock_irqsave(&zone->lock, flags); 1512 1513 while (count > 0) { 1514 struct list_head *list; 1515 int nr_pages; 1516 1517 /* Remove pages from lists in a round-robin fashion. */ 1518 do { 1519 if (++pindex > NR_PCP_LISTS - 1) 1520 pindex = 0; 1521 list = &pcp->lists[pindex]; 1522 } while (list_empty(list)); 1523 1524 order = pindex_to_order(pindex); 1525 nr_pages = 1 << order; 1526 do { 1527 unsigned long pfn; 1528 int mt; 1529 1530 page = list_last_entry(list, struct page, pcp_list); 1531 pfn = page_to_pfn(page); 1532 mt = get_pfnblock_migratetype(page, pfn); 1533 1534 /* must delete to avoid corrupting pcp list */ 1535 list_del(&page->pcp_list); 1536 count -= nr_pages; 1537 pcp->count -= nr_pages; 1538 1539 __free_one_page(page, pfn, zone, order, mt, FPI_NONE); 1540 trace_mm_page_pcpu_drain(page, order, mt); 1541 } while (count > 0 && !list_empty(list)); 1542 } 1543 1544 spin_unlock_irqrestore(&zone->lock, flags); 1545 } 1546 1547 /* Split a multi-block free page into its individual pageblocks. */ 1548 static void split_large_buddy(struct zone *zone, struct page *page, 1549 unsigned long pfn, int order, fpi_t fpi) 1550 { 1551 unsigned long end = pfn + (1 << order); 1552 1553 VM_WARN_ON_ONCE(!IS_ALIGNED(pfn, 1 << order)); 1554 /* Caller removed page from freelist, buddy info cleared! */ 1555 VM_WARN_ON_ONCE(PageBuddy(page)); 1556 1557 if (order > pageblock_order) 1558 order = pageblock_order; 1559 1560 do { 1561 int mt = get_pfnblock_migratetype(page, pfn); 1562 1563 __free_one_page(page, pfn, zone, order, mt, fpi); 1564 pfn += 1 << order; 1565 if (pfn == end) 1566 break; 1567 page = pfn_to_page(pfn); 1568 } while (1); 1569 } 1570 1571 static void add_page_to_zone_llist(struct zone *zone, struct page *page, 1572 unsigned int order) 1573 { 1574 /* Remember the order */ 1575 page->private = order; 1576 /* Add the page to the free list */ 1577 llist_add(&page->pcp_llist, &zone->trylock_free_pages); 1578 } 1579 1580 static void free_one_page(struct zone *zone, struct page *page, 1581 unsigned long pfn, unsigned int order, 1582 fpi_t fpi_flags) 1583 { 1584 struct llist_head *llhead; 1585 unsigned long flags; 1586 1587 if (unlikely(fpi_flags & FPI_TRYLOCK)) { 1588 if (!spin_trylock_irqsave(&zone->lock, flags)) { 1589 add_page_to_zone_llist(zone, page, order); 1590 return; 1591 } 1592 } else { 1593 spin_lock_irqsave(&zone->lock, flags); 1594 } 1595 1596 /* The lock succeeded. Process deferred pages. */ 1597 llhead = &zone->trylock_free_pages; 1598 if (unlikely(!llist_empty(llhead) && !(fpi_flags & FPI_TRYLOCK))) { 1599 struct llist_node *llnode; 1600 struct page *p, *tmp; 1601 1602 llnode = llist_del_all(llhead); 1603 llist_for_each_entry_safe(p, tmp, llnode, pcp_llist) { 1604 unsigned int p_order = p->private; 1605 1606 split_large_buddy(zone, p, page_to_pfn(p), p_order, fpi_flags); 1607 __count_vm_events(PGFREE, 1 << p_order); 1608 } 1609 } 1610 split_large_buddy(zone, page, pfn, order, fpi_flags); 1611 spin_unlock_irqrestore(&zone->lock, flags); 1612 1613 __count_vm_events(PGFREE, 1 << order); 1614 } 1615 1616 static void __free_pages_ok(struct page *page, unsigned int order, 1617 fpi_t fpi_flags) 1618 { 1619 unsigned long pfn = page_to_pfn(page); 1620 struct zone *zone = page_zone(page); 1621 1622 if (__free_pages_prepare(page, order, fpi_flags)) 1623 free_one_page(zone, page, pfn, order, fpi_flags); 1624 } 1625 1626 void __meminit __free_pages_core(struct page *page, unsigned int order, 1627 enum meminit_context context) 1628 { 1629 unsigned int nr_pages = 1 << order; 1630 struct page *p = page; 1631 unsigned int loop; 1632 1633 /* 1634 * When initializing the memmap, __init_single_page() sets the refcount 1635 * of all pages to 1 ("allocated"/"not free"). We have to set the 1636 * refcount of all involved pages to 0. 1637 * 1638 * Note that hotplugged memory pages are initialized to PageOffline(). 1639 * Pages freed from memblock might be marked as reserved. 1640 */ 1641 if (IS_ENABLED(CONFIG_MEMORY_HOTPLUG) && 1642 unlikely(context == MEMINIT_HOTPLUG)) { 1643 for (loop = 0; loop < nr_pages; loop++, p++) { 1644 VM_WARN_ON_ONCE(PageReserved(p)); 1645 __ClearPageOffline(p); 1646 set_page_count(p, 0); 1647 } 1648 1649 adjust_managed_page_count(page, nr_pages); 1650 } else { 1651 for (loop = 0; loop < nr_pages; loop++, p++) { 1652 __ClearPageReserved(p); 1653 set_page_count(p, 0); 1654 } 1655 1656 /* memblock adjusts totalram_pages() manually. */ 1657 atomic_long_add(nr_pages, &page_zone(page)->managed_pages); 1658 } 1659 1660 if (page_contains_unaccepted(page, order)) { 1661 if (order == MAX_PAGE_ORDER && __free_unaccepted(page)) 1662 return; 1663 1664 accept_memory(page_to_phys(page), PAGE_SIZE << order); 1665 } 1666 1667 /* 1668 * Bypass PCP and place fresh pages right to the tail, primarily 1669 * relevant for memory onlining. 1670 */ 1671 __free_pages_ok(page, order, FPI_TO_TAIL); 1672 } 1673 1674 /* 1675 * Check that the whole (or subset of) a pageblock given by the interval of 1676 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it 1677 * with the migration of free compaction scanner. 1678 * 1679 * Return struct page pointer of start_pfn, or NULL if checks were not passed. 1680 * 1681 * It's possible on some configurations to have a setup like node0 node1 node0 1682 * i.e. it's possible that all pages within a zones range of pages do not 1683 * belong to a single zone. We assume that a border between node0 and node1 1684 * can occur within a single pageblock, but not a node0 node1 node0 1685 * interleaving within a single pageblock. It is therefore sufficient to check 1686 * the first and last page of a pageblock and avoid checking each individual 1687 * page in a pageblock. 1688 * 1689 * Note: the function may return non-NULL struct page even for a page block 1690 * which contains a memory hole (i.e. there is no physical memory for a subset 1691 * of the pfn range). For example, if the pageblock order is MAX_PAGE_ORDER, which 1692 * will fall into 2 sub-sections, and the end pfn of the pageblock may be hole 1693 * even though the start pfn is online and valid. This should be safe most of 1694 * the time because struct pages are still initialized via init_unavailable_range() 1695 * and pfn walkers shouldn't touch any physical memory range for which they do 1696 * not recognize any specific metadata in struct pages. 1697 */ 1698 struct page *__pageblock_pfn_to_page(unsigned long start_pfn, 1699 unsigned long end_pfn, struct zone *zone) 1700 { 1701 struct page *start_page; 1702 struct page *end_page; 1703 1704 /* end_pfn is one past the range we are checking */ 1705 end_pfn--; 1706 1707 if (!pfn_valid(end_pfn)) 1708 return NULL; 1709 1710 start_page = pfn_to_online_page(start_pfn); 1711 if (!start_page) 1712 return NULL; 1713 1714 if (page_zone(start_page) != zone) 1715 return NULL; 1716 1717 end_page = pfn_to_page(end_pfn); 1718 1719 /* This gives a shorter code than deriving page_zone(end_page) */ 1720 if (page_zone_id(start_page) != page_zone_id(end_page)) 1721 return NULL; 1722 1723 return start_page; 1724 } 1725 1726 /* 1727 * The order of subdivision here is critical for the IO subsystem. 1728 * Please do not alter this order without good reasons and regression 1729 * testing. Specifically, as large blocks of memory are subdivided, 1730 * the order in which smaller blocks are delivered depends on the order 1731 * they're subdivided in this function. This is the primary factor 1732 * influencing the order in which pages are delivered to the IO 1733 * subsystem according to empirical testing, and this is also justified 1734 * by considering the behavior of a buddy system containing a single 1735 * large block of memory acted on by a series of small allocations. 1736 * This behavior is a critical factor in sglist merging's success. 1737 * 1738 * -- nyc 1739 */ 1740 static inline unsigned int expand(struct zone *zone, struct page *page, int low, 1741 int high, int migratetype) 1742 { 1743 unsigned int size = 1 << high; 1744 unsigned int nr_added = 0; 1745 1746 while (high > low) { 1747 high--; 1748 size >>= 1; 1749 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); 1750 1751 /* 1752 * Mark as guard pages (or page), that will allow to 1753 * merge back to allocator when buddy will be freed. 1754 * Corresponding page table entries will not be touched, 1755 * pages will stay not present in virtual address space 1756 */ 1757 if (set_page_guard(zone, &page[size], high)) 1758 continue; 1759 1760 __add_to_free_list(&page[size], zone, high, migratetype, false); 1761 set_buddy_order(&page[size], high); 1762 nr_added += size; 1763 } 1764 1765 return nr_added; 1766 } 1767 1768 static __always_inline void page_del_and_expand(struct zone *zone, 1769 struct page *page, int low, 1770 int high, int migratetype) 1771 { 1772 int nr_pages = 1 << high; 1773 1774 __del_page_from_free_list(page, zone, high, migratetype); 1775 nr_pages -= expand(zone, page, low, high, migratetype); 1776 account_freepages(zone, -nr_pages, migratetype); 1777 } 1778 1779 static void check_new_page_bad(struct page *page) 1780 { 1781 if (unlikely(PageHWPoison(page))) { 1782 /* Don't complain about hwpoisoned pages */ 1783 if (PageBuddy(page)) 1784 __ClearPageBuddy(page); 1785 return; 1786 } 1787 1788 bad_page(page, 1789 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_PREP)); 1790 } 1791 1792 /* 1793 * This page is about to be returned from the page allocator 1794 */ 1795 static bool check_new_page(struct page *page) 1796 { 1797 if (likely(page_expected_state(page, 1798 PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON))) 1799 return false; 1800 1801 check_new_page_bad(page); 1802 return true; 1803 } 1804 1805 static inline bool check_new_pages(struct page *page, unsigned int order) 1806 { 1807 if (is_check_pages_enabled()) { 1808 for (int i = 0; i < (1 << order); i++) { 1809 struct page *p = page + i; 1810 1811 if (check_new_page(p)) 1812 return true; 1813 } 1814 } 1815 1816 return false; 1817 } 1818 1819 static inline bool should_skip_kasan_unpoison(gfp_t flags) 1820 { 1821 /* Don't skip if a software KASAN mode is enabled. */ 1822 if (IS_ENABLED(CONFIG_KASAN_GENERIC) || 1823 IS_ENABLED(CONFIG_KASAN_SW_TAGS)) 1824 return false; 1825 1826 /* Skip, if hardware tag-based KASAN is not enabled. */ 1827 if (!kasan_hw_tags_enabled()) 1828 return true; 1829 1830 /* 1831 * With hardware tag-based KASAN enabled, skip if this has been 1832 * requested via __GFP_SKIP_KASAN. 1833 */ 1834 return flags & __GFP_SKIP_KASAN; 1835 } 1836 1837 static inline bool should_skip_init(gfp_t flags) 1838 { 1839 /* Don't skip, if hardware tag-based KASAN is not enabled. */ 1840 if (!kasan_hw_tags_enabled()) 1841 return false; 1842 1843 /* For hardware tag-based KASAN, skip if requested. */ 1844 return (flags & __GFP_SKIP_ZERO); 1845 } 1846 1847 inline void post_alloc_hook(struct page *page, unsigned int order, 1848 gfp_t gfp_flags) 1849 { 1850 const bool zero_tags = gfp_flags & __GFP_ZEROTAGS; 1851 bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags) && 1852 !should_skip_init(gfp_flags); 1853 int i; 1854 1855 set_page_private(page, 0); 1856 1857 arch_alloc_page(page, order); 1858 debug_pagealloc_map_pages(page, 1 << order); 1859 1860 /* 1861 * Page unpoisoning must happen before memory initialization. 1862 * Otherwise, the poison pattern will be overwritten for __GFP_ZERO 1863 * allocations and the page unpoisoning code will complain. 1864 */ 1865 kernel_unpoison_pages(page, 1 << order); 1866 1867 /* 1868 * As memory initialization might be integrated into KASAN, 1869 * KASAN unpoisoning and memory initializion code must be 1870 * kept together to avoid discrepancies in behavior. 1871 */ 1872 1873 /* 1874 * Clearing tags can efficiently clear the memory for us as well, if 1875 * required. 1876 */ 1877 if (zero_tags) 1878 init = tag_clear_highpages(page, 1 << order, /* clear_pages= */init); 1879 1880 if (!should_skip_kasan_unpoison(gfp_flags) && 1881 kasan_unpoison_pages(page, order, init)) { 1882 /* Take note that memory was initialized by KASAN. */ 1883 if (kasan_has_integrated_init()) 1884 init = false; 1885 } else { 1886 /* 1887 * If memory tags have not been set by KASAN, reset the page 1888 * tags to ensure page_address() dereferencing does not fault. 1889 */ 1890 for (i = 0; i != 1 << order; ++i) 1891 page_kasan_tag_reset(page + i); 1892 } 1893 /* If memory is still not initialized, initialize it now. */ 1894 if (init) 1895 kernel_init_pages(page, 1 << order); 1896 1897 set_page_owner(page, order, gfp_flags); 1898 page_table_check_alloc(page, order); 1899 pgalloc_tag_add(page, current, 1 << order); 1900 } 1901 1902 static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, 1903 unsigned int alloc_flags) 1904 { 1905 post_alloc_hook(page, order, gfp_flags); 1906 1907 if (order && (gfp_flags & __GFP_COMP)) 1908 prep_compound_page(page, order); 1909 1910 /* 1911 * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to 1912 * allocate the page. The expectation is that the caller is taking 1913 * steps that will free more memory. The caller should avoid the page 1914 * being used for !PFMEMALLOC purposes. 1915 */ 1916 if (alloc_flags & ALLOC_NO_WATERMARKS) 1917 set_page_pfmemalloc(page); 1918 else 1919 clear_page_pfmemalloc(page); 1920 } 1921 1922 /* 1923 * Go through the free lists for the given migratetype and remove 1924 * the smallest available page from the freelists 1925 */ 1926 static __always_inline 1927 struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, 1928 int migratetype) 1929 { 1930 unsigned int current_order; 1931 struct free_area *area; 1932 struct page *page; 1933 1934 /* Find a page of the appropriate size in the preferred list */ 1935 for (current_order = order; current_order < NR_PAGE_ORDERS; ++current_order) { 1936 area = &(zone->free_area[current_order]); 1937 page = get_page_from_free_area(area, migratetype); 1938 if (!page) 1939 continue; 1940 1941 page_del_and_expand(zone, page, order, current_order, 1942 migratetype); 1943 trace_mm_page_alloc_zone_locked(page, order, migratetype, 1944 pcp_allowed_order(order) && 1945 migratetype < MIGRATE_PCPTYPES); 1946 return page; 1947 } 1948 1949 return NULL; 1950 } 1951 1952 1953 /* 1954 * This array describes the order lists are fallen back to when 1955 * the free lists for the desirable migrate type are depleted 1956 * 1957 * The other migratetypes do not have fallbacks. 1958 */ 1959 static int fallbacks[MIGRATE_PCPTYPES][MIGRATE_PCPTYPES - 1] = { 1960 [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE }, 1961 [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE }, 1962 [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE }, 1963 }; 1964 1965 #ifdef CONFIG_CMA 1966 static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone, 1967 unsigned int order) 1968 { 1969 return __rmqueue_smallest(zone, order, MIGRATE_CMA); 1970 } 1971 #else 1972 static inline struct page *__rmqueue_cma_fallback(struct zone *zone, 1973 unsigned int order) { return NULL; } 1974 #endif 1975 1976 /* 1977 * Move all free pages of a block to new type's freelist. Caller needs to 1978 * change the block type. 1979 */ 1980 static int __move_freepages_block(struct zone *zone, unsigned long start_pfn, 1981 int old_mt, int new_mt) 1982 { 1983 struct page *page; 1984 unsigned long pfn, end_pfn; 1985 unsigned int order; 1986 int pages_moved = 0; 1987 1988 VM_WARN_ON(start_pfn & (pageblock_nr_pages - 1)); 1989 end_pfn = pageblock_end_pfn(start_pfn); 1990 1991 for (pfn = start_pfn; pfn < end_pfn;) { 1992 page = pfn_to_page(pfn); 1993 if (!PageBuddy(page)) { 1994 pfn++; 1995 continue; 1996 } 1997 1998 /* Make sure we are not inadvertently changing nodes */ 1999 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page); 2000 VM_BUG_ON_PAGE(page_zone(page) != zone, page); 2001 2002 order = buddy_order(page); 2003 2004 move_to_free_list(page, zone, order, old_mt, new_mt); 2005 2006 pfn += 1 << order; 2007 pages_moved += 1 << order; 2008 } 2009 2010 return pages_moved; 2011 } 2012 2013 static bool prep_move_freepages_block(struct zone *zone, struct page *page, 2014 unsigned long *start_pfn, 2015 int *num_free, int *num_movable) 2016 { 2017 unsigned long pfn, start, end; 2018 2019 pfn = page_to_pfn(page); 2020 start = pageblock_start_pfn(pfn); 2021 end = pageblock_end_pfn(pfn); 2022 2023 /* 2024 * The caller only has the lock for @zone, don't touch ranges 2025 * that straddle into other zones. While we could move part of 2026 * the range that's inside the zone, this call is usually 2027 * accompanied by other operations such as migratetype updates 2028 * which also should be locked. 2029 */ 2030 if (!zone_spans_pfn(zone, start)) 2031 return false; 2032 if (!zone_spans_pfn(zone, end - 1)) 2033 return false; 2034 2035 *start_pfn = start; 2036 2037 if (num_free) { 2038 *num_free = 0; 2039 *num_movable = 0; 2040 for (pfn = start; pfn < end;) { 2041 page = pfn_to_page(pfn); 2042 if (PageBuddy(page)) { 2043 int nr = 1 << buddy_order(page); 2044 2045 *num_free += nr; 2046 pfn += nr; 2047 continue; 2048 } 2049 /* 2050 * We assume that pages that could be isolated for 2051 * migration are movable. But we don't actually try 2052 * isolating, as that would be expensive. 2053 */ 2054 if (PageLRU(page) || page_has_movable_ops(page)) 2055 (*num_movable)++; 2056 pfn++; 2057 } 2058 } 2059 2060 return true; 2061 } 2062 2063 static int move_freepages_block(struct zone *zone, struct page *page, 2064 int old_mt, int new_mt) 2065 { 2066 unsigned long start_pfn; 2067 int res; 2068 2069 if (!prep_move_freepages_block(zone, page, &start_pfn, NULL, NULL)) 2070 return -1; 2071 2072 res = __move_freepages_block(zone, start_pfn, old_mt, new_mt); 2073 set_pageblock_migratetype(pfn_to_page(start_pfn), new_mt); 2074 2075 return res; 2076 2077 } 2078 2079 #ifdef CONFIG_MEMORY_ISOLATION 2080 /* Look for a buddy that straddles start_pfn */ 2081 static unsigned long find_large_buddy(unsigned long start_pfn) 2082 { 2083 /* 2084 * If start_pfn is not an order-0 PageBuddy, next PageBuddy containing 2085 * start_pfn has minimal order of __ffs(start_pfn) + 1. Start checking 2086 * the order with __ffs(start_pfn). If start_pfn is order-0 PageBuddy, 2087 * the starting order does not matter. 2088 */ 2089 int order = start_pfn ? __ffs(start_pfn) : MAX_PAGE_ORDER; 2090 struct page *page; 2091 unsigned long pfn = start_pfn; 2092 2093 while (!PageBuddy(page = pfn_to_page(pfn))) { 2094 /* Nothing found */ 2095 if (++order > MAX_PAGE_ORDER) 2096 return start_pfn; 2097 pfn &= ~0UL << order; 2098 } 2099 2100 /* 2101 * Found a preceding buddy, but does it straddle? 2102 */ 2103 if (pfn + (1 << buddy_order(page)) > start_pfn) 2104 return pfn; 2105 2106 /* Nothing found */ 2107 return start_pfn; 2108 } 2109 2110 static inline void toggle_pageblock_isolate(struct page *page, bool isolate) 2111 { 2112 if (isolate) 2113 set_pageblock_isolate(page); 2114 else 2115 clear_pageblock_isolate(page); 2116 } 2117 2118 /** 2119 * __move_freepages_block_isolate - move free pages in block for page isolation 2120 * @zone: the zone 2121 * @page: the pageblock page 2122 * @isolate: to isolate the given pageblock or unisolate it 2123 * 2124 * This is similar to move_freepages_block(), but handles the special 2125 * case encountered in page isolation, where the block of interest 2126 * might be part of a larger buddy spanning multiple pageblocks. 2127 * 2128 * Unlike the regular page allocator path, which moves pages while 2129 * stealing buddies off the freelist, page isolation is interested in 2130 * arbitrary pfn ranges that may have overlapping buddies on both ends. 2131 * 2132 * This function handles that. Straddling buddies are split into 2133 * individual pageblocks. Only the block of interest is moved. 2134 * 2135 * Returns %true if pages could be moved, %false otherwise. 2136 */ 2137 static bool __move_freepages_block_isolate(struct zone *zone, 2138 struct page *page, bool isolate) 2139 { 2140 unsigned long start_pfn, buddy_pfn; 2141 int from_mt; 2142 int to_mt; 2143 struct page *buddy; 2144 2145 if (isolate == get_pageblock_isolate(page)) { 2146 VM_WARN_ONCE(1, "%s a pageblock that is already in that state", 2147 isolate ? "Isolate" : "Unisolate"); 2148 return false; 2149 } 2150 2151 if (!prep_move_freepages_block(zone, page, &start_pfn, NULL, NULL)) 2152 return false; 2153 2154 /* No splits needed if buddies can't span multiple blocks */ 2155 if (pageblock_order == MAX_PAGE_ORDER) 2156 goto move; 2157 2158 buddy_pfn = find_large_buddy(start_pfn); 2159 buddy = pfn_to_page(buddy_pfn); 2160 /* We're a part of a larger buddy */ 2161 if (PageBuddy(buddy) && buddy_order(buddy) > pageblock_order) { 2162 int order = buddy_order(buddy); 2163 2164 del_page_from_free_list(buddy, zone, order, 2165 get_pfnblock_migratetype(buddy, buddy_pfn)); 2166 toggle_pageblock_isolate(page, isolate); 2167 split_large_buddy(zone, buddy, buddy_pfn, order, FPI_NONE); 2168 return true; 2169 } 2170 2171 move: 2172 /* Use MIGRATETYPE_MASK to get non-isolate migratetype */ 2173 if (isolate) { 2174 from_mt = __get_pfnblock_flags_mask(page, page_to_pfn(page), 2175 MIGRATETYPE_MASK); 2176 to_mt = MIGRATE_ISOLATE; 2177 } else { 2178 from_mt = MIGRATE_ISOLATE; 2179 to_mt = __get_pfnblock_flags_mask(page, page_to_pfn(page), 2180 MIGRATETYPE_MASK); 2181 } 2182 2183 __move_freepages_block(zone, start_pfn, from_mt, to_mt); 2184 toggle_pageblock_isolate(pfn_to_page(start_pfn), isolate); 2185 2186 return true; 2187 } 2188 2189 bool pageblock_isolate_and_move_free_pages(struct zone *zone, struct page *page) 2190 { 2191 return __move_freepages_block_isolate(zone, page, true); 2192 } 2193 2194 bool pageblock_unisolate_and_move_free_pages(struct zone *zone, struct page *page) 2195 { 2196 return __move_freepages_block_isolate(zone, page, false); 2197 } 2198 2199 #endif /* CONFIG_MEMORY_ISOLATION */ 2200 2201 static inline bool boost_watermark(struct zone *zone) 2202 { 2203 unsigned long max_boost; 2204 2205 if (!watermark_boost_factor) 2206 return false; 2207 /* 2208 * Don't bother in zones that are unlikely to produce results. 2209 * On small machines, including kdump capture kernels running 2210 * in a small area, boosting the watermark can cause an out of 2211 * memory situation immediately. 2212 */ 2213 if ((pageblock_nr_pages * 4) > zone_managed_pages(zone)) 2214 return false; 2215 2216 max_boost = mult_frac(zone->_watermark[WMARK_HIGH], 2217 watermark_boost_factor, 10000); 2218 2219 /* 2220 * high watermark may be uninitialised if fragmentation occurs 2221 * very early in boot so do not boost. We do not fall 2222 * through and boost by pageblock_nr_pages as failing 2223 * allocations that early means that reclaim is not going 2224 * to help and it may even be impossible to reclaim the 2225 * boosted watermark resulting in a hang. 2226 */ 2227 if (!max_boost) 2228 return false; 2229 2230 max_boost = max(pageblock_nr_pages, max_boost); 2231 2232 zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages, 2233 max_boost); 2234 2235 return true; 2236 } 2237 2238 /* 2239 * When we are falling back to another migratetype during allocation, should we 2240 * try to claim an entire block to satisfy further allocations, instead of 2241 * polluting multiple pageblocks? 2242 */ 2243 static bool should_try_claim_block(unsigned int order, int start_mt) 2244 { 2245 /* 2246 * Leaving this order check is intended, although there is 2247 * relaxed order check in next check. The reason is that 2248 * we can actually claim the whole pageblock if this condition met, 2249 * but, below check doesn't guarantee it and that is just heuristic 2250 * so could be changed anytime. 2251 */ 2252 if (order >= pageblock_order) 2253 return true; 2254 2255 /* 2256 * Above a certain threshold, always try to claim, as it's likely there 2257 * will be more free pages in the pageblock. 2258 */ 2259 if (order >= pageblock_order / 2) 2260 return true; 2261 2262 /* 2263 * Unmovable/reclaimable allocations would cause permanent 2264 * fragmentations if they fell back to allocating from a movable block 2265 * (polluting it), so we try to claim the whole block regardless of the 2266 * allocation size. Later movable allocations can always steal from this 2267 * block, which is less problematic. 2268 */ 2269 if (start_mt == MIGRATE_RECLAIMABLE || start_mt == MIGRATE_UNMOVABLE) 2270 return true; 2271 2272 if (page_group_by_mobility_disabled) 2273 return true; 2274 2275 /* 2276 * Movable pages won't cause permanent fragmentation, so when you alloc 2277 * small pages, we just need to temporarily steal unmovable or 2278 * reclaimable pages that are closest to the request size. After a 2279 * while, memory compaction may occur to form large contiguous pages, 2280 * and the next movable allocation may not need to steal. 2281 */ 2282 return false; 2283 } 2284 2285 /* 2286 * Check whether there is a suitable fallback freepage with requested order. 2287 * If claimable is true, this function returns fallback_mt only if 2288 * we would do this whole-block claiming. This would help to reduce 2289 * fragmentation due to mixed migratetype pages in one pageblock. 2290 */ 2291 int find_suitable_fallback(struct free_area *area, unsigned int order, 2292 int migratetype, bool claimable) 2293 { 2294 int i; 2295 2296 if (claimable && !should_try_claim_block(order, migratetype)) 2297 return -2; 2298 2299 if (area->nr_free == 0) 2300 return -1; 2301 2302 for (i = 0; i < MIGRATE_PCPTYPES - 1 ; i++) { 2303 int fallback_mt = fallbacks[migratetype][i]; 2304 2305 if (!free_area_empty(area, fallback_mt)) 2306 return fallback_mt; 2307 } 2308 2309 return -1; 2310 } 2311 2312 /* 2313 * This function implements actual block claiming behaviour. If order is large 2314 * enough, we can claim the whole pageblock for the requested migratetype. If 2315 * not, we check the pageblock for constituent pages; if at least half of the 2316 * pages are free or compatible, we can still claim the whole block, so pages 2317 * freed in the future will be put on the correct free list. 2318 */ 2319 static struct page * 2320 try_to_claim_block(struct zone *zone, struct page *page, 2321 int current_order, int order, int start_type, 2322 int block_type, unsigned int alloc_flags) 2323 { 2324 int free_pages, movable_pages, alike_pages; 2325 unsigned long start_pfn; 2326 2327 /* Take ownership for orders >= pageblock_order */ 2328 if (current_order >= pageblock_order) { 2329 unsigned int nr_added; 2330 2331 del_page_from_free_list(page, zone, current_order, block_type); 2332 change_pageblock_range(page, current_order, start_type); 2333 nr_added = expand(zone, page, order, current_order, start_type); 2334 account_freepages(zone, nr_added, start_type); 2335 return page; 2336 } 2337 2338 /* 2339 * Boost watermarks to increase reclaim pressure to reduce the 2340 * likelihood of future fallbacks. Wake kswapd now as the node 2341 * may be balanced overall and kswapd will not wake naturally. 2342 */ 2343 if (boost_watermark(zone) && (alloc_flags & ALLOC_KSWAPD)) 2344 set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); 2345 2346 /* moving whole block can fail due to zone boundary conditions */ 2347 if (!prep_move_freepages_block(zone, page, &start_pfn, &free_pages, 2348 &movable_pages)) 2349 return NULL; 2350 2351 /* 2352 * Determine how many pages are compatible with our allocation. 2353 * For movable allocation, it's the number of movable pages which 2354 * we just obtained. For other types it's a bit more tricky. 2355 */ 2356 if (start_type == MIGRATE_MOVABLE) { 2357 alike_pages = movable_pages; 2358 } else { 2359 /* 2360 * If we are falling back a RECLAIMABLE or UNMOVABLE allocation 2361 * to MOVABLE pageblock, consider all non-movable pages as 2362 * compatible. If it's UNMOVABLE falling back to RECLAIMABLE or 2363 * vice versa, be conservative since we can't distinguish the 2364 * exact migratetype of non-movable pages. 2365 */ 2366 if (block_type == MIGRATE_MOVABLE) 2367 alike_pages = pageblock_nr_pages 2368 - (free_pages + movable_pages); 2369 else 2370 alike_pages = 0; 2371 } 2372 /* 2373 * If a sufficient number of pages in the block are either free or of 2374 * compatible migratability as our allocation, claim the whole block. 2375 */ 2376 if (free_pages + alike_pages >= (1 << (pageblock_order-1)) || 2377 page_group_by_mobility_disabled) { 2378 __move_freepages_block(zone, start_pfn, block_type, start_type); 2379 set_pageblock_migratetype(pfn_to_page(start_pfn), start_type); 2380 return __rmqueue_smallest(zone, order, start_type); 2381 } 2382 2383 return NULL; 2384 } 2385 2386 /* 2387 * Try to allocate from some fallback migratetype by claiming the entire block, 2388 * i.e. converting it to the allocation's start migratetype. 2389 * 2390 * The use of signed ints for order and current_order is a deliberate 2391 * deviation from the rest of this file, to make the for loop 2392 * condition simpler. 2393 */ 2394 static __always_inline struct page * 2395 __rmqueue_claim(struct zone *zone, int order, int start_migratetype, 2396 unsigned int alloc_flags) 2397 { 2398 struct free_area *area; 2399 int current_order; 2400 int min_order = order; 2401 struct page *page; 2402 int fallback_mt; 2403 2404 /* 2405 * Do not steal pages from freelists belonging to other pageblocks 2406 * i.e. orders < pageblock_order. If there are no local zones free, 2407 * the zonelists will be reiterated without ALLOC_NOFRAGMENT. 2408 */ 2409 if (order < pageblock_order && alloc_flags & ALLOC_NOFRAGMENT) 2410 min_order = pageblock_order; 2411 2412 /* 2413 * Find the largest available free page in the other list. This roughly 2414 * approximates finding the pageblock with the most free pages, which 2415 * would be too costly to do exactly. 2416 */ 2417 for (current_order = MAX_PAGE_ORDER; current_order >= min_order; 2418 --current_order) { 2419 area = &(zone->free_area[current_order]); 2420 fallback_mt = find_suitable_fallback(area, current_order, 2421 start_migratetype, true); 2422 2423 /* No block in that order */ 2424 if (fallback_mt == -1) 2425 continue; 2426 2427 /* Advanced into orders too low to claim, abort */ 2428 if (fallback_mt == -2) 2429 break; 2430 2431 page = get_page_from_free_area(area, fallback_mt); 2432 page = try_to_claim_block(zone, page, current_order, order, 2433 start_migratetype, fallback_mt, 2434 alloc_flags); 2435 if (page) { 2436 trace_mm_page_alloc_extfrag(page, order, current_order, 2437 start_migratetype, fallback_mt); 2438 return page; 2439 } 2440 } 2441 2442 return NULL; 2443 } 2444 2445 /* 2446 * Try to steal a single page from some fallback migratetype. Leave the rest of 2447 * the block as its current migratetype, potentially causing fragmentation. 2448 */ 2449 static __always_inline struct page * 2450 __rmqueue_steal(struct zone *zone, int order, int start_migratetype) 2451 { 2452 struct free_area *area; 2453 int current_order; 2454 struct page *page; 2455 int fallback_mt; 2456 2457 for (current_order = order; current_order < NR_PAGE_ORDERS; current_order++) { 2458 area = &(zone->free_area[current_order]); 2459 fallback_mt = find_suitable_fallback(area, current_order, 2460 start_migratetype, false); 2461 if (fallback_mt == -1) 2462 continue; 2463 2464 page = get_page_from_free_area(area, fallback_mt); 2465 page_del_and_expand(zone, page, order, current_order, fallback_mt); 2466 trace_mm_page_alloc_extfrag(page, order, current_order, 2467 start_migratetype, fallback_mt); 2468 return page; 2469 } 2470 2471 return NULL; 2472 } 2473 2474 enum rmqueue_mode { 2475 RMQUEUE_NORMAL, 2476 RMQUEUE_CMA, 2477 RMQUEUE_CLAIM, 2478 RMQUEUE_STEAL, 2479 }; 2480 2481 /* 2482 * Do the hard work of removing an element from the buddy allocator. 2483 * Call me with the zone->lock already held. 2484 */ 2485 static __always_inline struct page * 2486 __rmqueue(struct zone *zone, unsigned int order, int migratetype, 2487 unsigned int alloc_flags, enum rmqueue_mode *mode) 2488 { 2489 struct page *page; 2490 2491 if (IS_ENABLED(CONFIG_CMA)) { 2492 /* 2493 * Balance movable allocations between regular and CMA areas by 2494 * allocating from CMA when over half of the zone's free memory 2495 * is in the CMA area. 2496 */ 2497 if (alloc_flags & ALLOC_CMA && 2498 zone_page_state(zone, NR_FREE_CMA_PAGES) > 2499 zone_page_state(zone, NR_FREE_PAGES) / 2) { 2500 page = __rmqueue_cma_fallback(zone, order); 2501 if (page) 2502 return page; 2503 } 2504 } 2505 2506 /* 2507 * First try the freelists of the requested migratetype, then try 2508 * fallbacks modes with increasing levels of fragmentation risk. 2509 * 2510 * The fallback logic is expensive and rmqueue_bulk() calls in 2511 * a loop with the zone->lock held, meaning the freelists are 2512 * not subject to any outside changes. Remember in *mode where 2513 * we found pay dirt, to save us the search on the next call. 2514 */ 2515 switch (*mode) { 2516 case RMQUEUE_NORMAL: 2517 page = __rmqueue_smallest(zone, order, migratetype); 2518 if (page) 2519 return page; 2520 fallthrough; 2521 case RMQUEUE_CMA: 2522 if (alloc_flags & ALLOC_CMA) { 2523 page = __rmqueue_cma_fallback(zone, order); 2524 if (page) { 2525 *mode = RMQUEUE_CMA; 2526 return page; 2527 } 2528 } 2529 fallthrough; 2530 case RMQUEUE_CLAIM: 2531 page = __rmqueue_claim(zone, order, migratetype, alloc_flags); 2532 if (page) { 2533 /* Replenished preferred freelist, back to normal mode. */ 2534 *mode = RMQUEUE_NORMAL; 2535 return page; 2536 } 2537 fallthrough; 2538 case RMQUEUE_STEAL: 2539 if (!(alloc_flags & ALLOC_NOFRAGMENT)) { 2540 page = __rmqueue_steal(zone, order, migratetype); 2541 if (page) { 2542 *mode = RMQUEUE_STEAL; 2543 return page; 2544 } 2545 } 2546 } 2547 return NULL; 2548 } 2549 2550 /* 2551 * Obtain a specified number of elements from the buddy allocator, all under 2552 * a single hold of the lock, for efficiency. Add them to the supplied list. 2553 * Returns the number of new pages which were placed at *list. 2554 */ 2555 static int rmqueue_bulk(struct zone *zone, unsigned int order, 2556 unsigned long count, struct list_head *list, 2557 int migratetype, unsigned int alloc_flags) 2558 { 2559 enum rmqueue_mode rmqm = RMQUEUE_NORMAL; 2560 unsigned long flags; 2561 int i; 2562 2563 if (unlikely(alloc_flags & ALLOC_TRYLOCK)) { 2564 if (!spin_trylock_irqsave(&zone->lock, flags)) 2565 return 0; 2566 } else { 2567 spin_lock_irqsave(&zone->lock, flags); 2568 } 2569 for (i = 0; i < count; ++i) { 2570 struct page *page = __rmqueue(zone, order, migratetype, 2571 alloc_flags, &rmqm); 2572 if (unlikely(page == NULL)) 2573 break; 2574 2575 /* 2576 * Split buddy pages returned by expand() are received here in 2577 * physical page order. The page is added to the tail of 2578 * caller's list. From the callers perspective, the linked list 2579 * is ordered by page number under some conditions. This is 2580 * useful for IO devices that can forward direction from the 2581 * head, thus also in the physical page order. This is useful 2582 * for IO devices that can merge IO requests if the physical 2583 * pages are ordered properly. 2584 */ 2585 list_add_tail(&page->pcp_list, list); 2586 } 2587 spin_unlock_irqrestore(&zone->lock, flags); 2588 2589 return i; 2590 } 2591 2592 /* 2593 * Called from the vmstat counter updater to decay the PCP high. 2594 * Return whether there are addition works to do. 2595 */ 2596 bool decay_pcp_high(struct zone *zone, struct per_cpu_pages *pcp) 2597 { 2598 int high_min, to_drain, to_drain_batched, batch; 2599 unsigned long UP_flags; 2600 bool todo = false; 2601 2602 high_min = READ_ONCE(pcp->high_min); 2603 batch = READ_ONCE(pcp->batch); 2604 /* 2605 * Decrease pcp->high periodically to try to free possible 2606 * idle PCP pages. And, avoid to free too many pages to 2607 * control latency. This caps pcp->high decrement too. 2608 */ 2609 if (pcp->high > high_min) { 2610 pcp->high = max3(pcp->count - (batch << CONFIG_PCP_BATCH_SCALE_MAX), 2611 pcp->high - (pcp->high >> 3), high_min); 2612 if (pcp->high > high_min) 2613 todo = true; 2614 } 2615 2616 to_drain = pcp->count - pcp->high; 2617 while (to_drain > 0) { 2618 to_drain_batched = min(to_drain, batch); 2619 pcp_spin_lock_maybe_irqsave(pcp, UP_flags); 2620 free_pcppages_bulk(zone, to_drain_batched, pcp, 0); 2621 pcp_spin_unlock_maybe_irqrestore(pcp, UP_flags); 2622 todo = true; 2623 2624 to_drain -= to_drain_batched; 2625 } 2626 2627 return todo; 2628 } 2629 2630 #ifdef CONFIG_NUMA 2631 /* 2632 * Called from the vmstat counter updater to drain pagesets of this 2633 * currently executing processor on remote nodes after they have 2634 * expired. 2635 */ 2636 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) 2637 { 2638 unsigned long UP_flags; 2639 int to_drain, batch; 2640 2641 batch = READ_ONCE(pcp->batch); 2642 to_drain = min(pcp->count, batch); 2643 if (to_drain > 0) { 2644 pcp_spin_lock_maybe_irqsave(pcp, UP_flags); 2645 free_pcppages_bulk(zone, to_drain, pcp, 0); 2646 pcp_spin_unlock_maybe_irqrestore(pcp, UP_flags); 2647 } 2648 } 2649 #endif 2650 2651 /* 2652 * Drain pcplists of the indicated processor and zone. 2653 */ 2654 static void drain_pages_zone(unsigned int cpu, struct zone *zone) 2655 { 2656 struct per_cpu_pages *pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); 2657 unsigned long UP_flags; 2658 int count; 2659 2660 do { 2661 pcp_spin_lock_maybe_irqsave(pcp, UP_flags); 2662 count = pcp->count; 2663 if (count) { 2664 int to_drain = min(count, 2665 pcp->batch << CONFIG_PCP_BATCH_SCALE_MAX); 2666 2667 free_pcppages_bulk(zone, to_drain, pcp, 0); 2668 count -= to_drain; 2669 } 2670 pcp_spin_unlock_maybe_irqrestore(pcp, UP_flags); 2671 } while (count); 2672 } 2673 2674 /* 2675 * Drain pcplists of all zones on the indicated processor. 2676 */ 2677 static void drain_pages(unsigned int cpu) 2678 { 2679 struct zone *zone; 2680 2681 for_each_populated_zone(zone) { 2682 drain_pages_zone(cpu, zone); 2683 } 2684 } 2685 2686 /* 2687 * Spill all of this CPU's per-cpu pages back into the buddy allocator. 2688 */ 2689 void drain_local_pages(struct zone *zone) 2690 { 2691 int cpu = smp_processor_id(); 2692 2693 if (zone) 2694 drain_pages_zone(cpu, zone); 2695 else 2696 drain_pages(cpu); 2697 } 2698 2699 /* 2700 * The implementation of drain_all_pages(), exposing an extra parameter to 2701 * drain on all cpus. 2702 * 2703 * drain_all_pages() is optimized to only execute on cpus where pcplists are 2704 * not empty. The check for non-emptiness can however race with a free to 2705 * pcplist that has not yet increased the pcp->count from 0 to 1. Callers 2706 * that need the guarantee that every CPU has drained can disable the 2707 * optimizing racy check. 2708 */ 2709 static void __drain_all_pages(struct zone *zone, bool force_all_cpus) 2710 { 2711 int cpu; 2712 2713 /* 2714 * Allocate in the BSS so we won't require allocation in 2715 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y 2716 */ 2717 static cpumask_t cpus_with_pcps; 2718 2719 /* 2720 * Do not drain if one is already in progress unless it's specific to 2721 * a zone. Such callers are primarily CMA and memory hotplug and need 2722 * the drain to be complete when the call returns. 2723 */ 2724 if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) { 2725 if (!zone) 2726 return; 2727 mutex_lock(&pcpu_drain_mutex); 2728 } 2729 2730 /* 2731 * We don't care about racing with CPU hotplug event 2732 * as offline notification will cause the notified 2733 * cpu to drain that CPU pcps and on_each_cpu_mask 2734 * disables preemption as part of its processing 2735 */ 2736 for_each_online_cpu(cpu) { 2737 struct per_cpu_pages *pcp; 2738 struct zone *z; 2739 bool has_pcps = false; 2740 2741 if (force_all_cpus) { 2742 /* 2743 * The pcp.count check is racy, some callers need a 2744 * guarantee that no cpu is missed. 2745 */ 2746 has_pcps = true; 2747 } else if (zone) { 2748 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); 2749 if (pcp->count) 2750 has_pcps = true; 2751 } else { 2752 for_each_populated_zone(z) { 2753 pcp = per_cpu_ptr(z->per_cpu_pageset, cpu); 2754 if (pcp->count) { 2755 has_pcps = true; 2756 break; 2757 } 2758 } 2759 } 2760 2761 if (has_pcps) 2762 cpumask_set_cpu(cpu, &cpus_with_pcps); 2763 else 2764 cpumask_clear_cpu(cpu, &cpus_with_pcps); 2765 } 2766 2767 for_each_cpu(cpu, &cpus_with_pcps) { 2768 if (zone) 2769 drain_pages_zone(cpu, zone); 2770 else 2771 drain_pages(cpu); 2772 } 2773 2774 mutex_unlock(&pcpu_drain_mutex); 2775 } 2776 2777 /* 2778 * Spill all the per-cpu pages from all CPUs back into the buddy allocator. 2779 * 2780 * When zone parameter is non-NULL, spill just the single zone's pages. 2781 */ 2782 void drain_all_pages(struct zone *zone) 2783 { 2784 __drain_all_pages(zone, false); 2785 } 2786 2787 static int nr_pcp_free(struct per_cpu_pages *pcp, int batch, int high, bool free_high) 2788 { 2789 int min_nr_free, max_nr_free; 2790 2791 /* Free as much as possible if batch freeing high-order pages. */ 2792 if (unlikely(free_high)) 2793 return min(pcp->count, batch << CONFIG_PCP_BATCH_SCALE_MAX); 2794 2795 /* Check for PCP disabled or boot pageset */ 2796 if (unlikely(high < batch)) 2797 return 1; 2798 2799 /* Leave at least pcp->batch pages on the list */ 2800 min_nr_free = batch; 2801 max_nr_free = high - batch; 2802 2803 /* 2804 * Increase the batch number to the number of the consecutive 2805 * freed pages to reduce zone lock contention. 2806 */ 2807 batch = clamp_t(int, pcp->free_count, min_nr_free, max_nr_free); 2808 2809 return batch; 2810 } 2811 2812 static int nr_pcp_high(struct per_cpu_pages *pcp, struct zone *zone, 2813 int batch, bool free_high) 2814 { 2815 int high, high_min, high_max; 2816 2817 high_min = READ_ONCE(pcp->high_min); 2818 high_max = READ_ONCE(pcp->high_max); 2819 high = pcp->high = clamp(pcp->high, high_min, high_max); 2820 2821 if (unlikely(!high)) 2822 return 0; 2823 2824 if (unlikely(free_high)) { 2825 pcp->high = max(high - (batch << CONFIG_PCP_BATCH_SCALE_MAX), 2826 high_min); 2827 return 0; 2828 } 2829 2830 /* 2831 * If reclaim is active, limit the number of pages that can be 2832 * stored on pcp lists 2833 */ 2834 if (test_bit(ZONE_RECLAIM_ACTIVE, &zone->flags)) { 2835 int free_count = max_t(int, pcp->free_count, batch); 2836 2837 pcp->high = max(high - free_count, high_min); 2838 return min(batch << 2, pcp->high); 2839 } 2840 2841 if (high_min == high_max) 2842 return high; 2843 2844 if (test_bit(ZONE_BELOW_HIGH, &zone->flags)) { 2845 int free_count = max_t(int, pcp->free_count, batch); 2846 2847 pcp->high = max(high - free_count, high_min); 2848 high = max(pcp->count, high_min); 2849 } else if (pcp->count >= high) { 2850 int need_high = pcp->free_count + batch; 2851 2852 /* pcp->high should be large enough to hold batch freed pages */ 2853 if (pcp->high < need_high) 2854 pcp->high = clamp(need_high, high_min, high_max); 2855 } 2856 2857 return high; 2858 } 2859 2860 static void free_frozen_page_commit(struct zone *zone, 2861 struct per_cpu_pages *pcp, struct page *page, int migratetype, 2862 unsigned int order, fpi_t fpi_flags) 2863 { 2864 int high, batch; 2865 int pindex; 2866 bool free_high = false; 2867 2868 /* 2869 * On freeing, reduce the number of pages that are batch allocated. 2870 * See nr_pcp_alloc() where alloc_factor is increased for subsequent 2871 * allocations. 2872 */ 2873 pcp->alloc_factor >>= 1; 2874 __count_vm_events(PGFREE, 1 << order); 2875 pindex = order_to_pindex(migratetype, order); 2876 list_add(&page->pcp_list, &pcp->lists[pindex]); 2877 pcp->count += 1 << order; 2878 2879 batch = READ_ONCE(pcp->batch); 2880 /* 2881 * As high-order pages other than THP's stored on PCP can contribute 2882 * to fragmentation, limit the number stored when PCP is heavily 2883 * freeing without allocation. The remainder after bulk freeing 2884 * stops will be drained from vmstat refresh context. 2885 */ 2886 if (order && order <= PAGE_ALLOC_COSTLY_ORDER) { 2887 free_high = (pcp->free_count >= (batch + pcp->high_min / 2) && 2888 (pcp->flags & PCPF_PREV_FREE_HIGH_ORDER) && 2889 (!(pcp->flags & PCPF_FREE_HIGH_BATCH) || 2890 pcp->count >= batch)); 2891 pcp->flags |= PCPF_PREV_FREE_HIGH_ORDER; 2892 } else if (pcp->flags & PCPF_PREV_FREE_HIGH_ORDER) { 2893 pcp->flags &= ~PCPF_PREV_FREE_HIGH_ORDER; 2894 } 2895 if (pcp->free_count < (batch << CONFIG_PCP_BATCH_SCALE_MAX)) 2896 pcp->free_count += (1 << order); 2897 2898 if (unlikely(fpi_flags & FPI_TRYLOCK)) { 2899 /* 2900 * Do not attempt to take a zone lock. Let pcp->count get 2901 * over high mark temporarily. 2902 */ 2903 return; 2904 } 2905 2906 high = nr_pcp_high(pcp, zone, batch, free_high); 2907 if (pcp->count < high) 2908 return; 2909 2910 free_pcppages_bulk(zone, nr_pcp_free(pcp, batch, high, free_high), 2911 pcp, pindex); 2912 if (test_bit(ZONE_BELOW_HIGH, &zone->flags) && 2913 zone_watermark_ok(zone, 0, high_wmark_pages(zone), 2914 ZONE_MOVABLE, 0)) { 2915 struct pglist_data *pgdat = zone->zone_pgdat; 2916 clear_bit(ZONE_BELOW_HIGH, &zone->flags); 2917 2918 /* 2919 * Assume that memory pressure on this node is gone and may be 2920 * in a reclaimable state. If a memory fallback node exists, 2921 * direct reclaim may not have been triggered, causing a 2922 * 'hopeless node' to stay in that state for a while. Let 2923 * kswapd work again by resetting kswapd_failures. 2924 */ 2925 if (atomic_read(&pgdat->kswapd_failures) >= MAX_RECLAIM_RETRIES && 2926 next_memory_node(pgdat->node_id) < MAX_NUMNODES) 2927 atomic_set(&pgdat->kswapd_failures, 0); 2928 } 2929 } 2930 2931 /* 2932 * Free a pcp page 2933 */ 2934 static void __free_frozen_pages(struct page *page, unsigned int order, 2935 fpi_t fpi_flags) 2936 { 2937 unsigned long __maybe_unused UP_flags; 2938 struct per_cpu_pages *pcp; 2939 struct zone *zone; 2940 unsigned long pfn = page_to_pfn(page); 2941 int migratetype; 2942 2943 if (!pcp_allowed_order(order)) { 2944 __free_pages_ok(page, order, fpi_flags); 2945 return; 2946 } 2947 2948 if (!__free_pages_prepare(page, order, fpi_flags)) 2949 return; 2950 2951 /* 2952 * We only track unmovable, reclaimable and movable on pcp lists. 2953 * Place ISOLATE pages on the isolated list because they are being 2954 * offlined but treat HIGHATOMIC and CMA as movable pages so we can 2955 * get those areas back if necessary. Otherwise, we may have to free 2956 * excessively into the page allocator 2957 */ 2958 zone = page_zone(page); 2959 migratetype = get_pfnblock_migratetype(page, pfn); 2960 if (unlikely(migratetype >= MIGRATE_PCPTYPES)) { 2961 if (unlikely(is_migrate_isolate(migratetype))) { 2962 free_one_page(zone, page, pfn, order, fpi_flags); 2963 return; 2964 } 2965 migratetype = MIGRATE_MOVABLE; 2966 } 2967 2968 if (unlikely((fpi_flags & FPI_TRYLOCK) && IS_ENABLED(CONFIG_PREEMPT_RT) 2969 && (in_nmi() || in_hardirq()))) { 2970 add_page_to_zone_llist(zone, page, order); 2971 return; 2972 } 2973 pcp_trylock_prepare(UP_flags); 2974 pcp = pcp_spin_trylock(zone->per_cpu_pageset); 2975 if (pcp) { 2976 free_frozen_page_commit(zone, pcp, page, migratetype, order, fpi_flags); 2977 pcp_spin_unlock(pcp); 2978 } else { 2979 free_one_page(zone, page, pfn, order, fpi_flags); 2980 } 2981 pcp_trylock_finish(UP_flags); 2982 } 2983 2984 void free_frozen_pages(struct page *page, unsigned int order) 2985 { 2986 __free_frozen_pages(page, order, FPI_NONE); 2987 } 2988 2989 /* 2990 * Free a batch of folios 2991 */ 2992 void free_unref_folios(struct folio_batch *folios) 2993 { 2994 unsigned long __maybe_unused UP_flags; 2995 struct per_cpu_pages *pcp = NULL; 2996 struct zone *locked_zone = NULL; 2997 int i, j; 2998 2999 /* Prepare folios for freeing */ 3000 for (i = 0, j = 0; i < folios->nr; i++) { 3001 struct folio *folio = folios->folios[i]; 3002 unsigned long pfn = folio_pfn(folio); 3003 unsigned int order = folio_order(folio); 3004 3005 if (!__free_pages_prepare(&folio->page, order, FPI_NONE)) 3006 continue; 3007 /* 3008 * Free orders not handled on the PCP directly to the 3009 * allocator. 3010 */ 3011 if (!pcp_allowed_order(order)) { 3012 free_one_page(folio_zone(folio), &folio->page, 3013 pfn, order, FPI_NONE); 3014 continue; 3015 } 3016 folio->private = (void *)(unsigned long)order; 3017 if (j != i) 3018 folios->folios[j] = folio; 3019 j++; 3020 } 3021 folios->nr = j; 3022 3023 for (i = 0; i < folios->nr; i++) { 3024 struct folio *folio = folios->folios[i]; 3025 struct zone *zone = folio_zone(folio); 3026 unsigned long pfn = folio_pfn(folio); 3027 unsigned int order = (unsigned long)folio->private; 3028 int migratetype; 3029 3030 folio->private = NULL; 3031 migratetype = get_pfnblock_migratetype(&folio->page, pfn); 3032 3033 /* Different zone requires a different pcp lock */ 3034 if (zone != locked_zone || 3035 is_migrate_isolate(migratetype)) { 3036 if (pcp) { 3037 pcp_spin_unlock(pcp); 3038 pcp_trylock_finish(UP_flags); 3039 locked_zone = NULL; 3040 pcp = NULL; 3041 } 3042 3043 /* 3044 * Free isolated pages directly to the 3045 * allocator, see comment in free_frozen_pages. 3046 */ 3047 if (is_migrate_isolate(migratetype)) { 3048 free_one_page(zone, &folio->page, pfn, 3049 order, FPI_NONE); 3050 continue; 3051 } 3052 3053 /* 3054 * trylock is necessary as folios may be getting freed 3055 * from IRQ or SoftIRQ context after an IO completion. 3056 */ 3057 pcp_trylock_prepare(UP_flags); 3058 pcp = pcp_spin_trylock(zone->per_cpu_pageset); 3059 if (unlikely(!pcp)) { 3060 pcp_trylock_finish(UP_flags); 3061 free_one_page(zone, &folio->page, pfn, 3062 order, FPI_NONE); 3063 continue; 3064 } 3065 locked_zone = zone; 3066 } 3067 3068 /* 3069 * Non-isolated types over MIGRATE_PCPTYPES get added 3070 * to the MIGRATE_MOVABLE pcp list. 3071 */ 3072 if (unlikely(migratetype >= MIGRATE_PCPTYPES)) 3073 migratetype = MIGRATE_MOVABLE; 3074 3075 trace_mm_page_free_batched(&folio->page); 3076 free_frozen_page_commit(zone, pcp, &folio->page, migratetype, 3077 order, FPI_NONE); 3078 } 3079 3080 if (pcp) { 3081 pcp_spin_unlock(pcp); 3082 pcp_trylock_finish(UP_flags); 3083 } 3084 folio_batch_reinit(folios); 3085 } 3086 3087 /* 3088 * split_page takes a non-compound higher-order page, and splits it into 3089 * n (1<<order) sub-pages: page[0..n] 3090 * Each sub-page must be freed individually. 3091 * 3092 * Note: this is probably too low level an operation for use in drivers. 3093 * Please consult with lkml before using this in your driver. 3094 */ 3095 void split_page(struct page *page, unsigned int order) 3096 { 3097 int i; 3098 3099 VM_BUG_ON_PAGE(PageCompound(page), page); 3100 VM_BUG_ON_PAGE(!page_count(page), page); 3101 3102 for (i = 1; i < (1 << order); i++) 3103 set_page_refcounted(page + i); 3104 split_page_owner(page, order, 0); 3105 pgalloc_tag_split(page_folio(page), order, 0); 3106 split_page_memcg(page, order); 3107 } 3108 EXPORT_SYMBOL_GPL(split_page); 3109 3110 int __isolate_free_page(struct page *page, unsigned int order) 3111 { 3112 struct zone *zone = page_zone(page); 3113 int mt = get_pageblock_migratetype(page); 3114 3115 if (!is_migrate_isolate(mt)) { 3116 unsigned long watermark; 3117 /* 3118 * Obey watermarks as if the page was being allocated. We can 3119 * emulate a high-order watermark check with a raised order-0 3120 * watermark, because we already know our high-order page 3121 * exists. 3122 */ 3123 watermark = zone->_watermark[WMARK_MIN] + (1UL << order); 3124 if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA)) 3125 return 0; 3126 } 3127 3128 del_page_from_free_list(page, zone, order, mt); 3129 3130 /* 3131 * Set the pageblock if the isolated page is at least half of a 3132 * pageblock 3133 */ 3134 if (order >= pageblock_order - 1) { 3135 struct page *endpage = page + (1 << order) - 1; 3136 for (; page < endpage; page += pageblock_nr_pages) { 3137 int mt = get_pageblock_migratetype(page); 3138 /* 3139 * Only change normal pageblocks (i.e., they can merge 3140 * with others) 3141 */ 3142 if (migratetype_is_mergeable(mt)) 3143 move_freepages_block(zone, page, mt, 3144 MIGRATE_MOVABLE); 3145 } 3146 } 3147 3148 return 1UL << order; 3149 } 3150 3151 /** 3152 * __putback_isolated_page - Return a now-isolated page back where we got it 3153 * @page: Page that was isolated 3154 * @order: Order of the isolated page 3155 * @mt: The page's pageblock's migratetype 3156 * 3157 * This function is meant to return a page pulled from the free lists via 3158 * __isolate_free_page back to the free lists they were pulled from. 3159 */ 3160 void __putback_isolated_page(struct page *page, unsigned int order, int mt) 3161 { 3162 struct zone *zone = page_zone(page); 3163 3164 /* zone lock should be held when this function is called */ 3165 lockdep_assert_held(&zone->lock); 3166 3167 /* Return isolated page to tail of freelist. */ 3168 __free_one_page(page, page_to_pfn(page), zone, order, mt, 3169 FPI_SKIP_REPORT_NOTIFY | FPI_TO_TAIL); 3170 } 3171 3172 /* 3173 * Update NUMA hit/miss statistics 3174 */ 3175 static inline void zone_statistics(struct zone *preferred_zone, struct zone *z, 3176 long nr_account) 3177 { 3178 #ifdef CONFIG_NUMA 3179 enum numa_stat_item local_stat = NUMA_LOCAL; 3180 3181 /* skip numa counters update if numa stats is disabled */ 3182 if (!static_branch_likely(&vm_numa_stat_key)) 3183 return; 3184 3185 if (zone_to_nid(z) != numa_node_id()) 3186 local_stat = NUMA_OTHER; 3187 3188 if (zone_to_nid(z) == zone_to_nid(preferred_zone)) 3189 __count_numa_events(z, NUMA_HIT, nr_account); 3190 else { 3191 __count_numa_events(z, NUMA_MISS, nr_account); 3192 __count_numa_events(preferred_zone, NUMA_FOREIGN, nr_account); 3193 } 3194 __count_numa_events(z, local_stat, nr_account); 3195 #endif 3196 } 3197 3198 static __always_inline 3199 struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone, 3200 unsigned int order, unsigned int alloc_flags, 3201 int migratetype) 3202 { 3203 struct page *page; 3204 unsigned long flags; 3205 3206 do { 3207 page = NULL; 3208 if (unlikely(alloc_flags & ALLOC_TRYLOCK)) { 3209 if (!spin_trylock_irqsave(&zone->lock, flags)) 3210 return NULL; 3211 } else { 3212 spin_lock_irqsave(&zone->lock, flags); 3213 } 3214 if (alloc_flags & ALLOC_HIGHATOMIC) 3215 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); 3216 if (!page) { 3217 enum rmqueue_mode rmqm = RMQUEUE_NORMAL; 3218 3219 page = __rmqueue(zone, order, migratetype, alloc_flags, &rmqm); 3220 3221 /* 3222 * If the allocation fails, allow OOM handling and 3223 * order-0 (atomic) allocs access to HIGHATOMIC 3224 * reserves as failing now is worse than failing a 3225 * high-order atomic allocation in the future. 3226 */ 3227 if (!page && (alloc_flags & (ALLOC_OOM|ALLOC_NON_BLOCK))) 3228 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); 3229 3230 if (!page) { 3231 spin_unlock_irqrestore(&zone->lock, flags); 3232 return NULL; 3233 } 3234 } 3235 spin_unlock_irqrestore(&zone->lock, flags); 3236 } while (check_new_pages(page, order)); 3237 3238 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); 3239 zone_statistics(preferred_zone, zone, 1); 3240 3241 return page; 3242 } 3243 3244 static int nr_pcp_alloc(struct per_cpu_pages *pcp, struct zone *zone, int order) 3245 { 3246 int high, base_batch, batch, max_nr_alloc; 3247 int high_max, high_min; 3248 3249 base_batch = READ_ONCE(pcp->batch); 3250 high_min = READ_ONCE(pcp->high_min); 3251 high_max = READ_ONCE(pcp->high_max); 3252 high = pcp->high = clamp(pcp->high, high_min, high_max); 3253 3254 /* Check for PCP disabled or boot pageset */ 3255 if (unlikely(high < base_batch)) 3256 return 1; 3257 3258 if (order) 3259 batch = base_batch; 3260 else 3261 batch = (base_batch << pcp->alloc_factor); 3262 3263 /* 3264 * If we had larger pcp->high, we could avoid to allocate from 3265 * zone. 3266 */ 3267 if (high_min != high_max && !test_bit(ZONE_BELOW_HIGH, &zone->flags)) 3268 high = pcp->high = min(high + batch, high_max); 3269 3270 if (!order) { 3271 max_nr_alloc = max(high - pcp->count - base_batch, base_batch); 3272 /* 3273 * Double the number of pages allocated each time there is 3274 * subsequent allocation of order-0 pages without any freeing. 3275 */ 3276 if (batch <= max_nr_alloc && 3277 pcp->alloc_factor < CONFIG_PCP_BATCH_SCALE_MAX) 3278 pcp->alloc_factor++; 3279 batch = min(batch, max_nr_alloc); 3280 } 3281 3282 /* 3283 * Scale batch relative to order if batch implies free pages 3284 * can be stored on the PCP. Batch can be 1 for small zones or 3285 * for boot pagesets which should never store free pages as 3286 * the pages may belong to arbitrary zones. 3287 */ 3288 if (batch > 1) 3289 batch = max(batch >> order, 2); 3290 3291 return batch; 3292 } 3293 3294 /* Remove page from the per-cpu list, caller must protect the list */ 3295 static inline 3296 struct page *__rmqueue_pcplist(struct zone *zone, unsigned int order, 3297 int migratetype, 3298 unsigned int alloc_flags, 3299 struct per_cpu_pages *pcp, 3300 struct list_head *list) 3301 { 3302 struct page *page; 3303 3304 do { 3305 if (list_empty(list)) { 3306 int batch = nr_pcp_alloc(pcp, zone, order); 3307 int alloced; 3308 3309 alloced = rmqueue_bulk(zone, order, 3310 batch, list, 3311 migratetype, alloc_flags); 3312 3313 pcp->count += alloced << order; 3314 if (unlikely(list_empty(list))) 3315 return NULL; 3316 } 3317 3318 page = list_first_entry(list, struct page, pcp_list); 3319 list_del(&page->pcp_list); 3320 pcp->count -= 1 << order; 3321 } while (check_new_pages(page, order)); 3322 3323 return page; 3324 } 3325 3326 /* Lock and remove page from the per-cpu list */ 3327 static struct page *rmqueue_pcplist(struct zone *preferred_zone, 3328 struct zone *zone, unsigned int order, 3329 int migratetype, unsigned int alloc_flags) 3330 { 3331 struct per_cpu_pages *pcp; 3332 struct list_head *list; 3333 struct page *page; 3334 unsigned long __maybe_unused UP_flags; 3335 3336 /* spin_trylock may fail due to a parallel drain or IRQ reentrancy. */ 3337 pcp_trylock_prepare(UP_flags); 3338 pcp = pcp_spin_trylock(zone->per_cpu_pageset); 3339 if (!pcp) { 3340 pcp_trylock_finish(UP_flags); 3341 return NULL; 3342 } 3343 3344 /* 3345 * On allocation, reduce the number of pages that are batch freed. 3346 * See nr_pcp_free() where free_factor is increased for subsequent 3347 * frees. 3348 */ 3349 pcp->free_count >>= 1; 3350 list = &pcp->lists[order_to_pindex(migratetype, order)]; 3351 page = __rmqueue_pcplist(zone, order, migratetype, alloc_flags, pcp, list); 3352 pcp_spin_unlock(pcp); 3353 pcp_trylock_finish(UP_flags); 3354 if (page) { 3355 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); 3356 zone_statistics(preferred_zone, zone, 1); 3357 } 3358 return page; 3359 } 3360 3361 /* 3362 * Allocate a page from the given zone. 3363 * Use pcplists for THP or "cheap" high-order allocations. 3364 */ 3365 3366 /* 3367 * Do not instrument rmqueue() with KMSAN. This function may call 3368 * __msan_poison_alloca() through a call to set_pfnblock_migratetype(). 3369 * If __msan_poison_alloca() attempts to allocate pages for the stack depot, it 3370 * may call rmqueue() again, which will result in a deadlock. 3371 */ 3372 __no_sanitize_memory 3373 static inline 3374 struct page *rmqueue(struct zone *preferred_zone, 3375 struct zone *zone, unsigned int order, 3376 gfp_t gfp_flags, unsigned int alloc_flags, 3377 int migratetype) 3378 { 3379 struct page *page; 3380 3381 if (likely(pcp_allowed_order(order))) { 3382 page = rmqueue_pcplist(preferred_zone, zone, order, 3383 migratetype, alloc_flags); 3384 if (likely(page)) 3385 goto out; 3386 } 3387 3388 page = rmqueue_buddy(preferred_zone, zone, order, alloc_flags, 3389 migratetype); 3390 3391 out: 3392 /* Separate test+clear to avoid unnecessary atomics */ 3393 if ((alloc_flags & ALLOC_KSWAPD) && 3394 unlikely(test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags))) { 3395 clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); 3396 wakeup_kswapd(zone, 0, 0, zone_idx(zone)); 3397 } 3398 3399 VM_BUG_ON_PAGE(page && bad_range(zone, page), page); 3400 return page; 3401 } 3402 3403 /* 3404 * Reserve the pageblock(s) surrounding an allocation request for 3405 * exclusive use of high-order atomic allocations if there are no 3406 * empty page blocks that contain a page with a suitable order 3407 */ 3408 static void reserve_highatomic_pageblock(struct page *page, int order, 3409 struct zone *zone) 3410 { 3411 int mt; 3412 unsigned long max_managed, flags; 3413 3414 /* 3415 * The number reserved as: minimum is 1 pageblock, maximum is 3416 * roughly 1% of a zone. But if 1% of a zone falls below a 3417 * pageblock size, then don't reserve any pageblocks. 3418 * Check is race-prone but harmless. 3419 */ 3420 if ((zone_managed_pages(zone) / 100) < pageblock_nr_pages) 3421 return; 3422 max_managed = ALIGN((zone_managed_pages(zone) / 100), pageblock_nr_pages); 3423 if (zone->nr_reserved_highatomic >= max_managed) 3424 return; 3425 3426 spin_lock_irqsave(&zone->lock, flags); 3427 3428 /* Recheck the nr_reserved_highatomic limit under the lock */ 3429 if (zone->nr_reserved_highatomic >= max_managed) 3430 goto out_unlock; 3431 3432 /* Yoink! */ 3433 mt = get_pageblock_migratetype(page); 3434 /* Only reserve normal pageblocks (i.e., they can merge with others) */ 3435 if (!migratetype_is_mergeable(mt)) 3436 goto out_unlock; 3437 3438 if (order < pageblock_order) { 3439 if (move_freepages_block(zone, page, mt, MIGRATE_HIGHATOMIC) == -1) 3440 goto out_unlock; 3441 zone->nr_reserved_highatomic += pageblock_nr_pages; 3442 } else { 3443 change_pageblock_range(page, order, MIGRATE_HIGHATOMIC); 3444 zone->nr_reserved_highatomic += 1 << order; 3445 } 3446 3447 out_unlock: 3448 spin_unlock_irqrestore(&zone->lock, flags); 3449 } 3450 3451 /* 3452 * Used when an allocation is about to fail under memory pressure. This 3453 * potentially hurts the reliability of high-order allocations when under 3454 * intense memory pressure but failed atomic allocations should be easier 3455 * to recover from than an OOM. 3456 * 3457 * If @force is true, try to unreserve pageblocks even though highatomic 3458 * pageblock is exhausted. 3459 */ 3460 static bool unreserve_highatomic_pageblock(const struct alloc_context *ac, 3461 bool force) 3462 { 3463 struct zonelist *zonelist = ac->zonelist; 3464 unsigned long flags; 3465 struct zoneref *z; 3466 struct zone *zone; 3467 struct page *page; 3468 int order; 3469 int ret; 3470 3471 for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->highest_zoneidx, 3472 ac->nodemask) { 3473 /* 3474 * Preserve at least one pageblock unless memory pressure 3475 * is really high. 3476 */ 3477 if (!force && zone->nr_reserved_highatomic <= 3478 pageblock_nr_pages) 3479 continue; 3480 3481 spin_lock_irqsave(&zone->lock, flags); 3482 for (order = 0; order < NR_PAGE_ORDERS; order++) { 3483 struct free_area *area = &(zone->free_area[order]); 3484 unsigned long size; 3485 3486 page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC); 3487 if (!page) 3488 continue; 3489 3490 size = max(pageblock_nr_pages, 1UL << order); 3491 /* 3492 * It should never happen but changes to 3493 * locking could inadvertently allow a per-cpu 3494 * drain to add pages to MIGRATE_HIGHATOMIC 3495 * while unreserving so be safe and watch for 3496 * underflows. 3497 */ 3498 if (WARN_ON_ONCE(size > zone->nr_reserved_highatomic)) 3499 size = zone->nr_reserved_highatomic; 3500 zone->nr_reserved_highatomic -= size; 3501 3502 /* 3503 * Convert to ac->migratetype and avoid the normal 3504 * pageblock stealing heuristics. Minimally, the caller 3505 * is doing the work and needs the pages. More 3506 * importantly, if the block was always converted to 3507 * MIGRATE_UNMOVABLE or another type then the number 3508 * of pageblocks that cannot be completely freed 3509 * may increase. 3510 */ 3511 if (order < pageblock_order) 3512 ret = move_freepages_block(zone, page, 3513 MIGRATE_HIGHATOMIC, 3514 ac->migratetype); 3515 else { 3516 move_to_free_list(page, zone, order, 3517 MIGRATE_HIGHATOMIC, 3518 ac->migratetype); 3519 change_pageblock_range(page, order, 3520 ac->migratetype); 3521 ret = 1; 3522 } 3523 /* 3524 * Reserving the block(s) already succeeded, 3525 * so this should not fail on zone boundaries. 3526 */ 3527 WARN_ON_ONCE(ret == -1); 3528 if (ret > 0) { 3529 spin_unlock_irqrestore(&zone->lock, flags); 3530 return ret; 3531 } 3532 } 3533 spin_unlock_irqrestore(&zone->lock, flags); 3534 } 3535 3536 return false; 3537 } 3538 3539 static inline long __zone_watermark_unusable_free(struct zone *z, 3540 unsigned int order, unsigned int alloc_flags) 3541 { 3542 long unusable_free = (1 << order) - 1; 3543 3544 /* 3545 * If the caller does not have rights to reserves below the min 3546 * watermark then subtract the free pages reserved for highatomic. 3547 */ 3548 if (likely(!(alloc_flags & ALLOC_RESERVES))) 3549 unusable_free += READ_ONCE(z->nr_free_highatomic); 3550 3551 #ifdef CONFIG_CMA 3552 /* If allocation can't use CMA areas don't use free CMA pages */ 3553 if (!(alloc_flags & ALLOC_CMA)) 3554 unusable_free += zone_page_state(z, NR_FREE_CMA_PAGES); 3555 #endif 3556 3557 return unusable_free; 3558 } 3559 3560 /* 3561 * Return true if free base pages are above 'mark'. For high-order checks it 3562 * will return true of the order-0 watermark is reached and there is at least 3563 * one free page of a suitable size. Checking now avoids taking the zone lock 3564 * to check in the allocation paths if no pages are free. 3565 */ 3566 bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 3567 int highest_zoneidx, unsigned int alloc_flags, 3568 long free_pages) 3569 { 3570 long min = mark; 3571 int o; 3572 3573 /* free_pages may go negative - that's OK */ 3574 free_pages -= __zone_watermark_unusable_free(z, order, alloc_flags); 3575 3576 if (unlikely(alloc_flags & ALLOC_RESERVES)) { 3577 /* 3578 * __GFP_HIGH allows access to 50% of the min reserve as well 3579 * as OOM. 3580 */ 3581 if (alloc_flags & ALLOC_MIN_RESERVE) { 3582 min -= min / 2; 3583 3584 /* 3585 * Non-blocking allocations (e.g. GFP_ATOMIC) can 3586 * access more reserves than just __GFP_HIGH. Other 3587 * non-blocking allocations requests such as GFP_NOWAIT 3588 * or (GFP_KERNEL & ~__GFP_DIRECT_RECLAIM) do not get 3589 * access to the min reserve. 3590 */ 3591 if (alloc_flags & ALLOC_NON_BLOCK) 3592 min -= min / 4; 3593 } 3594 3595 /* 3596 * OOM victims can try even harder than the normal reserve 3597 * users on the grounds that it's definitely going to be in 3598 * the exit path shortly and free memory. Any allocation it 3599 * makes during the free path will be small and short-lived. 3600 */ 3601 if (alloc_flags & ALLOC_OOM) 3602 min -= min / 2; 3603 } 3604 3605 /* 3606 * Check watermarks for an order-0 allocation request. If these 3607 * are not met, then a high-order request also cannot go ahead 3608 * even if a suitable page happened to be free. 3609 */ 3610 if (free_pages <= min + z->lowmem_reserve[highest_zoneidx]) 3611 return false; 3612 3613 /* If this is an order-0 request then the watermark is fine */ 3614 if (!order) 3615 return true; 3616 3617 /* For a high-order request, check at least one suitable page is free */ 3618 for (o = order; o < NR_PAGE_ORDERS; o++) { 3619 struct free_area *area = &z->free_area[o]; 3620 int mt; 3621 3622 if (!area->nr_free) 3623 continue; 3624 3625 for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) { 3626 if (!free_area_empty(area, mt)) 3627 return true; 3628 } 3629 3630 #ifdef CONFIG_CMA 3631 if ((alloc_flags & ALLOC_CMA) && 3632 !free_area_empty(area, MIGRATE_CMA)) { 3633 return true; 3634 } 3635 #endif 3636 if ((alloc_flags & (ALLOC_HIGHATOMIC|ALLOC_OOM)) && 3637 !free_area_empty(area, MIGRATE_HIGHATOMIC)) { 3638 return true; 3639 } 3640 } 3641 return false; 3642 } 3643 3644 bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 3645 int highest_zoneidx, unsigned int alloc_flags) 3646 { 3647 return __zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags, 3648 zone_page_state(z, NR_FREE_PAGES)); 3649 } 3650 3651 static inline bool zone_watermark_fast(struct zone *z, unsigned int order, 3652 unsigned long mark, int highest_zoneidx, 3653 unsigned int alloc_flags, gfp_t gfp_mask) 3654 { 3655 long free_pages; 3656 3657 free_pages = zone_page_state(z, NR_FREE_PAGES); 3658 3659 /* 3660 * Fast check for order-0 only. If this fails then the reserves 3661 * need to be calculated. 3662 */ 3663 if (!order) { 3664 long usable_free; 3665 long reserved; 3666 3667 usable_free = free_pages; 3668 reserved = __zone_watermark_unusable_free(z, 0, alloc_flags); 3669 3670 /* reserved may over estimate high-atomic reserves. */ 3671 usable_free -= min(usable_free, reserved); 3672 if (usable_free > mark + z->lowmem_reserve[highest_zoneidx]) 3673 return true; 3674 } 3675 3676 if (__zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags, 3677 free_pages)) 3678 return true; 3679 3680 /* 3681 * Ignore watermark boosting for __GFP_HIGH order-0 allocations 3682 * when checking the min watermark. The min watermark is the 3683 * point where boosting is ignored so that kswapd is woken up 3684 * when below the low watermark. 3685 */ 3686 if (unlikely(!order && (alloc_flags & ALLOC_MIN_RESERVE) && z->watermark_boost 3687 && ((alloc_flags & ALLOC_WMARK_MASK) == WMARK_MIN))) { 3688 mark = z->_watermark[WMARK_MIN]; 3689 return __zone_watermark_ok(z, order, mark, highest_zoneidx, 3690 alloc_flags, free_pages); 3691 } 3692 3693 return false; 3694 } 3695 3696 #ifdef CONFIG_NUMA 3697 int __read_mostly node_reclaim_distance = RECLAIM_DISTANCE; 3698 3699 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 3700 { 3701 return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <= 3702 node_reclaim_distance; 3703 } 3704 #else /* CONFIG_NUMA */ 3705 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) 3706 { 3707 return true; 3708 } 3709 #endif /* CONFIG_NUMA */ 3710 3711 /* 3712 * The restriction on ZONE_DMA32 as being a suitable zone to use to avoid 3713 * fragmentation is subtle. If the preferred zone was HIGHMEM then 3714 * premature use of a lower zone may cause lowmem pressure problems that 3715 * are worse than fragmentation. If the next zone is ZONE_DMA then it is 3716 * probably too small. It only makes sense to spread allocations to avoid 3717 * fragmentation between the Normal and DMA32 zones. 3718 */ 3719 static inline unsigned int 3720 alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) 3721 { 3722 unsigned int alloc_flags; 3723 3724 /* 3725 * __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD 3726 * to save a branch. 3727 */ 3728 alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM); 3729 3730 if (defrag_mode) { 3731 alloc_flags |= ALLOC_NOFRAGMENT; 3732 return alloc_flags; 3733 } 3734 3735 #ifdef CONFIG_ZONE_DMA32 3736 if (!zone) 3737 return alloc_flags; 3738 3739 if (zone_idx(zone) != ZONE_NORMAL) 3740 return alloc_flags; 3741 3742 /* 3743 * If ZONE_DMA32 exists, assume it is the one after ZONE_NORMAL and 3744 * the pointer is within zone->zone_pgdat->node_zones[]. Also assume 3745 * on UMA that if Normal is populated then so is DMA32. 3746 */ 3747 BUILD_BUG_ON(ZONE_NORMAL - ZONE_DMA32 != 1); 3748 if (nr_online_nodes > 1 && !populated_zone(--zone)) 3749 return alloc_flags; 3750 3751 alloc_flags |= ALLOC_NOFRAGMENT; 3752 #endif /* CONFIG_ZONE_DMA32 */ 3753 return alloc_flags; 3754 } 3755 3756 /* Must be called after current_gfp_context() which can change gfp_mask */ 3757 static inline unsigned int gfp_to_alloc_flags_cma(gfp_t gfp_mask, 3758 unsigned int alloc_flags) 3759 { 3760 #ifdef CONFIG_CMA 3761 if (gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE) 3762 alloc_flags |= ALLOC_CMA; 3763 #endif 3764 return alloc_flags; 3765 } 3766 3767 /* 3768 * get_page_from_freelist goes through the zonelist trying to allocate 3769 * a page. 3770 */ 3771 static struct page * 3772 get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, 3773 const struct alloc_context *ac) 3774 { 3775 struct zoneref *z; 3776 struct zone *zone; 3777 struct pglist_data *last_pgdat = NULL; 3778 bool last_pgdat_dirty_ok = false; 3779 bool no_fallback; 3780 bool skip_kswapd_nodes = nr_online_nodes > 1; 3781 bool skipped_kswapd_nodes = false; 3782 3783 retry: 3784 /* 3785 * Scan zonelist, looking for a zone with enough free. 3786 * See also cpuset_current_node_allowed() comment in kernel/cgroup/cpuset.c. 3787 */ 3788 no_fallback = alloc_flags & ALLOC_NOFRAGMENT; 3789 z = ac->preferred_zoneref; 3790 for_next_zone_zonelist_nodemask(zone, z, ac->highest_zoneidx, 3791 ac->nodemask) { 3792 struct page *page; 3793 unsigned long mark; 3794 3795 if (cpusets_enabled() && 3796 (alloc_flags & ALLOC_CPUSET) && 3797 !__cpuset_zone_allowed(zone, gfp_mask)) 3798 continue; 3799 /* 3800 * When allocating a page cache page for writing, we 3801 * want to get it from a node that is within its dirty 3802 * limit, such that no single node holds more than its 3803 * proportional share of globally allowed dirty pages. 3804 * The dirty limits take into account the node's 3805 * lowmem reserves and high watermark so that kswapd 3806 * should be able to balance it without having to 3807 * write pages from its LRU list. 3808 * 3809 * XXX: For now, allow allocations to potentially 3810 * exceed the per-node dirty limit in the slowpath 3811 * (spread_dirty_pages unset) before going into reclaim, 3812 * which is important when on a NUMA setup the allowed 3813 * nodes are together not big enough to reach the 3814 * global limit. The proper fix for these situations 3815 * will require awareness of nodes in the 3816 * dirty-throttling and the flusher threads. 3817 */ 3818 if (ac->spread_dirty_pages) { 3819 if (last_pgdat != zone->zone_pgdat) { 3820 last_pgdat = zone->zone_pgdat; 3821 last_pgdat_dirty_ok = node_dirty_ok(zone->zone_pgdat); 3822 } 3823 3824 if (!last_pgdat_dirty_ok) 3825 continue; 3826 } 3827 3828 if (no_fallback && !defrag_mode && nr_online_nodes > 1 && 3829 zone != zonelist_zone(ac->preferred_zoneref)) { 3830 int local_nid; 3831 3832 /* 3833 * If moving to a remote node, retry but allow 3834 * fragmenting fallbacks. Locality is more important 3835 * than fragmentation avoidance. 3836 */ 3837 local_nid = zonelist_node_idx(ac->preferred_zoneref); 3838 if (zone_to_nid(zone) != local_nid) { 3839 alloc_flags &= ~ALLOC_NOFRAGMENT; 3840 goto retry; 3841 } 3842 } 3843 3844 /* 3845 * If kswapd is already active on a node, keep looking 3846 * for other nodes that might be idle. This can happen 3847 * if another process has NUMA bindings and is causing 3848 * kswapd wakeups on only some nodes. Avoid accidental 3849 * "node_reclaim_mode"-like behavior in this case. 3850 */ 3851 if (skip_kswapd_nodes && 3852 !waitqueue_active(&zone->zone_pgdat->kswapd_wait)) { 3853 skipped_kswapd_nodes = true; 3854 continue; 3855 } 3856 3857 cond_accept_memory(zone, order, alloc_flags); 3858 3859 /* 3860 * Detect whether the number of free pages is below high 3861 * watermark. If so, we will decrease pcp->high and free 3862 * PCP pages in free path to reduce the possibility of 3863 * premature page reclaiming. Detection is done here to 3864 * avoid to do that in hotter free path. 3865 */ 3866 if (test_bit(ZONE_BELOW_HIGH, &zone->flags)) 3867 goto check_alloc_wmark; 3868 3869 mark = high_wmark_pages(zone); 3870 if (zone_watermark_fast(zone, order, mark, 3871 ac->highest_zoneidx, alloc_flags, 3872 gfp_mask)) 3873 goto try_this_zone; 3874 else 3875 set_bit(ZONE_BELOW_HIGH, &zone->flags); 3876 3877 check_alloc_wmark: 3878 mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK); 3879 if (!zone_watermark_fast(zone, order, mark, 3880 ac->highest_zoneidx, alloc_flags, 3881 gfp_mask)) { 3882 int ret; 3883 3884 if (cond_accept_memory(zone, order, alloc_flags)) 3885 goto try_this_zone; 3886 3887 /* 3888 * Watermark failed for this zone, but see if we can 3889 * grow this zone if it contains deferred pages. 3890 */ 3891 if (deferred_pages_enabled()) { 3892 if (_deferred_grow_zone(zone, order)) 3893 goto try_this_zone; 3894 } 3895 /* Checked here to keep the fast path fast */ 3896 BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK); 3897 if (alloc_flags & ALLOC_NO_WATERMARKS) 3898 goto try_this_zone; 3899 3900 if (!node_reclaim_enabled() || 3901 !zone_allows_reclaim(zonelist_zone(ac->preferred_zoneref), zone)) 3902 continue; 3903 3904 ret = node_reclaim(zone->zone_pgdat, gfp_mask, order); 3905 switch (ret) { 3906 case NODE_RECLAIM_NOSCAN: 3907 /* did not scan */ 3908 continue; 3909 case NODE_RECLAIM_FULL: 3910 /* scanned but unreclaimable */ 3911 continue; 3912 default: 3913 /* did we reclaim enough */ 3914 if (zone_watermark_ok(zone, order, mark, 3915 ac->highest_zoneidx, alloc_flags)) 3916 goto try_this_zone; 3917 3918 continue; 3919 } 3920 } 3921 3922 try_this_zone: 3923 page = rmqueue(zonelist_zone(ac->preferred_zoneref), zone, order, 3924 gfp_mask, alloc_flags, ac->migratetype); 3925 if (page) { 3926 prep_new_page(page, order, gfp_mask, alloc_flags); 3927 3928 /* 3929 * If this is a high-order atomic allocation then check 3930 * if the pageblock should be reserved for the future 3931 */ 3932 if (unlikely(alloc_flags & ALLOC_HIGHATOMIC)) 3933 reserve_highatomic_pageblock(page, order, zone); 3934 3935 return page; 3936 } else { 3937 if (cond_accept_memory(zone, order, alloc_flags)) 3938 goto try_this_zone; 3939 3940 /* Try again if zone has deferred pages */ 3941 if (deferred_pages_enabled()) { 3942 if (_deferred_grow_zone(zone, order)) 3943 goto try_this_zone; 3944 } 3945 } 3946 } 3947 3948 /* 3949 * If we skipped over nodes with active kswapds and found no 3950 * idle nodes, retry and place anywhere the watermarks permit. 3951 */ 3952 if (skip_kswapd_nodes && skipped_kswapd_nodes) { 3953 skip_kswapd_nodes = false; 3954 goto retry; 3955 } 3956 3957 /* 3958 * It's possible on a UMA machine to get through all zones that are 3959 * fragmented. If avoiding fragmentation, reset and try again. 3960 */ 3961 if (no_fallback && !defrag_mode) { 3962 alloc_flags &= ~ALLOC_NOFRAGMENT; 3963 goto retry; 3964 } 3965 3966 return NULL; 3967 } 3968 3969 static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask) 3970 { 3971 unsigned int filter = SHOW_MEM_FILTER_NODES; 3972 3973 /* 3974 * This documents exceptions given to allocations in certain 3975 * contexts that are allowed to allocate outside current's set 3976 * of allowed nodes. 3977 */ 3978 if (!(gfp_mask & __GFP_NOMEMALLOC)) 3979 if (tsk_is_oom_victim(current) || 3980 (current->flags & (PF_MEMALLOC | PF_EXITING))) 3981 filter &= ~SHOW_MEM_FILTER_NODES; 3982 if (!in_task() || !(gfp_mask & __GFP_DIRECT_RECLAIM)) 3983 filter &= ~SHOW_MEM_FILTER_NODES; 3984 3985 __show_mem(filter, nodemask, gfp_zone(gfp_mask)); 3986 } 3987 3988 void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...) 3989 { 3990 struct va_format vaf; 3991 va_list args; 3992 static DEFINE_RATELIMIT_STATE(nopage_rs, 10*HZ, 1); 3993 3994 if ((gfp_mask & __GFP_NOWARN) || 3995 !__ratelimit(&nopage_rs) || 3996 ((gfp_mask & __GFP_DMA) && !has_managed_dma())) 3997 return; 3998 3999 va_start(args, fmt); 4000 vaf.fmt = fmt; 4001 vaf.va = &args; 4002 pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl", 4003 current->comm, &vaf, gfp_mask, &gfp_mask, 4004 nodemask_pr_args(nodemask)); 4005 va_end(args); 4006 4007 cpuset_print_current_mems_allowed(); 4008 pr_cont("\n"); 4009 dump_stack(); 4010 warn_alloc_show_mem(gfp_mask, nodemask); 4011 } 4012 4013 static inline struct page * 4014 __alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order, 4015 unsigned int alloc_flags, 4016 const struct alloc_context *ac) 4017 { 4018 struct page *page; 4019 4020 page = get_page_from_freelist(gfp_mask, order, 4021 alloc_flags|ALLOC_CPUSET, ac); 4022 /* 4023 * fallback to ignore cpuset restriction if our nodes 4024 * are depleted 4025 */ 4026 if (!page) 4027 page = get_page_from_freelist(gfp_mask, order, 4028 alloc_flags, ac); 4029 return page; 4030 } 4031 4032 static inline struct page * 4033 __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, 4034 const struct alloc_context *ac, unsigned long *did_some_progress) 4035 { 4036 struct oom_control oc = { 4037 .zonelist = ac->zonelist, 4038 .nodemask = ac->nodemask, 4039 .memcg = NULL, 4040 .gfp_mask = gfp_mask, 4041 .order = order, 4042 }; 4043 struct page *page; 4044 4045 *did_some_progress = 0; 4046 4047 /* 4048 * Acquire the oom lock. If that fails, somebody else is 4049 * making progress for us. 4050 */ 4051 if (!mutex_trylock(&oom_lock)) { 4052 *did_some_progress = 1; 4053 schedule_timeout_uninterruptible(1); 4054 return NULL; 4055 } 4056 4057 /* 4058 * Go through the zonelist yet one more time, keep very high watermark 4059 * here, this is only to catch a parallel oom killing, we must fail if 4060 * we're still under heavy pressure. But make sure that this reclaim 4061 * attempt shall not depend on __GFP_DIRECT_RECLAIM && !__GFP_NORETRY 4062 * allocation which will never fail due to oom_lock already held. 4063 */ 4064 page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) & 4065 ~__GFP_DIRECT_RECLAIM, order, 4066 ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac); 4067 if (page) 4068 goto out; 4069 4070 /* Coredumps can quickly deplete all memory reserves */ 4071 if (current->flags & PF_DUMPCORE) 4072 goto out; 4073 /* The OOM killer will not help higher order allocs */ 4074 if (order > PAGE_ALLOC_COSTLY_ORDER) 4075 goto out; 4076 /* 4077 * We have already exhausted all our reclaim opportunities without any 4078 * success so it is time to admit defeat. We will skip the OOM killer 4079 * because it is very likely that the caller has a more reasonable 4080 * fallback than shooting a random task. 4081 * 4082 * The OOM killer may not free memory on a specific node. 4083 */ 4084 if (gfp_mask & (__GFP_RETRY_MAYFAIL | __GFP_THISNODE)) 4085 goto out; 4086 /* The OOM killer does not needlessly kill tasks for lowmem */ 4087 if (ac->highest_zoneidx < ZONE_NORMAL) 4088 goto out; 4089 if (pm_suspended_storage()) 4090 goto out; 4091 /* 4092 * XXX: GFP_NOFS allocations should rather fail than rely on 4093 * other request to make a forward progress. 4094 * We are in an unfortunate situation where out_of_memory cannot 4095 * do much for this context but let's try it to at least get 4096 * access to memory reserved if the current task is killed (see 4097 * out_of_memory). Once filesystems are ready to handle allocation 4098 * failures more gracefully we should just bail out here. 4099 */ 4100 4101 /* Exhausted what can be done so it's blame time */ 4102 if (out_of_memory(&oc) || 4103 WARN_ON_ONCE_GFP(gfp_mask & __GFP_NOFAIL, gfp_mask)) { 4104 *did_some_progress = 1; 4105 4106 /* 4107 * Help non-failing allocations by giving them access to memory 4108 * reserves 4109 */ 4110 if (gfp_mask & __GFP_NOFAIL) 4111 page = __alloc_pages_cpuset_fallback(gfp_mask, order, 4112 ALLOC_NO_WATERMARKS, ac); 4113 } 4114 out: 4115 mutex_unlock(&oom_lock); 4116 return page; 4117 } 4118 4119 /* 4120 * Maximum number of compaction retries with a progress before OOM 4121 * killer is consider as the only way to move forward. 4122 */ 4123 #define MAX_COMPACT_RETRIES 16 4124 4125 #ifdef CONFIG_COMPACTION 4126 /* Try memory compaction for high-order allocations before reclaim */ 4127 static struct page * 4128 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 4129 unsigned int alloc_flags, const struct alloc_context *ac, 4130 enum compact_priority prio, enum compact_result *compact_result) 4131 { 4132 struct page *page = NULL; 4133 unsigned long pflags; 4134 unsigned int noreclaim_flag; 4135 4136 if (!order) 4137 return NULL; 4138 4139 psi_memstall_enter(&pflags); 4140 delayacct_compact_start(); 4141 noreclaim_flag = memalloc_noreclaim_save(); 4142 4143 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, 4144 prio, &page); 4145 4146 memalloc_noreclaim_restore(noreclaim_flag); 4147 psi_memstall_leave(&pflags); 4148 delayacct_compact_end(); 4149 4150 if (*compact_result == COMPACT_SKIPPED) 4151 return NULL; 4152 /* 4153 * At least in one zone compaction wasn't deferred or skipped, so let's 4154 * count a compaction stall 4155 */ 4156 count_vm_event(COMPACTSTALL); 4157 4158 /* Prep a captured page if available */ 4159 if (page) 4160 prep_new_page(page, order, gfp_mask, alloc_flags); 4161 4162 /* Try get a page from the freelist if available */ 4163 if (!page) 4164 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4165 4166 if (page) { 4167 struct zone *zone = page_zone(page); 4168 4169 zone->compact_blockskip_flush = false; 4170 compaction_defer_reset(zone, order, true); 4171 count_vm_event(COMPACTSUCCESS); 4172 return page; 4173 } 4174 4175 /* 4176 * It's bad if compaction run occurs and fails. The most likely reason 4177 * is that pages exist, but not enough to satisfy watermarks. 4178 */ 4179 count_vm_event(COMPACTFAIL); 4180 4181 cond_resched(); 4182 4183 return NULL; 4184 } 4185 4186 static inline bool 4187 should_compact_retry(struct alloc_context *ac, int order, int alloc_flags, 4188 enum compact_result compact_result, 4189 enum compact_priority *compact_priority, 4190 int *compaction_retries) 4191 { 4192 int max_retries = MAX_COMPACT_RETRIES; 4193 int min_priority; 4194 bool ret = false; 4195 int retries = *compaction_retries; 4196 enum compact_priority priority = *compact_priority; 4197 4198 if (!order) 4199 return false; 4200 4201 if (fatal_signal_pending(current)) 4202 return false; 4203 4204 /* 4205 * Compaction was skipped due to a lack of free order-0 4206 * migration targets. Continue if reclaim can help. 4207 */ 4208 if (compact_result == COMPACT_SKIPPED) { 4209 ret = compaction_zonelist_suitable(ac, order, alloc_flags); 4210 goto out; 4211 } 4212 4213 /* 4214 * Compaction managed to coalesce some page blocks, but the 4215 * allocation failed presumably due to a race. Retry some. 4216 */ 4217 if (compact_result == COMPACT_SUCCESS) { 4218 /* 4219 * !costly requests are much more important than 4220 * __GFP_RETRY_MAYFAIL costly ones because they are de 4221 * facto nofail and invoke OOM killer to move on while 4222 * costly can fail and users are ready to cope with 4223 * that. 1/4 retries is rather arbitrary but we would 4224 * need much more detailed feedback from compaction to 4225 * make a better decision. 4226 */ 4227 if (order > PAGE_ALLOC_COSTLY_ORDER) 4228 max_retries /= 4; 4229 4230 if (++(*compaction_retries) <= max_retries) { 4231 ret = true; 4232 goto out; 4233 } 4234 } 4235 4236 /* 4237 * Compaction failed. Retry with increasing priority. 4238 */ 4239 min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ? 4240 MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY; 4241 4242 if (*compact_priority > min_priority) { 4243 (*compact_priority)--; 4244 *compaction_retries = 0; 4245 ret = true; 4246 } 4247 out: 4248 trace_compact_retry(order, priority, compact_result, retries, max_retries, ret); 4249 return ret; 4250 } 4251 #else 4252 static inline struct page * 4253 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, 4254 unsigned int alloc_flags, const struct alloc_context *ac, 4255 enum compact_priority prio, enum compact_result *compact_result) 4256 { 4257 *compact_result = COMPACT_SKIPPED; 4258 return NULL; 4259 } 4260 4261 static inline bool 4262 should_compact_retry(struct alloc_context *ac, int order, int alloc_flags, 4263 enum compact_result compact_result, 4264 enum compact_priority *compact_priority, 4265 int *compaction_retries) 4266 { 4267 struct zone *zone; 4268 struct zoneref *z; 4269 4270 if (!order || order > PAGE_ALLOC_COSTLY_ORDER) 4271 return false; 4272 4273 /* 4274 * There are setups with compaction disabled which would prefer to loop 4275 * inside the allocator rather than hit the oom killer prematurely. 4276 * Let's give them a good hope and keep retrying while the order-0 4277 * watermarks are OK. 4278 */ 4279 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, 4280 ac->highest_zoneidx, ac->nodemask) { 4281 if (zone_watermark_ok(zone, 0, min_wmark_pages(zone), 4282 ac->highest_zoneidx, alloc_flags)) 4283 return true; 4284 } 4285 return false; 4286 } 4287 #endif /* CONFIG_COMPACTION */ 4288 4289 #ifdef CONFIG_LOCKDEP 4290 static struct lockdep_map __fs_reclaim_map = 4291 STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map); 4292 4293 static bool __need_reclaim(gfp_t gfp_mask) 4294 { 4295 /* no reclaim without waiting on it */ 4296 if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) 4297 return false; 4298 4299 /* this guy won't enter reclaim */ 4300 if (current->flags & PF_MEMALLOC) 4301 return false; 4302 4303 if (gfp_mask & __GFP_NOLOCKDEP) 4304 return false; 4305 4306 return true; 4307 } 4308 4309 void __fs_reclaim_acquire(unsigned long ip) 4310 { 4311 lock_acquire_exclusive(&__fs_reclaim_map, 0, 0, NULL, ip); 4312 } 4313 4314 void __fs_reclaim_release(unsigned long ip) 4315 { 4316 lock_release(&__fs_reclaim_map, ip); 4317 } 4318 4319 void fs_reclaim_acquire(gfp_t gfp_mask) 4320 { 4321 gfp_mask = current_gfp_context(gfp_mask); 4322 4323 if (__need_reclaim(gfp_mask)) { 4324 if (gfp_mask & __GFP_FS) 4325 __fs_reclaim_acquire(_RET_IP_); 4326 4327 #ifdef CONFIG_MMU_NOTIFIER 4328 lock_map_acquire(&__mmu_notifier_invalidate_range_start_map); 4329 lock_map_release(&__mmu_notifier_invalidate_range_start_map); 4330 #endif 4331 4332 } 4333 } 4334 EXPORT_SYMBOL_GPL(fs_reclaim_acquire); 4335 4336 void fs_reclaim_release(gfp_t gfp_mask) 4337 { 4338 gfp_mask = current_gfp_context(gfp_mask); 4339 4340 if (__need_reclaim(gfp_mask)) { 4341 if (gfp_mask & __GFP_FS) 4342 __fs_reclaim_release(_RET_IP_); 4343 } 4344 } 4345 EXPORT_SYMBOL_GPL(fs_reclaim_release); 4346 #endif 4347 4348 /* 4349 * Zonelists may change due to hotplug during allocation. Detect when zonelists 4350 * have been rebuilt so allocation retries. Reader side does not lock and 4351 * retries the allocation if zonelist changes. Writer side is protected by the 4352 * embedded spin_lock. 4353 */ 4354 static DEFINE_SEQLOCK(zonelist_update_seq); 4355 4356 static unsigned int zonelist_iter_begin(void) 4357 { 4358 if (IS_ENABLED(CONFIG_MEMORY_HOTREMOVE)) 4359 return read_seqbegin(&zonelist_update_seq); 4360 4361 return 0; 4362 } 4363 4364 static unsigned int check_retry_zonelist(unsigned int seq) 4365 { 4366 if (IS_ENABLED(CONFIG_MEMORY_HOTREMOVE)) 4367 return read_seqretry(&zonelist_update_seq, seq); 4368 4369 return seq; 4370 } 4371 4372 /* Perform direct synchronous page reclaim */ 4373 static unsigned long 4374 __perform_reclaim(gfp_t gfp_mask, unsigned int order, 4375 const struct alloc_context *ac) 4376 { 4377 unsigned int noreclaim_flag; 4378 unsigned long progress; 4379 4380 cond_resched(); 4381 4382 /* We now go into synchronous reclaim */ 4383 cpuset_memory_pressure_bump(); 4384 fs_reclaim_acquire(gfp_mask); 4385 noreclaim_flag = memalloc_noreclaim_save(); 4386 4387 progress = try_to_free_pages(ac->zonelist, order, gfp_mask, 4388 ac->nodemask); 4389 4390 memalloc_noreclaim_restore(noreclaim_flag); 4391 fs_reclaim_release(gfp_mask); 4392 4393 cond_resched(); 4394 4395 return progress; 4396 } 4397 4398 /* The really slow allocator path where we enter direct reclaim */ 4399 static inline struct page * 4400 __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, 4401 unsigned int alloc_flags, const struct alloc_context *ac, 4402 unsigned long *did_some_progress) 4403 { 4404 struct page *page = NULL; 4405 unsigned long pflags; 4406 bool drained = false; 4407 4408 psi_memstall_enter(&pflags); 4409 *did_some_progress = __perform_reclaim(gfp_mask, order, ac); 4410 if (unlikely(!(*did_some_progress))) 4411 goto out; 4412 4413 retry: 4414 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4415 4416 /* 4417 * If an allocation failed after direct reclaim, it could be because 4418 * pages are pinned on the per-cpu lists or in high alloc reserves. 4419 * Shrink them and try again 4420 */ 4421 if (!page && !drained) { 4422 unreserve_highatomic_pageblock(ac, false); 4423 drain_all_pages(NULL); 4424 drained = true; 4425 goto retry; 4426 } 4427 out: 4428 psi_memstall_leave(&pflags); 4429 4430 return page; 4431 } 4432 4433 static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask, 4434 const struct alloc_context *ac) 4435 { 4436 struct zoneref *z; 4437 struct zone *zone; 4438 pg_data_t *last_pgdat = NULL; 4439 enum zone_type highest_zoneidx = ac->highest_zoneidx; 4440 unsigned int reclaim_order; 4441 4442 if (defrag_mode) 4443 reclaim_order = max(order, pageblock_order); 4444 else 4445 reclaim_order = order; 4446 4447 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, highest_zoneidx, 4448 ac->nodemask) { 4449 if (!managed_zone(zone)) 4450 continue; 4451 if (last_pgdat == zone->zone_pgdat) 4452 continue; 4453 wakeup_kswapd(zone, gfp_mask, reclaim_order, highest_zoneidx); 4454 last_pgdat = zone->zone_pgdat; 4455 } 4456 } 4457 4458 static inline unsigned int 4459 gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order) 4460 { 4461 unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; 4462 4463 /* 4464 * __GFP_HIGH is assumed to be the same as ALLOC_MIN_RESERVE 4465 * and __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD 4466 * to save two branches. 4467 */ 4468 BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_MIN_RESERVE); 4469 BUILD_BUG_ON(__GFP_KSWAPD_RECLAIM != (__force gfp_t) ALLOC_KSWAPD); 4470 4471 /* 4472 * The caller may dip into page reserves a bit more if the caller 4473 * cannot run direct reclaim, or if the caller has realtime scheduling 4474 * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will 4475 * set both ALLOC_NON_BLOCK and ALLOC_MIN_RESERVE(__GFP_HIGH). 4476 */ 4477 alloc_flags |= (__force int) 4478 (gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM)); 4479 4480 if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) { 4481 /* 4482 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even 4483 * if it can't schedule. 4484 */ 4485 if (!(gfp_mask & __GFP_NOMEMALLOC)) { 4486 alloc_flags |= ALLOC_NON_BLOCK; 4487 4488 if (order > 0 && (alloc_flags & ALLOC_MIN_RESERVE)) 4489 alloc_flags |= ALLOC_HIGHATOMIC; 4490 } 4491 4492 /* 4493 * Ignore cpuset mems for non-blocking __GFP_HIGH (probably 4494 * GFP_ATOMIC) rather than fail, see the comment for 4495 * cpuset_current_node_allowed(). 4496 */ 4497 if (alloc_flags & ALLOC_MIN_RESERVE) 4498 alloc_flags &= ~ALLOC_CPUSET; 4499 } else if (unlikely(rt_or_dl_task(current)) && in_task()) 4500 alloc_flags |= ALLOC_MIN_RESERVE; 4501 4502 alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, alloc_flags); 4503 4504 if (defrag_mode) 4505 alloc_flags |= ALLOC_NOFRAGMENT; 4506 4507 return alloc_flags; 4508 } 4509 4510 static bool oom_reserves_allowed(struct task_struct *tsk) 4511 { 4512 if (!tsk_is_oom_victim(tsk)) 4513 return false; 4514 4515 /* 4516 * !MMU doesn't have oom reaper so give access to memory reserves 4517 * only to the thread with TIF_MEMDIE set 4518 */ 4519 if (!IS_ENABLED(CONFIG_MMU) && !test_thread_flag(TIF_MEMDIE)) 4520 return false; 4521 4522 return true; 4523 } 4524 4525 /* 4526 * Distinguish requests which really need access to full memory 4527 * reserves from oom victims which can live with a portion of it 4528 */ 4529 static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask) 4530 { 4531 if (unlikely(gfp_mask & __GFP_NOMEMALLOC)) 4532 return 0; 4533 if (gfp_mask & __GFP_MEMALLOC) 4534 return ALLOC_NO_WATERMARKS; 4535 if (in_serving_softirq() && (current->flags & PF_MEMALLOC)) 4536 return ALLOC_NO_WATERMARKS; 4537 if (!in_interrupt()) { 4538 if (current->flags & PF_MEMALLOC) 4539 return ALLOC_NO_WATERMARKS; 4540 else if (oom_reserves_allowed(current)) 4541 return ALLOC_OOM; 4542 } 4543 4544 return 0; 4545 } 4546 4547 bool gfp_pfmemalloc_allowed(gfp_t gfp_mask) 4548 { 4549 return !!__gfp_pfmemalloc_flags(gfp_mask); 4550 } 4551 4552 /* 4553 * Checks whether it makes sense to retry the reclaim to make a forward progress 4554 * for the given allocation request. 4555 * 4556 * We give up when we either have tried MAX_RECLAIM_RETRIES in a row 4557 * without success, or when we couldn't even meet the watermark if we 4558 * reclaimed all remaining pages on the LRU lists. 4559 * 4560 * Returns true if a retry is viable or false to enter the oom path. 4561 */ 4562 static inline bool 4563 should_reclaim_retry(gfp_t gfp_mask, unsigned order, 4564 struct alloc_context *ac, int alloc_flags, 4565 bool did_some_progress, int *no_progress_loops) 4566 { 4567 struct zone *zone; 4568 struct zoneref *z; 4569 bool ret = false; 4570 4571 /* 4572 * Costly allocations might have made a progress but this doesn't mean 4573 * their order will become available due to high fragmentation so 4574 * always increment the no progress counter for them 4575 */ 4576 if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER) 4577 *no_progress_loops = 0; 4578 else 4579 (*no_progress_loops)++; 4580 4581 if (*no_progress_loops > MAX_RECLAIM_RETRIES) 4582 goto out; 4583 4584 4585 /* 4586 * Keep reclaiming pages while there is a chance this will lead 4587 * somewhere. If none of the target zones can satisfy our allocation 4588 * request even if all reclaimable pages are considered then we are 4589 * screwed and have to go OOM. 4590 */ 4591 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, 4592 ac->highest_zoneidx, ac->nodemask) { 4593 unsigned long available; 4594 unsigned long reclaimable; 4595 unsigned long min_wmark = min_wmark_pages(zone); 4596 bool wmark; 4597 4598 if (cpusets_enabled() && 4599 (alloc_flags & ALLOC_CPUSET) && 4600 !__cpuset_zone_allowed(zone, gfp_mask)) 4601 continue; 4602 4603 available = reclaimable = zone_reclaimable_pages(zone); 4604 available += zone_page_state_snapshot(zone, NR_FREE_PAGES); 4605 4606 /* 4607 * Would the allocation succeed if we reclaimed all 4608 * reclaimable pages? 4609 */ 4610 wmark = __zone_watermark_ok(zone, order, min_wmark, 4611 ac->highest_zoneidx, alloc_flags, available); 4612 trace_reclaim_retry_zone(z, order, reclaimable, 4613 available, min_wmark, *no_progress_loops, wmark); 4614 if (wmark) { 4615 ret = true; 4616 break; 4617 } 4618 } 4619 4620 /* 4621 * Memory allocation/reclaim might be called from a WQ context and the 4622 * current implementation of the WQ concurrency control doesn't 4623 * recognize that a particular WQ is congested if the worker thread is 4624 * looping without ever sleeping. Therefore we have to do a short sleep 4625 * here rather than calling cond_resched(). 4626 */ 4627 if (current->flags & PF_WQ_WORKER) 4628 schedule_timeout_uninterruptible(1); 4629 else 4630 cond_resched(); 4631 out: 4632 /* Before OOM, exhaust highatomic_reserve */ 4633 if (!ret) 4634 return unreserve_highatomic_pageblock(ac, true); 4635 4636 return ret; 4637 } 4638 4639 static inline bool 4640 check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac) 4641 { 4642 /* 4643 * It's possible that cpuset's mems_allowed and the nodemask from 4644 * mempolicy don't intersect. This should be normally dealt with by 4645 * policy_nodemask(), but it's possible to race with cpuset update in 4646 * such a way the check therein was true, and then it became false 4647 * before we got our cpuset_mems_cookie here. 4648 * This assumes that for all allocations, ac->nodemask can come only 4649 * from MPOL_BIND mempolicy (whose documented semantics is to be ignored 4650 * when it does not intersect with the cpuset restrictions) or the 4651 * caller can deal with a violated nodemask. 4652 */ 4653 if (cpusets_enabled() && ac->nodemask && 4654 !cpuset_nodemask_valid_mems_allowed(ac->nodemask)) { 4655 ac->nodemask = NULL; 4656 return true; 4657 } 4658 4659 /* 4660 * When updating a task's mems_allowed or mempolicy nodemask, it is 4661 * possible to race with parallel threads in such a way that our 4662 * allocation can fail while the mask is being updated. If we are about 4663 * to fail, check if the cpuset changed during allocation and if so, 4664 * retry. 4665 */ 4666 if (read_mems_allowed_retry(cpuset_mems_cookie)) 4667 return true; 4668 4669 return false; 4670 } 4671 4672 static inline struct page * 4673 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, 4674 struct alloc_context *ac) 4675 { 4676 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM; 4677 bool can_compact = gfp_compaction_allowed(gfp_mask); 4678 bool nofail = gfp_mask & __GFP_NOFAIL; 4679 const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER; 4680 struct page *page = NULL; 4681 unsigned int alloc_flags; 4682 unsigned long did_some_progress; 4683 enum compact_priority compact_priority; 4684 enum compact_result compact_result; 4685 int compaction_retries; 4686 int no_progress_loops; 4687 unsigned int cpuset_mems_cookie; 4688 unsigned int zonelist_iter_cookie; 4689 int reserve_flags; 4690 4691 if (unlikely(nofail)) { 4692 /* 4693 * We most definitely don't want callers attempting to 4694 * allocate greater than order-1 page units with __GFP_NOFAIL. 4695 */ 4696 WARN_ON_ONCE(order > 1); 4697 /* 4698 * Also we don't support __GFP_NOFAIL without __GFP_DIRECT_RECLAIM, 4699 * otherwise, we may result in lockup. 4700 */ 4701 WARN_ON_ONCE(!can_direct_reclaim); 4702 /* 4703 * PF_MEMALLOC request from this context is rather bizarre 4704 * because we cannot reclaim anything and only can loop waiting 4705 * for somebody to do a work for us. 4706 */ 4707 WARN_ON_ONCE(current->flags & PF_MEMALLOC); 4708 } 4709 4710 restart: 4711 compaction_retries = 0; 4712 no_progress_loops = 0; 4713 compact_result = COMPACT_SKIPPED; 4714 compact_priority = DEF_COMPACT_PRIORITY; 4715 cpuset_mems_cookie = read_mems_allowed_begin(); 4716 zonelist_iter_cookie = zonelist_iter_begin(); 4717 4718 /* 4719 * The fast path uses conservative alloc_flags to succeed only until 4720 * kswapd needs to be woken up, and to avoid the cost of setting up 4721 * alloc_flags precisely. So we do that now. 4722 */ 4723 alloc_flags = gfp_to_alloc_flags(gfp_mask, order); 4724 4725 /* 4726 * We need to recalculate the starting point for the zonelist iterator 4727 * because we might have used different nodemask in the fast path, or 4728 * there was a cpuset modification and we are retrying - otherwise we 4729 * could end up iterating over non-eligible zones endlessly. 4730 */ 4731 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4732 ac->highest_zoneidx, ac->nodemask); 4733 if (!zonelist_zone(ac->preferred_zoneref)) 4734 goto nopage; 4735 4736 /* 4737 * Check for insane configurations where the cpuset doesn't contain 4738 * any suitable zone to satisfy the request - e.g. non-movable 4739 * GFP_HIGHUSER allocations from MOVABLE nodes only. 4740 */ 4741 if (cpusets_insane_config() && (gfp_mask & __GFP_HARDWALL)) { 4742 struct zoneref *z = first_zones_zonelist(ac->zonelist, 4743 ac->highest_zoneidx, 4744 &cpuset_current_mems_allowed); 4745 if (!zonelist_zone(z)) 4746 goto nopage; 4747 } 4748 4749 if (alloc_flags & ALLOC_KSWAPD) 4750 wake_all_kswapds(order, gfp_mask, ac); 4751 4752 /* 4753 * The adjusted alloc_flags might result in immediate success, so try 4754 * that first 4755 */ 4756 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4757 if (page) 4758 goto got_pg; 4759 4760 /* 4761 * For costly allocations, try direct compaction first, as it's likely 4762 * that we have enough base pages and don't need to reclaim. For non- 4763 * movable high-order allocations, do that as well, as compaction will 4764 * try prevent permanent fragmentation by migrating from blocks of the 4765 * same migratetype. 4766 * Don't try this for allocations that are allowed to ignore 4767 * watermarks, as the ALLOC_NO_WATERMARKS attempt didn't yet happen. 4768 */ 4769 if (can_direct_reclaim && can_compact && 4770 (costly_order || 4771 (order > 0 && ac->migratetype != MIGRATE_MOVABLE)) 4772 && !gfp_pfmemalloc_allowed(gfp_mask)) { 4773 page = __alloc_pages_direct_compact(gfp_mask, order, 4774 alloc_flags, ac, 4775 INIT_COMPACT_PRIORITY, 4776 &compact_result); 4777 if (page) 4778 goto got_pg; 4779 4780 /* 4781 * Checks for costly allocations with __GFP_NORETRY, which 4782 * includes some THP page fault allocations 4783 */ 4784 if (costly_order && (gfp_mask & __GFP_NORETRY)) { 4785 /* 4786 * If allocating entire pageblock(s) and compaction 4787 * failed because all zones are below low watermarks 4788 * or is prohibited because it recently failed at this 4789 * order, fail immediately unless the allocator has 4790 * requested compaction and reclaim retry. 4791 * 4792 * Reclaim is 4793 * - potentially very expensive because zones are far 4794 * below their low watermarks or this is part of very 4795 * bursty high order allocations, 4796 * - not guaranteed to help because isolate_freepages() 4797 * may not iterate over freed pages as part of its 4798 * linear scan, and 4799 * - unlikely to make entire pageblocks free on its 4800 * own. 4801 */ 4802 if (compact_result == COMPACT_SKIPPED || 4803 compact_result == COMPACT_DEFERRED) 4804 goto nopage; 4805 4806 /* 4807 * THP page faults may attempt local node only first, 4808 * but are then allowed to only compact, not reclaim, 4809 * see alloc_pages_mpol(). 4810 * 4811 * Compaction can fail for other reasons than those 4812 * checked above and we don't want such THP allocations 4813 * to put reclaim pressure on a single node in a 4814 * situation where other nodes might have plenty of 4815 * available memory. 4816 */ 4817 if (gfp_mask & __GFP_THISNODE) 4818 goto nopage; 4819 4820 /* 4821 * Looks like reclaim/compaction is worth trying, but 4822 * sync compaction could be very expensive, so keep 4823 * using async compaction. 4824 */ 4825 compact_priority = INIT_COMPACT_PRIORITY; 4826 } 4827 } 4828 4829 retry: 4830 /* 4831 * Deal with possible cpuset update races or zonelist updates to avoid 4832 * infinite retries. 4833 */ 4834 if (check_retry_cpuset(cpuset_mems_cookie, ac) || 4835 check_retry_zonelist(zonelist_iter_cookie)) 4836 goto restart; 4837 4838 /* Ensure kswapd doesn't accidentally go to sleep as long as we loop */ 4839 if (alloc_flags & ALLOC_KSWAPD) 4840 wake_all_kswapds(order, gfp_mask, ac); 4841 4842 reserve_flags = __gfp_pfmemalloc_flags(gfp_mask); 4843 if (reserve_flags) 4844 alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, reserve_flags) | 4845 (alloc_flags & ALLOC_KSWAPD); 4846 4847 /* 4848 * Reset the nodemask and zonelist iterators if memory policies can be 4849 * ignored. These allocations are high priority and system rather than 4850 * user oriented. 4851 */ 4852 if (!(alloc_flags & ALLOC_CPUSET) || reserve_flags) { 4853 ac->nodemask = NULL; 4854 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 4855 ac->highest_zoneidx, ac->nodemask); 4856 } 4857 4858 /* Attempt with potentially adjusted zonelist and alloc_flags */ 4859 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); 4860 if (page) 4861 goto got_pg; 4862 4863 /* Caller is not willing to reclaim, we can't balance anything */ 4864 if (!can_direct_reclaim) 4865 goto nopage; 4866 4867 /* Avoid recursion of direct reclaim */ 4868 if (current->flags & PF_MEMALLOC) 4869 goto nopage; 4870 4871 /* Try direct reclaim and then allocating */ 4872 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, 4873 &did_some_progress); 4874 if (page) 4875 goto got_pg; 4876 4877 /* Try direct compaction and then allocating */ 4878 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac, 4879 compact_priority, &compact_result); 4880 if (page) 4881 goto got_pg; 4882 4883 /* Do not loop if specifically requested */ 4884 if (gfp_mask & __GFP_NORETRY) 4885 goto nopage; 4886 4887 /* 4888 * Do not retry costly high order allocations unless they are 4889 * __GFP_RETRY_MAYFAIL and we can compact 4890 */ 4891 if (costly_order && (!can_compact || 4892 !(gfp_mask & __GFP_RETRY_MAYFAIL))) 4893 goto nopage; 4894 4895 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags, 4896 did_some_progress > 0, &no_progress_loops)) 4897 goto retry; 4898 4899 /* 4900 * It doesn't make any sense to retry for the compaction if the order-0 4901 * reclaim is not able to make any progress because the current 4902 * implementation of the compaction depends on the sufficient amount 4903 * of free memory (see __compaction_suitable) 4904 */ 4905 if (did_some_progress > 0 && can_compact && 4906 should_compact_retry(ac, order, alloc_flags, 4907 compact_result, &compact_priority, 4908 &compaction_retries)) 4909 goto retry; 4910 4911 /* Reclaim/compaction failed to prevent the fallback */ 4912 if (defrag_mode && (alloc_flags & ALLOC_NOFRAGMENT)) { 4913 alloc_flags &= ~ALLOC_NOFRAGMENT; 4914 goto retry; 4915 } 4916 4917 /* 4918 * Deal with possible cpuset update races or zonelist updates to avoid 4919 * a unnecessary OOM kill. 4920 */ 4921 if (check_retry_cpuset(cpuset_mems_cookie, ac) || 4922 check_retry_zonelist(zonelist_iter_cookie)) 4923 goto restart; 4924 4925 /* Reclaim has failed us, start killing things */ 4926 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress); 4927 if (page) 4928 goto got_pg; 4929 4930 /* Avoid allocations with no watermarks from looping endlessly */ 4931 if (tsk_is_oom_victim(current) && 4932 (alloc_flags & ALLOC_OOM || 4933 (gfp_mask & __GFP_NOMEMALLOC))) 4934 goto nopage; 4935 4936 /* Retry as long as the OOM killer is making progress */ 4937 if (did_some_progress) { 4938 no_progress_loops = 0; 4939 goto retry; 4940 } 4941 4942 nopage: 4943 /* 4944 * Deal with possible cpuset update races or zonelist updates to avoid 4945 * a unnecessary OOM kill. 4946 */ 4947 if (check_retry_cpuset(cpuset_mems_cookie, ac) || 4948 check_retry_zonelist(zonelist_iter_cookie)) 4949 goto restart; 4950 4951 /* 4952 * Make sure that __GFP_NOFAIL request doesn't leak out and make sure 4953 * we always retry 4954 */ 4955 if (unlikely(nofail)) { 4956 /* 4957 * Lacking direct_reclaim we can't do anything to reclaim memory, 4958 * we disregard these unreasonable nofail requests and still 4959 * return NULL 4960 */ 4961 if (!can_direct_reclaim) 4962 goto fail; 4963 4964 /* 4965 * Help non-failing allocations by giving some access to memory 4966 * reserves normally used for high priority non-blocking 4967 * allocations but do not use ALLOC_NO_WATERMARKS because this 4968 * could deplete whole memory reserves which would just make 4969 * the situation worse. 4970 */ 4971 page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_MIN_RESERVE, ac); 4972 if (page) 4973 goto got_pg; 4974 4975 cond_resched(); 4976 goto retry; 4977 } 4978 fail: 4979 warn_alloc(gfp_mask, ac->nodemask, 4980 "page allocation failure: order:%u", order); 4981 got_pg: 4982 return page; 4983 } 4984 4985 static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, 4986 int preferred_nid, nodemask_t *nodemask, 4987 struct alloc_context *ac, gfp_t *alloc_gfp, 4988 unsigned int *alloc_flags) 4989 { 4990 ac->highest_zoneidx = gfp_zone(gfp_mask); 4991 ac->zonelist = node_zonelist(preferred_nid, gfp_mask); 4992 ac->nodemask = nodemask; 4993 ac->migratetype = gfp_migratetype(gfp_mask); 4994 4995 if (cpusets_enabled()) { 4996 *alloc_gfp |= __GFP_HARDWALL; 4997 /* 4998 * When we are in the interrupt context, it is irrelevant 4999 * to the current task context. It means that any node ok. 5000 */ 5001 if (in_task() && !ac->nodemask) 5002 ac->nodemask = &cpuset_current_mems_allowed; 5003 else 5004 *alloc_flags |= ALLOC_CPUSET; 5005 } 5006 5007 might_alloc(gfp_mask); 5008 5009 /* 5010 * Don't invoke should_fail logic, since it may call 5011 * get_random_u32() and printk() which need to spin_lock. 5012 */ 5013 if (!(*alloc_flags & ALLOC_TRYLOCK) && 5014 should_fail_alloc_page(gfp_mask, order)) 5015 return false; 5016 5017 *alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, *alloc_flags); 5018 5019 /* Dirty zone balancing only done in the fast path */ 5020 ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE); 5021 5022 /* 5023 * The preferred zone is used for statistics but crucially it is 5024 * also used as the starting point for the zonelist iterator. It 5025 * may get reset for allocations that ignore memory policies. 5026 */ 5027 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, 5028 ac->highest_zoneidx, ac->nodemask); 5029 5030 return true; 5031 } 5032 5033 /* 5034 * __alloc_pages_bulk - Allocate a number of order-0 pages to an array 5035 * @gfp: GFP flags for the allocation 5036 * @preferred_nid: The preferred NUMA node ID to allocate from 5037 * @nodemask: Set of nodes to allocate from, may be NULL 5038 * @nr_pages: The number of pages desired in the array 5039 * @page_array: Array to store the pages 5040 * 5041 * This is a batched version of the page allocator that attempts to 5042 * allocate nr_pages quickly. Pages are added to the page_array. 5043 * 5044 * Note that only NULL elements are populated with pages and nr_pages 5045 * is the maximum number of pages that will be stored in the array. 5046 * 5047 * Returns the number of pages in the array. 5048 */ 5049 unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid, 5050 nodemask_t *nodemask, int nr_pages, 5051 struct page **page_array) 5052 { 5053 struct page *page; 5054 unsigned long __maybe_unused UP_flags; 5055 struct zone *zone; 5056 struct zoneref *z; 5057 struct per_cpu_pages *pcp; 5058 struct list_head *pcp_list; 5059 struct alloc_context ac; 5060 gfp_t alloc_gfp; 5061 unsigned int alloc_flags = ALLOC_WMARK_LOW; 5062 int nr_populated = 0, nr_account = 0; 5063 5064 /* 5065 * Skip populated array elements to determine if any pages need 5066 * to be allocated before disabling IRQs. 5067 */ 5068 while (nr_populated < nr_pages && page_array[nr_populated]) 5069 nr_populated++; 5070 5071 /* No pages requested? */ 5072 if (unlikely(nr_pages <= 0)) 5073 goto out; 5074 5075 /* Already populated array? */ 5076 if (unlikely(nr_pages - nr_populated == 0)) 5077 goto out; 5078 5079 /* Bulk allocator does not support memcg accounting. */ 5080 if (memcg_kmem_online() && (gfp & __GFP_ACCOUNT)) 5081 goto failed; 5082 5083 /* Use the single page allocator for one page. */ 5084 if (nr_pages - nr_populated == 1) 5085 goto failed; 5086 5087 #ifdef CONFIG_PAGE_OWNER 5088 /* 5089 * PAGE_OWNER may recurse into the allocator to allocate space to 5090 * save the stack with pagesets.lock held. Releasing/reacquiring 5091 * removes much of the performance benefit of bulk allocation so 5092 * force the caller to allocate one page at a time as it'll have 5093 * similar performance to added complexity to the bulk allocator. 5094 */ 5095 if (static_branch_unlikely(&page_owner_inited)) 5096 goto failed; 5097 #endif 5098 5099 /* May set ALLOC_NOFRAGMENT, fragmentation will return 1 page. */ 5100 gfp &= gfp_allowed_mask; 5101 alloc_gfp = gfp; 5102 if (!prepare_alloc_pages(gfp, 0, preferred_nid, nodemask, &ac, &alloc_gfp, &alloc_flags)) 5103 goto out; 5104 gfp = alloc_gfp; 5105 5106 /* Find an allowed local zone that meets the low watermark. */ 5107 z = ac.preferred_zoneref; 5108 for_next_zone_zonelist_nodemask(zone, z, ac.highest_zoneidx, ac.nodemask) { 5109 unsigned long mark; 5110 5111 if (cpusets_enabled() && (alloc_flags & ALLOC_CPUSET) && 5112 !__cpuset_zone_allowed(zone, gfp)) { 5113 continue; 5114 } 5115 5116 if (nr_online_nodes > 1 && zone != zonelist_zone(ac.preferred_zoneref) && 5117 zone_to_nid(zone) != zonelist_node_idx(ac.preferred_zoneref)) { 5118 goto failed; 5119 } 5120 5121 cond_accept_memory(zone, 0, alloc_flags); 5122 retry_this_zone: 5123 mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK) + nr_pages; 5124 if (zone_watermark_fast(zone, 0, mark, 5125 zonelist_zone_idx(ac.preferred_zoneref), 5126 alloc_flags, gfp)) { 5127 break; 5128 } 5129 5130 if (cond_accept_memory(zone, 0, alloc_flags)) 5131 goto retry_this_zone; 5132 5133 /* Try again if zone has deferred pages */ 5134 if (deferred_pages_enabled()) { 5135 if (_deferred_grow_zone(zone, 0)) 5136 goto retry_this_zone; 5137 } 5138 } 5139 5140 /* 5141 * If there are no allowed local zones that meets the watermarks then 5142 * try to allocate a single page and reclaim if necessary. 5143 */ 5144 if (unlikely(!zone)) 5145 goto failed; 5146 5147 /* spin_trylock may fail due to a parallel drain or IRQ reentrancy. */ 5148 pcp_trylock_prepare(UP_flags); 5149 pcp = pcp_spin_trylock(zone->per_cpu_pageset); 5150 if (!pcp) 5151 goto failed_irq; 5152 5153 /* Attempt the batch allocation */ 5154 pcp_list = &pcp->lists[order_to_pindex(ac.migratetype, 0)]; 5155 while (nr_populated < nr_pages) { 5156 5157 /* Skip existing pages */ 5158 if (page_array[nr_populated]) { 5159 nr_populated++; 5160 continue; 5161 } 5162 5163 page = __rmqueue_pcplist(zone, 0, ac.migratetype, alloc_flags, 5164 pcp, pcp_list); 5165 if (unlikely(!page)) { 5166 /* Try and allocate at least one page */ 5167 if (!nr_account) { 5168 pcp_spin_unlock(pcp); 5169 goto failed_irq; 5170 } 5171 break; 5172 } 5173 nr_account++; 5174 5175 prep_new_page(page, 0, gfp, 0); 5176 set_page_refcounted(page); 5177 page_array[nr_populated++] = page; 5178 } 5179 5180 pcp_spin_unlock(pcp); 5181 pcp_trylock_finish(UP_flags); 5182 5183 __count_zid_vm_events(PGALLOC, zone_idx(zone), nr_account); 5184 zone_statistics(zonelist_zone(ac.preferred_zoneref), zone, nr_account); 5185 5186 out: 5187 return nr_populated; 5188 5189 failed_irq: 5190 pcp_trylock_finish(UP_flags); 5191 5192 failed: 5193 page = __alloc_pages_noprof(gfp, 0, preferred_nid, nodemask); 5194 if (page) 5195 page_array[nr_populated++] = page; 5196 goto out; 5197 } 5198 EXPORT_SYMBOL_GPL(alloc_pages_bulk_noprof); 5199 5200 /* 5201 * This is the 'heart' of the zoned buddy allocator. 5202 */ 5203 struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, 5204 int preferred_nid, nodemask_t *nodemask) 5205 { 5206 struct page *page; 5207 unsigned int alloc_flags = ALLOC_WMARK_LOW; 5208 gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */ 5209 struct alloc_context ac = { }; 5210 5211 /* 5212 * There are several places where we assume that the order value is sane 5213 * so bail out early if the request is out of bound. 5214 */ 5215 if (WARN_ON_ONCE_GFP(order > MAX_PAGE_ORDER, gfp)) 5216 return NULL; 5217 5218 gfp &= gfp_allowed_mask; 5219 /* 5220 * Apply scoped allocation constraints. This is mainly about GFP_NOFS 5221 * resp. GFP_NOIO which has to be inherited for all allocation requests 5222 * from a particular context which has been marked by 5223 * memalloc_no{fs,io}_{save,restore}. And PF_MEMALLOC_PIN which ensures 5224 * movable zones are not used during allocation. 5225 */ 5226 gfp = current_gfp_context(gfp); 5227 alloc_gfp = gfp; 5228 if (!prepare_alloc_pages(gfp, order, preferred_nid, nodemask, &ac, 5229 &alloc_gfp, &alloc_flags)) 5230 return NULL; 5231 5232 /* 5233 * Forbid the first pass from falling back to types that fragment 5234 * memory until all local zones are considered. 5235 */ 5236 alloc_flags |= alloc_flags_nofragment(zonelist_zone(ac.preferred_zoneref), gfp); 5237 5238 /* First allocation attempt */ 5239 page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac); 5240 if (likely(page)) 5241 goto out; 5242 5243 alloc_gfp = gfp; 5244 ac.spread_dirty_pages = false; 5245 5246 /* 5247 * Restore the original nodemask if it was potentially replaced with 5248 * &cpuset_current_mems_allowed to optimize the fast-path attempt. 5249 */ 5250 ac.nodemask = nodemask; 5251 5252 page = __alloc_pages_slowpath(alloc_gfp, order, &ac); 5253 5254 out: 5255 if (memcg_kmem_online() && (gfp & __GFP_ACCOUNT) && page && 5256 unlikely(__memcg_kmem_charge_page(page, gfp, order) != 0)) { 5257 free_frozen_pages(page, order); 5258 page = NULL; 5259 } 5260 5261 trace_mm_page_alloc(page, order, alloc_gfp, ac.migratetype); 5262 kmsan_alloc_page(page, order, alloc_gfp); 5263 5264 return page; 5265 } 5266 EXPORT_SYMBOL(__alloc_frozen_pages_noprof); 5267 5268 struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order, 5269 int preferred_nid, nodemask_t *nodemask) 5270 { 5271 struct page *page; 5272 5273 page = __alloc_frozen_pages_noprof(gfp, order, preferred_nid, nodemask); 5274 if (page) 5275 set_page_refcounted(page); 5276 return page; 5277 } 5278 EXPORT_SYMBOL(__alloc_pages_noprof); 5279 5280 struct folio *__folio_alloc_noprof(gfp_t gfp, unsigned int order, int preferred_nid, 5281 nodemask_t *nodemask) 5282 { 5283 struct page *page = __alloc_pages_noprof(gfp | __GFP_COMP, order, 5284 preferred_nid, nodemask); 5285 return page_rmappable_folio(page); 5286 } 5287 EXPORT_SYMBOL(__folio_alloc_noprof); 5288 5289 /* 5290 * Common helper functions. Never use with __GFP_HIGHMEM because the returned 5291 * address cannot represent highmem pages. Use alloc_pages and then kmap if 5292 * you need to access high mem. 5293 */ 5294 unsigned long get_free_pages_noprof(gfp_t gfp_mask, unsigned int order) 5295 { 5296 struct page *page; 5297 5298 page = alloc_pages_noprof(gfp_mask & ~__GFP_HIGHMEM, order); 5299 if (!page) 5300 return 0; 5301 return (unsigned long) page_address(page); 5302 } 5303 EXPORT_SYMBOL(get_free_pages_noprof); 5304 5305 unsigned long get_zeroed_page_noprof(gfp_t gfp_mask) 5306 { 5307 return get_free_pages_noprof(gfp_mask | __GFP_ZERO, 0); 5308 } 5309 EXPORT_SYMBOL(get_zeroed_page_noprof); 5310 5311 static void ___free_pages(struct page *page, unsigned int order, 5312 fpi_t fpi_flags) 5313 { 5314 /* get PageHead before we drop reference */ 5315 int head = PageHead(page); 5316 /* get alloc tag in case the page is released by others */ 5317 struct alloc_tag *tag = pgalloc_tag_get(page); 5318 5319 if (put_page_testzero(page)) 5320 __free_frozen_pages(page, order, fpi_flags); 5321 else if (!head) { 5322 pgalloc_tag_sub_pages(tag, (1 << order) - 1); 5323 while (order-- > 0) { 5324 /* 5325 * The "tail" pages of this non-compound high-order 5326 * page will have no code tags, so to avoid warnings 5327 * mark them as empty. 5328 */ 5329 clear_page_tag_ref(page + (1 << order)); 5330 __free_frozen_pages(page + (1 << order), order, 5331 fpi_flags); 5332 } 5333 } 5334 } 5335 5336 /** 5337 * __free_pages - Free pages allocated with alloc_pages(). 5338 * @page: The page pointer returned from alloc_pages(). 5339 * @order: The order of the allocation. 5340 * 5341 * This function can free multi-page allocations that are not compound 5342 * pages. It does not check that the @order passed in matches that of 5343 * the allocation, so it is easy to leak memory. Freeing more memory 5344 * than was allocated will probably emit a warning. 5345 * 5346 * If the last reference to this page is speculative, it will be released 5347 * by put_page() which only frees the first page of a non-compound 5348 * allocation. To prevent the remaining pages from being leaked, we free 5349 * the subsequent pages here. If you want to use the page's reference 5350 * count to decide when to free the allocation, you should allocate a 5351 * compound page, and use put_page() instead of __free_pages(). 5352 * 5353 * Context: May be called in interrupt context or while holding a normal 5354 * spinlock, but not in NMI context or while holding a raw spinlock. 5355 */ 5356 void __free_pages(struct page *page, unsigned int order) 5357 { 5358 ___free_pages(page, order, FPI_NONE); 5359 } 5360 EXPORT_SYMBOL(__free_pages); 5361 5362 /* 5363 * Can be called while holding raw_spin_lock or from IRQ and NMI for any 5364 * page type (not only those that came from alloc_pages_nolock) 5365 */ 5366 void free_pages_nolock(struct page *page, unsigned int order) 5367 { 5368 ___free_pages(page, order, FPI_TRYLOCK); 5369 } 5370 5371 /** 5372 * free_pages - Free pages allocated with __get_free_pages(). 5373 * @addr: The virtual address tied to a page returned from __get_free_pages(). 5374 * @order: The order of the allocation. 5375 * 5376 * This function behaves the same as __free_pages(). Use this function 5377 * to free pages when you only have a valid virtual address. If you have 5378 * the page, call __free_pages() instead. 5379 */ 5380 void free_pages(unsigned long addr, unsigned int order) 5381 { 5382 if (addr != 0) { 5383 VM_BUG_ON(!virt_addr_valid((void *)addr)); 5384 __free_pages(virt_to_page((void *)addr), order); 5385 } 5386 } 5387 5388 EXPORT_SYMBOL(free_pages); 5389 5390 static void *make_alloc_exact(unsigned long addr, unsigned int order, 5391 size_t size) 5392 { 5393 if (addr) { 5394 unsigned long nr = DIV_ROUND_UP(size, PAGE_SIZE); 5395 struct page *page = virt_to_page((void *)addr); 5396 struct page *last = page + nr; 5397 5398 split_page_owner(page, order, 0); 5399 pgalloc_tag_split(page_folio(page), order, 0); 5400 split_page_memcg(page, order); 5401 while (page < --last) 5402 set_page_refcounted(last); 5403 5404 last = page + (1UL << order); 5405 for (page += nr; page < last; page++) 5406 __free_pages_ok(page, 0, FPI_TO_TAIL); 5407 } 5408 return (void *)addr; 5409 } 5410 5411 /** 5412 * alloc_pages_exact - allocate an exact number physically-contiguous pages. 5413 * @size: the number of bytes to allocate 5414 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP 5415 * 5416 * This function is similar to alloc_pages(), except that it allocates the 5417 * minimum number of pages to satisfy the request. alloc_pages() can only 5418 * allocate memory in power-of-two pages. 5419 * 5420 * This function is also limited by MAX_PAGE_ORDER. 5421 * 5422 * Memory allocated by this function must be released by free_pages_exact(). 5423 * 5424 * Return: pointer to the allocated area or %NULL in case of error. 5425 */ 5426 void *alloc_pages_exact_noprof(size_t size, gfp_t gfp_mask) 5427 { 5428 unsigned int order = get_order(size); 5429 unsigned long addr; 5430 5431 if (WARN_ON_ONCE(gfp_mask & (__GFP_COMP | __GFP_HIGHMEM))) 5432 gfp_mask &= ~(__GFP_COMP | __GFP_HIGHMEM); 5433 5434 addr = get_free_pages_noprof(gfp_mask, order); 5435 return make_alloc_exact(addr, order, size); 5436 } 5437 EXPORT_SYMBOL(alloc_pages_exact_noprof); 5438 5439 /** 5440 * alloc_pages_exact_nid - allocate an exact number of physically-contiguous 5441 * pages on a node. 5442 * @nid: the preferred node ID where memory should be allocated 5443 * @size: the number of bytes to allocate 5444 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP 5445 * 5446 * Like alloc_pages_exact(), but try to allocate on node nid first before falling 5447 * back. 5448 * 5449 * Return: pointer to the allocated area or %NULL in case of error. 5450 */ 5451 void * __meminit alloc_pages_exact_nid_noprof(int nid, size_t size, gfp_t gfp_mask) 5452 { 5453 unsigned int order = get_order(size); 5454 struct page *p; 5455 5456 if (WARN_ON_ONCE(gfp_mask & (__GFP_COMP | __GFP_HIGHMEM))) 5457 gfp_mask &= ~(__GFP_COMP | __GFP_HIGHMEM); 5458 5459 p = alloc_pages_node_noprof(nid, gfp_mask, order); 5460 if (!p) 5461 return NULL; 5462 return make_alloc_exact((unsigned long)page_address(p), order, size); 5463 } 5464 5465 /** 5466 * free_pages_exact - release memory allocated via alloc_pages_exact() 5467 * @virt: the value returned by alloc_pages_exact. 5468 * @size: size of allocation, same value as passed to alloc_pages_exact(). 5469 * 5470 * Release the memory allocated by a previous call to alloc_pages_exact. 5471 */ 5472 void free_pages_exact(void *virt, size_t size) 5473 { 5474 unsigned long addr = (unsigned long)virt; 5475 unsigned long end = addr + PAGE_ALIGN(size); 5476 5477 while (addr < end) { 5478 free_page(addr); 5479 addr += PAGE_SIZE; 5480 } 5481 } 5482 EXPORT_SYMBOL(free_pages_exact); 5483 5484 /** 5485 * nr_free_zone_pages - count number of pages beyond high watermark 5486 * @offset: The zone index of the highest zone 5487 * 5488 * nr_free_zone_pages() counts the number of pages which are beyond the 5489 * high watermark within all zones at or below a given zone index. For each 5490 * zone, the number of pages is calculated as: 5491 * 5492 * nr_free_zone_pages = managed_pages - high_pages 5493 * 5494 * Return: number of pages beyond high watermark. 5495 */ 5496 static unsigned long nr_free_zone_pages(int offset) 5497 { 5498 struct zoneref *z; 5499 struct zone *zone; 5500 5501 /* Just pick one node, since fallback list is circular */ 5502 unsigned long sum = 0; 5503 5504 struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL); 5505 5506 for_each_zone_zonelist(zone, z, zonelist, offset) { 5507 unsigned long size = zone_managed_pages(zone); 5508 unsigned long high = high_wmark_pages(zone); 5509 if (size > high) 5510 sum += size - high; 5511 } 5512 5513 return sum; 5514 } 5515 5516 /** 5517 * nr_free_buffer_pages - count number of pages beyond high watermark 5518 * 5519 * nr_free_buffer_pages() counts the number of pages which are beyond the high 5520 * watermark within ZONE_DMA and ZONE_NORMAL. 5521 * 5522 * Return: number of pages beyond high watermark within ZONE_DMA and 5523 * ZONE_NORMAL. 5524 */ 5525 unsigned long nr_free_buffer_pages(void) 5526 { 5527 return nr_free_zone_pages(gfp_zone(GFP_USER)); 5528 } 5529 EXPORT_SYMBOL_GPL(nr_free_buffer_pages); 5530 5531 static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref) 5532 { 5533 zoneref->zone = zone; 5534 zoneref->zone_idx = zone_idx(zone); 5535 } 5536 5537 /* 5538 * Builds allocation fallback zone lists. 5539 * 5540 * Add all populated zones of a node to the zonelist. 5541 */ 5542 static int build_zonerefs_node(pg_data_t *pgdat, struct zoneref *zonerefs) 5543 { 5544 struct zone *zone; 5545 enum zone_type zone_type = MAX_NR_ZONES; 5546 int nr_zones = 0; 5547 5548 do { 5549 zone_type--; 5550 zone = pgdat->node_zones + zone_type; 5551 if (populated_zone(zone)) { 5552 zoneref_set_zone(zone, &zonerefs[nr_zones++]); 5553 check_highest_zone(zone_type); 5554 } 5555 } while (zone_type); 5556 5557 return nr_zones; 5558 } 5559 5560 #ifdef CONFIG_NUMA 5561 5562 static int __parse_numa_zonelist_order(char *s) 5563 { 5564 /* 5565 * We used to support different zonelists modes but they turned 5566 * out to be just not useful. Let's keep the warning in place 5567 * if somebody still use the cmd line parameter so that we do 5568 * not fail it silently 5569 */ 5570 if (!(*s == 'd' || *s == 'D' || *s == 'n' || *s == 'N')) { 5571 pr_warn("Ignoring unsupported numa_zonelist_order value: %s\n", s); 5572 return -EINVAL; 5573 } 5574 return 0; 5575 } 5576 5577 static char numa_zonelist_order[] = "Node"; 5578 #define NUMA_ZONELIST_ORDER_LEN 16 5579 /* 5580 * sysctl handler for numa_zonelist_order 5581 */ 5582 static int numa_zonelist_order_handler(const struct ctl_table *table, int write, 5583 void *buffer, size_t *length, loff_t *ppos) 5584 { 5585 if (write) 5586 return __parse_numa_zonelist_order(buffer); 5587 return proc_dostring(table, write, buffer, length, ppos); 5588 } 5589 5590 static int node_load[MAX_NUMNODES]; 5591 5592 /** 5593 * find_next_best_node - find the next node that should appear in a given node's fallback list 5594 * @node: node whose fallback list we're appending 5595 * @used_node_mask: nodemask_t of already used nodes 5596 * 5597 * We use a number of factors to determine which is the next node that should 5598 * appear on a given node's fallback list. The node should not have appeared 5599 * already in @node's fallback list, and it should be the next closest node 5600 * according to the distance array (which contains arbitrary distance values 5601 * from each node to each node in the system), and should also prefer nodes 5602 * with no CPUs, since presumably they'll have very little allocation pressure 5603 * on them otherwise. 5604 * 5605 * Return: node id of the found node or %NUMA_NO_NODE if no node is found. 5606 */ 5607 int find_next_best_node(int node, nodemask_t *used_node_mask) 5608 { 5609 int n, val; 5610 int min_val = INT_MAX; 5611 int best_node = NUMA_NO_NODE; 5612 5613 /* 5614 * Use the local node if we haven't already, but for memoryless local 5615 * node, we should skip it and fall back to other nodes. 5616 */ 5617 if (!node_isset(node, *used_node_mask) && node_state(node, N_MEMORY)) { 5618 node_set(node, *used_node_mask); 5619 return node; 5620 } 5621 5622 for_each_node_state(n, N_MEMORY) { 5623 5624 /* Don't want a node to appear more than once */ 5625 if (node_isset(n, *used_node_mask)) 5626 continue; 5627 5628 /* Use the distance array to find the distance */ 5629 val = node_distance(node, n); 5630 5631 /* Penalize nodes under us ("prefer the next node") */ 5632 val += (n < node); 5633 5634 /* Give preference to headless and unused nodes */ 5635 if (!cpumask_empty(cpumask_of_node(n))) 5636 val += PENALTY_FOR_NODE_WITH_CPUS; 5637 5638 /* Slight preference for less loaded node */ 5639 val *= MAX_NUMNODES; 5640 val += node_load[n]; 5641 5642 if (val < min_val) { 5643 min_val = val; 5644 best_node = n; 5645 } 5646 } 5647 5648 if (best_node >= 0) 5649 node_set(best_node, *used_node_mask); 5650 5651 return best_node; 5652 } 5653 5654 5655 /* 5656 * Build zonelists ordered by node and zones within node. 5657 * This results in maximum locality--normal zone overflows into local 5658 * DMA zone, if any--but risks exhausting DMA zone. 5659 */ 5660 static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order, 5661 unsigned nr_nodes) 5662 { 5663 struct zoneref *zonerefs; 5664 int i; 5665 5666 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; 5667 5668 for (i = 0; i < nr_nodes; i++) { 5669 int nr_zones; 5670 5671 pg_data_t *node = NODE_DATA(node_order[i]); 5672 5673 nr_zones = build_zonerefs_node(node, zonerefs); 5674 zonerefs += nr_zones; 5675 } 5676 zonerefs->zone = NULL; 5677 zonerefs->zone_idx = 0; 5678 } 5679 5680 /* 5681 * Build __GFP_THISNODE zonelists 5682 */ 5683 static void build_thisnode_zonelists(pg_data_t *pgdat) 5684 { 5685 struct zoneref *zonerefs; 5686 int nr_zones; 5687 5688 zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs; 5689 nr_zones = build_zonerefs_node(pgdat, zonerefs); 5690 zonerefs += nr_zones; 5691 zonerefs->zone = NULL; 5692 zonerefs->zone_idx = 0; 5693 } 5694 5695 static void build_zonelists(pg_data_t *pgdat) 5696 { 5697 static int node_order[MAX_NUMNODES]; 5698 int node, nr_nodes = 0; 5699 nodemask_t used_mask = NODE_MASK_NONE; 5700 int local_node, prev_node; 5701 5702 /* NUMA-aware ordering of nodes */ 5703 local_node = pgdat->node_id; 5704 prev_node = local_node; 5705 5706 memset(node_order, 0, sizeof(node_order)); 5707 while ((node = find_next_best_node(local_node, &used_mask)) >= 0) { 5708 /* 5709 * We don't want to pressure a particular node. 5710 * So adding penalty to the first node in same 5711 * distance group to make it round-robin. 5712 */ 5713 if (node_distance(local_node, node) != 5714 node_distance(local_node, prev_node)) 5715 node_load[node] += 1; 5716 5717 node_order[nr_nodes++] = node; 5718 prev_node = node; 5719 } 5720 5721 build_zonelists_in_node_order(pgdat, node_order, nr_nodes); 5722 build_thisnode_zonelists(pgdat); 5723 pr_info("Fallback order for Node %d: ", local_node); 5724 for (node = 0; node < nr_nodes; node++) 5725 pr_cont("%d ", node_order[node]); 5726 pr_cont("\n"); 5727 } 5728 5729 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 5730 /* 5731 * Return node id of node used for "local" allocations. 5732 * I.e., first node id of first zone in arg node's generic zonelist. 5733 * Used for initializing percpu 'numa_mem', which is used primarily 5734 * for kernel allocations, so use GFP_KERNEL flags to locate zonelist. 5735 */ 5736 int local_memory_node(int node) 5737 { 5738 struct zoneref *z; 5739 5740 z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL), 5741 gfp_zone(GFP_KERNEL), 5742 NULL); 5743 return zonelist_node_idx(z); 5744 } 5745 #endif 5746 5747 static void setup_min_unmapped_ratio(void); 5748 static void setup_min_slab_ratio(void); 5749 #else /* CONFIG_NUMA */ 5750 5751 static void build_zonelists(pg_data_t *pgdat) 5752 { 5753 struct zoneref *zonerefs; 5754 int nr_zones; 5755 5756 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs; 5757 nr_zones = build_zonerefs_node(pgdat, zonerefs); 5758 zonerefs += nr_zones; 5759 5760 zonerefs->zone = NULL; 5761 zonerefs->zone_idx = 0; 5762 } 5763 5764 #endif /* CONFIG_NUMA */ 5765 5766 /* 5767 * Boot pageset table. One per cpu which is going to be used for all 5768 * zones and all nodes. The parameters will be set in such a way 5769 * that an item put on a list will immediately be handed over to 5770 * the buddy list. This is safe since pageset manipulation is done 5771 * with interrupts disabled. 5772 * 5773 * The boot_pagesets must be kept even after bootup is complete for 5774 * unused processors and/or zones. They do play a role for bootstrapping 5775 * hotplugged processors. 5776 * 5777 * zoneinfo_show() and maybe other functions do 5778 * not check if the processor is online before following the pageset pointer. 5779 * Other parts of the kernel may not check if the zone is available. 5780 */ 5781 static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonestat *pzstats); 5782 /* These effectively disable the pcplists in the boot pageset completely */ 5783 #define BOOT_PAGESET_HIGH 0 5784 #define BOOT_PAGESET_BATCH 1 5785 static DEFINE_PER_CPU(struct per_cpu_pages, boot_pageset); 5786 static DEFINE_PER_CPU(struct per_cpu_zonestat, boot_zonestats); 5787 5788 static void __build_all_zonelists(void *data) 5789 { 5790 int nid; 5791 int __maybe_unused cpu; 5792 pg_data_t *self = data; 5793 unsigned long flags; 5794 5795 /* 5796 * The zonelist_update_seq must be acquired with irqsave because the 5797 * reader can be invoked from IRQ with GFP_ATOMIC. 5798 */ 5799 write_seqlock_irqsave(&zonelist_update_seq, flags); 5800 /* 5801 * Also disable synchronous printk() to prevent any printk() from 5802 * trying to hold port->lock, for 5803 * tty_insert_flip_string_and_push_buffer() on other CPU might be 5804 * calling kmalloc(GFP_ATOMIC | __GFP_NOWARN) with port->lock held. 5805 */ 5806 printk_deferred_enter(); 5807 5808 #ifdef CONFIG_NUMA 5809 memset(node_load, 0, sizeof(node_load)); 5810 #endif 5811 5812 /* 5813 * This node is hotadded and no memory is yet present. So just 5814 * building zonelists is fine - no need to touch other nodes. 5815 */ 5816 if (self && !node_online(self->node_id)) { 5817 build_zonelists(self); 5818 } else { 5819 /* 5820 * All possible nodes have pgdat preallocated 5821 * in free_area_init 5822 */ 5823 for_each_node(nid) { 5824 pg_data_t *pgdat = NODE_DATA(nid); 5825 5826 build_zonelists(pgdat); 5827 } 5828 5829 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 5830 /* 5831 * We now know the "local memory node" for each node-- 5832 * i.e., the node of the first zone in the generic zonelist. 5833 * Set up numa_mem percpu variable for on-line cpus. During 5834 * boot, only the boot cpu should be on-line; we'll init the 5835 * secondary cpus' numa_mem as they come on-line. During 5836 * node/memory hotplug, we'll fixup all on-line cpus. 5837 */ 5838 for_each_online_cpu(cpu) 5839 set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu))); 5840 #endif 5841 } 5842 5843 printk_deferred_exit(); 5844 write_sequnlock_irqrestore(&zonelist_update_seq, flags); 5845 } 5846 5847 static noinline void __init 5848 build_all_zonelists_init(void) 5849 { 5850 int cpu; 5851 5852 __build_all_zonelists(NULL); 5853 5854 /* 5855 * Initialize the boot_pagesets that are going to be used 5856 * for bootstrapping processors. The real pagesets for 5857 * each zone will be allocated later when the per cpu 5858 * allocator is available. 5859 * 5860 * boot_pagesets are used also for bootstrapping offline 5861 * cpus if the system is already booted because the pagesets 5862 * are needed to initialize allocators on a specific cpu too. 5863 * F.e. the percpu allocator needs the page allocator which 5864 * needs the percpu allocator in order to allocate its pagesets 5865 * (a chicken-egg dilemma). 5866 */ 5867 for_each_possible_cpu(cpu) 5868 per_cpu_pages_init(&per_cpu(boot_pageset, cpu), &per_cpu(boot_zonestats, cpu)); 5869 5870 mminit_verify_zonelist(); 5871 cpuset_init_current_mems_allowed(); 5872 } 5873 5874 /* 5875 * unless system_state == SYSTEM_BOOTING. 5876 * 5877 * __ref due to call of __init annotated helper build_all_zonelists_init 5878 * [protected by SYSTEM_BOOTING]. 5879 */ 5880 void __ref build_all_zonelists(pg_data_t *pgdat) 5881 { 5882 unsigned long vm_total_pages; 5883 5884 if (system_state == SYSTEM_BOOTING) { 5885 build_all_zonelists_init(); 5886 } else { 5887 __build_all_zonelists(pgdat); 5888 /* cpuset refresh routine should be here */ 5889 } 5890 /* Get the number of free pages beyond high watermark in all zones. */ 5891 vm_total_pages = nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE)); 5892 /* 5893 * Disable grouping by mobility if the number of pages in the 5894 * system is too low to allow the mechanism to work. It would be 5895 * more accurate, but expensive to check per-zone. This check is 5896 * made on memory-hotadd so a system can start with mobility 5897 * disabled and enable it later 5898 */ 5899 if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES)) 5900 page_group_by_mobility_disabled = 1; 5901 else 5902 page_group_by_mobility_disabled = 0; 5903 5904 pr_info("Built %u zonelists, mobility grouping %s. Total pages: %ld\n", 5905 nr_online_nodes, 5906 str_off_on(page_group_by_mobility_disabled), 5907 vm_total_pages); 5908 #ifdef CONFIG_NUMA 5909 pr_info("Policy zone: %s\n", zone_names[policy_zone]); 5910 #endif 5911 } 5912 5913 static int zone_batchsize(struct zone *zone) 5914 { 5915 #ifdef CONFIG_MMU 5916 int batch; 5917 5918 /* 5919 * The number of pages to batch allocate is either ~0.1% 5920 * of the zone or 1MB, whichever is smaller. The batch 5921 * size is striking a balance between allocation latency 5922 * and zone lock contention. 5923 */ 5924 batch = min(zone_managed_pages(zone) >> 10, SZ_1M / PAGE_SIZE); 5925 batch /= 4; /* We effectively *= 4 below */ 5926 if (batch < 1) 5927 batch = 1; 5928 5929 /* 5930 * Clamp the batch to a 2^n - 1 value. Having a power 5931 * of 2 value was found to be more likely to have 5932 * suboptimal cache aliasing properties in some cases. 5933 * 5934 * For example if 2 tasks are alternately allocating 5935 * batches of pages, one task can end up with a lot 5936 * of pages of one half of the possible page colors 5937 * and the other with pages of the other colors. 5938 */ 5939 batch = rounddown_pow_of_two(batch + batch/2) - 1; 5940 5941 return batch; 5942 5943 #else 5944 /* The deferral and batching of frees should be suppressed under NOMMU 5945 * conditions. 5946 * 5947 * The problem is that NOMMU needs to be able to allocate large chunks 5948 * of contiguous memory as there's no hardware page translation to 5949 * assemble apparent contiguous memory from discontiguous pages. 5950 * 5951 * Queueing large contiguous runs of pages for batching, however, 5952 * causes the pages to actually be freed in smaller chunks. As there 5953 * can be a significant delay between the individual batches being 5954 * recycled, this leads to the once large chunks of space being 5955 * fragmented and becoming unavailable for high-order allocations. 5956 */ 5957 return 0; 5958 #endif 5959 } 5960 5961 static int percpu_pagelist_high_fraction; 5962 static int zone_highsize(struct zone *zone, int batch, int cpu_online, 5963 int high_fraction) 5964 { 5965 #ifdef CONFIG_MMU 5966 int high; 5967 int nr_split_cpus; 5968 unsigned long total_pages; 5969 5970 if (!high_fraction) { 5971 /* 5972 * By default, the high value of the pcp is based on the zone 5973 * low watermark so that if they are full then background 5974 * reclaim will not be started prematurely. 5975 */ 5976 total_pages = low_wmark_pages(zone); 5977 } else { 5978 /* 5979 * If percpu_pagelist_high_fraction is configured, the high 5980 * value is based on a fraction of the managed pages in the 5981 * zone. 5982 */ 5983 total_pages = zone_managed_pages(zone) / high_fraction; 5984 } 5985 5986 /* 5987 * Split the high value across all online CPUs local to the zone. Note 5988 * that early in boot that CPUs may not be online yet and that during 5989 * CPU hotplug that the cpumask is not yet updated when a CPU is being 5990 * onlined. For memory nodes that have no CPUs, split the high value 5991 * across all online CPUs to mitigate the risk that reclaim is triggered 5992 * prematurely due to pages stored on pcp lists. 5993 */ 5994 nr_split_cpus = cpumask_weight(cpumask_of_node(zone_to_nid(zone))) + cpu_online; 5995 if (!nr_split_cpus) 5996 nr_split_cpus = num_online_cpus(); 5997 high = total_pages / nr_split_cpus; 5998 5999 /* 6000 * Ensure high is at least batch*4. The multiple is based on the 6001 * historical relationship between high and batch. 6002 */ 6003 high = max(high, batch << 2); 6004 6005 return high; 6006 #else 6007 return 0; 6008 #endif 6009 } 6010 6011 /* 6012 * pcp->high and pcp->batch values are related and generally batch is lower 6013 * than high. They are also related to pcp->count such that count is lower 6014 * than high, and as soon as it reaches high, the pcplist is flushed. 6015 * 6016 * However, guaranteeing these relations at all times would require e.g. write 6017 * barriers here but also careful usage of read barriers at the read side, and 6018 * thus be prone to error and bad for performance. Thus the update only prevents 6019 * store tearing. Any new users of pcp->batch, pcp->high_min and pcp->high_max 6020 * should ensure they can cope with those fields changing asynchronously, and 6021 * fully trust only the pcp->count field on the local CPU with interrupts 6022 * disabled. 6023 * 6024 * mutex_is_locked(&pcp_batch_high_lock) required when calling this function 6025 * outside of boot time (or some other assurance that no concurrent updaters 6026 * exist). 6027 */ 6028 static void pageset_update(struct per_cpu_pages *pcp, unsigned long high_min, 6029 unsigned long high_max, unsigned long batch) 6030 { 6031 WRITE_ONCE(pcp->batch, batch); 6032 WRITE_ONCE(pcp->high_min, high_min); 6033 WRITE_ONCE(pcp->high_max, high_max); 6034 } 6035 6036 static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonestat *pzstats) 6037 { 6038 int pindex; 6039 6040 memset(pcp, 0, sizeof(*pcp)); 6041 memset(pzstats, 0, sizeof(*pzstats)); 6042 6043 spin_lock_init(&pcp->lock); 6044 for (pindex = 0; pindex < NR_PCP_LISTS; pindex++) 6045 INIT_LIST_HEAD(&pcp->lists[pindex]); 6046 6047 /* 6048 * Set batch and high values safe for a boot pageset. A true percpu 6049 * pageset's initialization will update them subsequently. Here we don't 6050 * need to be as careful as pageset_update() as nobody can access the 6051 * pageset yet. 6052 */ 6053 pcp->high_min = BOOT_PAGESET_HIGH; 6054 pcp->high_max = BOOT_PAGESET_HIGH; 6055 pcp->batch = BOOT_PAGESET_BATCH; 6056 } 6057 6058 static void __zone_set_pageset_high_and_batch(struct zone *zone, unsigned long high_min, 6059 unsigned long high_max, unsigned long batch) 6060 { 6061 struct per_cpu_pages *pcp; 6062 int cpu; 6063 6064 for_each_possible_cpu(cpu) { 6065 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); 6066 pageset_update(pcp, high_min, high_max, batch); 6067 } 6068 } 6069 6070 /* 6071 * Calculate and set new high and batch values for all per-cpu pagesets of a 6072 * zone based on the zone's size. 6073 */ 6074 static void zone_set_pageset_high_and_batch(struct zone *zone, int cpu_online) 6075 { 6076 int new_high_min, new_high_max, new_batch; 6077 6078 new_batch = max(1, zone_batchsize(zone)); 6079 if (percpu_pagelist_high_fraction) { 6080 new_high_min = zone_highsize(zone, new_batch, cpu_online, 6081 percpu_pagelist_high_fraction); 6082 /* 6083 * PCP high is tuned manually, disable auto-tuning via 6084 * setting high_min and high_max to the manual value. 6085 */ 6086 new_high_max = new_high_min; 6087 } else { 6088 new_high_min = zone_highsize(zone, new_batch, cpu_online, 0); 6089 new_high_max = zone_highsize(zone, new_batch, cpu_online, 6090 MIN_PERCPU_PAGELIST_HIGH_FRACTION); 6091 } 6092 6093 if (zone->pageset_high_min == new_high_min && 6094 zone->pageset_high_max == new_high_max && 6095 zone->pageset_batch == new_batch) 6096 return; 6097 6098 zone->pageset_high_min = new_high_min; 6099 zone->pageset_high_max = new_high_max; 6100 zone->pageset_batch = new_batch; 6101 6102 __zone_set_pageset_high_and_batch(zone, new_high_min, new_high_max, 6103 new_batch); 6104 } 6105 6106 void __meminit setup_zone_pageset(struct zone *zone) 6107 { 6108 int cpu; 6109 6110 /* Size may be 0 on !SMP && !NUMA */ 6111 if (sizeof(struct per_cpu_zonestat) > 0) 6112 zone->per_cpu_zonestats = alloc_percpu(struct per_cpu_zonestat); 6113 6114 zone->per_cpu_pageset = alloc_percpu(struct per_cpu_pages); 6115 for_each_possible_cpu(cpu) { 6116 struct per_cpu_pages *pcp; 6117 struct per_cpu_zonestat *pzstats; 6118 6119 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); 6120 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 6121 per_cpu_pages_init(pcp, pzstats); 6122 } 6123 6124 zone_set_pageset_high_and_batch(zone, 0); 6125 } 6126 6127 /* 6128 * The zone indicated has a new number of managed_pages; batch sizes and percpu 6129 * page high values need to be recalculated. 6130 */ 6131 static void zone_pcp_update(struct zone *zone, int cpu_online) 6132 { 6133 mutex_lock(&pcp_batch_high_lock); 6134 zone_set_pageset_high_and_batch(zone, cpu_online); 6135 mutex_unlock(&pcp_batch_high_lock); 6136 } 6137 6138 static void zone_pcp_update_cacheinfo(struct zone *zone, unsigned int cpu) 6139 { 6140 struct per_cpu_pages *pcp; 6141 struct cpu_cacheinfo *cci; 6142 unsigned long UP_flags; 6143 6144 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); 6145 cci = get_cpu_cacheinfo(cpu); 6146 /* 6147 * If data cache slice of CPU is large enough, "pcp->batch" 6148 * pages can be preserved in PCP before draining PCP for 6149 * consecutive high-order pages freeing without allocation. 6150 * This can reduce zone lock contention without hurting 6151 * cache-hot pages sharing. 6152 */ 6153 pcp_spin_lock_maybe_irqsave(pcp, UP_flags); 6154 if ((cci->per_cpu_data_slice_size >> PAGE_SHIFT) > 3 * pcp->batch) 6155 pcp->flags |= PCPF_FREE_HIGH_BATCH; 6156 else 6157 pcp->flags &= ~PCPF_FREE_HIGH_BATCH; 6158 pcp_spin_unlock_maybe_irqrestore(pcp, UP_flags); 6159 } 6160 6161 void setup_pcp_cacheinfo(unsigned int cpu) 6162 { 6163 struct zone *zone; 6164 6165 for_each_populated_zone(zone) 6166 zone_pcp_update_cacheinfo(zone, cpu); 6167 } 6168 6169 /* 6170 * Allocate per cpu pagesets and initialize them. 6171 * Before this call only boot pagesets were available. 6172 */ 6173 void __init setup_per_cpu_pageset(void) 6174 { 6175 struct pglist_data *pgdat; 6176 struct zone *zone; 6177 int __maybe_unused cpu; 6178 6179 for_each_populated_zone(zone) 6180 setup_zone_pageset(zone); 6181 6182 #ifdef CONFIG_NUMA 6183 /* 6184 * Unpopulated zones continue using the boot pagesets. 6185 * The numa stats for these pagesets need to be reset. 6186 * Otherwise, they will end up skewing the stats of 6187 * the nodes these zones are associated with. 6188 */ 6189 for_each_possible_cpu(cpu) { 6190 struct per_cpu_zonestat *pzstats = &per_cpu(boot_zonestats, cpu); 6191 memset(pzstats->vm_numa_event, 0, 6192 sizeof(pzstats->vm_numa_event)); 6193 } 6194 #endif 6195 6196 for_each_online_pgdat(pgdat) 6197 pgdat->per_cpu_nodestats = 6198 alloc_percpu(struct per_cpu_nodestat); 6199 } 6200 6201 __meminit void zone_pcp_init(struct zone *zone) 6202 { 6203 /* 6204 * per cpu subsystem is not up at this point. The following code 6205 * relies on the ability of the linker to provide the 6206 * offset of a (static) per cpu variable into the per cpu area. 6207 */ 6208 zone->per_cpu_pageset = &boot_pageset; 6209 zone->per_cpu_zonestats = &boot_zonestats; 6210 zone->pageset_high_min = BOOT_PAGESET_HIGH; 6211 zone->pageset_high_max = BOOT_PAGESET_HIGH; 6212 zone->pageset_batch = BOOT_PAGESET_BATCH; 6213 6214 if (populated_zone(zone)) 6215 pr_debug(" %s zone: %lu pages, LIFO batch:%u\n", zone->name, 6216 zone->present_pages, zone_batchsize(zone)); 6217 } 6218 6219 static void setup_per_zone_lowmem_reserve(void); 6220 6221 void adjust_managed_page_count(struct page *page, long count) 6222 { 6223 atomic_long_add(count, &page_zone(page)->managed_pages); 6224 totalram_pages_add(count); 6225 setup_per_zone_lowmem_reserve(); 6226 } 6227 EXPORT_SYMBOL(adjust_managed_page_count); 6228 6229 unsigned long free_reserved_area(void *start, void *end, int poison, const char *s) 6230 { 6231 void *pos; 6232 unsigned long pages = 0; 6233 6234 start = (void *)PAGE_ALIGN((unsigned long)start); 6235 end = (void *)((unsigned long)end & PAGE_MASK); 6236 for (pos = start; pos < end; pos += PAGE_SIZE, pages++) { 6237 struct page *page = virt_to_page(pos); 6238 void *direct_map_addr; 6239 6240 /* 6241 * 'direct_map_addr' might be different from 'pos' 6242 * because some architectures' virt_to_page() 6243 * work with aliases. Getting the direct map 6244 * address ensures that we get a _writeable_ 6245 * alias for the memset(). 6246 */ 6247 direct_map_addr = page_address(page); 6248 /* 6249 * Perform a kasan-unchecked memset() since this memory 6250 * has not been initialized. 6251 */ 6252 direct_map_addr = kasan_reset_tag(direct_map_addr); 6253 if ((unsigned int)poison <= 0xFF) 6254 memset(direct_map_addr, poison, PAGE_SIZE); 6255 6256 free_reserved_page(page); 6257 } 6258 6259 if (pages && s) 6260 pr_info("Freeing %s memory: %ldK\n", s, K(pages)); 6261 6262 return pages; 6263 } 6264 6265 void free_reserved_page(struct page *page) 6266 { 6267 clear_page_tag_ref(page); 6268 ClearPageReserved(page); 6269 init_page_count(page); 6270 __free_page(page); 6271 adjust_managed_page_count(page, 1); 6272 } 6273 EXPORT_SYMBOL(free_reserved_page); 6274 6275 static int page_alloc_cpu_dead(unsigned int cpu) 6276 { 6277 struct zone *zone; 6278 6279 lru_add_drain_cpu(cpu); 6280 mlock_drain_remote(cpu); 6281 drain_pages(cpu); 6282 6283 /* 6284 * Spill the event counters of the dead processor 6285 * into the current processors event counters. 6286 * This artificially elevates the count of the current 6287 * processor. 6288 */ 6289 vm_events_fold_cpu(cpu); 6290 6291 /* 6292 * Zero the differential counters of the dead processor 6293 * so that the vm statistics are consistent. 6294 * 6295 * This is only okay since the processor is dead and cannot 6296 * race with what we are doing. 6297 */ 6298 cpu_vm_stats_fold(cpu); 6299 6300 for_each_populated_zone(zone) 6301 zone_pcp_update(zone, 0); 6302 6303 return 0; 6304 } 6305 6306 static int page_alloc_cpu_online(unsigned int cpu) 6307 { 6308 struct zone *zone; 6309 6310 for_each_populated_zone(zone) 6311 zone_pcp_update(zone, 1); 6312 return 0; 6313 } 6314 6315 void __init page_alloc_init_cpuhp(void) 6316 { 6317 int ret; 6318 6319 ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC, 6320 "mm/page_alloc:pcp", 6321 page_alloc_cpu_online, 6322 page_alloc_cpu_dead); 6323 WARN_ON(ret < 0); 6324 } 6325 6326 /* 6327 * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio 6328 * or min_free_kbytes changes. 6329 */ 6330 static void calculate_totalreserve_pages(void) 6331 { 6332 struct pglist_data *pgdat; 6333 unsigned long reserve_pages = 0; 6334 enum zone_type i, j; 6335 6336 for_each_online_pgdat(pgdat) { 6337 6338 pgdat->totalreserve_pages = 0; 6339 6340 for (i = 0; i < MAX_NR_ZONES; i++) { 6341 struct zone *zone = pgdat->node_zones + i; 6342 long max = 0; 6343 unsigned long managed_pages = zone_managed_pages(zone); 6344 6345 /* Find valid and maximum lowmem_reserve in the zone */ 6346 for (j = i; j < MAX_NR_ZONES; j++) 6347 max = max(max, zone->lowmem_reserve[j]); 6348 6349 /* we treat the high watermark as reserved pages. */ 6350 max += high_wmark_pages(zone); 6351 6352 max = min_t(unsigned long, max, managed_pages); 6353 6354 pgdat->totalreserve_pages += max; 6355 6356 reserve_pages += max; 6357 } 6358 } 6359 totalreserve_pages = reserve_pages; 6360 trace_mm_calculate_totalreserve_pages(totalreserve_pages); 6361 } 6362 6363 /* 6364 * setup_per_zone_lowmem_reserve - called whenever 6365 * sysctl_lowmem_reserve_ratio changes. Ensures that each zone 6366 * has a correct pages reserved value, so an adequate number of 6367 * pages are left in the zone after a successful __alloc_pages(). 6368 */ 6369 static void setup_per_zone_lowmem_reserve(void) 6370 { 6371 struct pglist_data *pgdat; 6372 enum zone_type i, j; 6373 6374 for_each_online_pgdat(pgdat) { 6375 for (i = 0; i < MAX_NR_ZONES - 1; i++) { 6376 struct zone *zone = &pgdat->node_zones[i]; 6377 int ratio = sysctl_lowmem_reserve_ratio[i]; 6378 bool clear = !ratio || !zone_managed_pages(zone); 6379 unsigned long managed_pages = 0; 6380 6381 for (j = i + 1; j < MAX_NR_ZONES; j++) { 6382 struct zone *upper_zone = &pgdat->node_zones[j]; 6383 6384 managed_pages += zone_managed_pages(upper_zone); 6385 6386 if (clear) 6387 zone->lowmem_reserve[j] = 0; 6388 else 6389 zone->lowmem_reserve[j] = managed_pages / ratio; 6390 trace_mm_setup_per_zone_lowmem_reserve(zone, upper_zone, 6391 zone->lowmem_reserve[j]); 6392 } 6393 } 6394 } 6395 6396 /* update totalreserve_pages */ 6397 calculate_totalreserve_pages(); 6398 } 6399 6400 static void __setup_per_zone_wmarks(void) 6401 { 6402 unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10); 6403 unsigned long lowmem_pages = 0; 6404 struct zone *zone; 6405 unsigned long flags; 6406 6407 /* Calculate total number of !ZONE_HIGHMEM and !ZONE_MOVABLE pages */ 6408 for_each_zone(zone) { 6409 if (!is_highmem(zone) && zone_idx(zone) != ZONE_MOVABLE) 6410 lowmem_pages += zone_managed_pages(zone); 6411 } 6412 6413 for_each_zone(zone) { 6414 u64 tmp; 6415 6416 spin_lock_irqsave(&zone->lock, flags); 6417 tmp = (u64)pages_min * zone_managed_pages(zone); 6418 tmp = div64_ul(tmp, lowmem_pages); 6419 if (is_highmem(zone) || zone_idx(zone) == ZONE_MOVABLE) { 6420 /* 6421 * __GFP_HIGH and PF_MEMALLOC allocations usually don't 6422 * need highmem and movable zones pages, so cap pages_min 6423 * to a small value here. 6424 * 6425 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN) 6426 * deltas control async page reclaim, and so should 6427 * not be capped for highmem and movable zones. 6428 */ 6429 unsigned long min_pages; 6430 6431 min_pages = zone_managed_pages(zone) / 1024; 6432 min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL); 6433 zone->_watermark[WMARK_MIN] = min_pages; 6434 } else { 6435 /* 6436 * If it's a lowmem zone, reserve a number of pages 6437 * proportionate to the zone's size. 6438 */ 6439 zone->_watermark[WMARK_MIN] = tmp; 6440 } 6441 6442 /* 6443 * Set the kswapd watermarks distance according to the 6444 * scale factor in proportion to available memory, but 6445 * ensure a minimum size on small systems. 6446 */ 6447 tmp = max_t(u64, tmp >> 2, 6448 mult_frac(zone_managed_pages(zone), 6449 watermark_scale_factor, 10000)); 6450 6451 zone->watermark_boost = 0; 6452 zone->_watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp; 6453 zone->_watermark[WMARK_HIGH] = low_wmark_pages(zone) + tmp; 6454 zone->_watermark[WMARK_PROMO] = high_wmark_pages(zone) + tmp; 6455 trace_mm_setup_per_zone_wmarks(zone); 6456 6457 spin_unlock_irqrestore(&zone->lock, flags); 6458 } 6459 6460 /* update totalreserve_pages */ 6461 calculate_totalreserve_pages(); 6462 } 6463 6464 /** 6465 * setup_per_zone_wmarks - called when min_free_kbytes changes 6466 * or when memory is hot-{added|removed} 6467 * 6468 * Ensures that the watermark[min,low,high] values for each zone are set 6469 * correctly with respect to min_free_kbytes. 6470 */ 6471 void setup_per_zone_wmarks(void) 6472 { 6473 struct zone *zone; 6474 static DEFINE_SPINLOCK(lock); 6475 6476 spin_lock(&lock); 6477 __setup_per_zone_wmarks(); 6478 spin_unlock(&lock); 6479 6480 /* 6481 * The watermark size have changed so update the pcpu batch 6482 * and high limits or the limits may be inappropriate. 6483 */ 6484 for_each_zone(zone) 6485 zone_pcp_update(zone, 0); 6486 } 6487 6488 /* 6489 * Initialise min_free_kbytes. 6490 * 6491 * For small machines we want it small (128k min). For large machines 6492 * we want it large (256MB max). But it is not linear, because network 6493 * bandwidth does not increase linearly with machine size. We use 6494 * 6495 * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy: 6496 * min_free_kbytes = sqrt(lowmem_kbytes * 16) 6497 * 6498 * which yields 6499 * 6500 * 16MB: 512k 6501 * 32MB: 724k 6502 * 64MB: 1024k 6503 * 128MB: 1448k 6504 * 256MB: 2048k 6505 * 512MB: 2896k 6506 * 1024MB: 4096k 6507 * 2048MB: 5792k 6508 * 4096MB: 8192k 6509 * 8192MB: 11584k 6510 * 16384MB: 16384k 6511 */ 6512 void calculate_min_free_kbytes(void) 6513 { 6514 unsigned long lowmem_kbytes; 6515 int new_min_free_kbytes; 6516 6517 lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10); 6518 new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16); 6519 6520 if (new_min_free_kbytes > user_min_free_kbytes) 6521 min_free_kbytes = clamp(new_min_free_kbytes, 128, 262144); 6522 else 6523 pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n", 6524 new_min_free_kbytes, user_min_free_kbytes); 6525 6526 } 6527 6528 int __meminit init_per_zone_wmark_min(void) 6529 { 6530 calculate_min_free_kbytes(); 6531 setup_per_zone_wmarks(); 6532 refresh_zone_stat_thresholds(); 6533 setup_per_zone_lowmem_reserve(); 6534 6535 #ifdef CONFIG_NUMA 6536 setup_min_unmapped_ratio(); 6537 setup_min_slab_ratio(); 6538 #endif 6539 6540 khugepaged_min_free_kbytes_update(); 6541 6542 return 0; 6543 } 6544 postcore_initcall(init_per_zone_wmark_min) 6545 6546 /* 6547 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so 6548 * that we can call two helper functions whenever min_free_kbytes 6549 * changes. 6550 */ 6551 static int min_free_kbytes_sysctl_handler(const struct ctl_table *table, int write, 6552 void *buffer, size_t *length, loff_t *ppos) 6553 { 6554 int rc; 6555 6556 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 6557 if (rc) 6558 return rc; 6559 6560 if (write) { 6561 user_min_free_kbytes = min_free_kbytes; 6562 setup_per_zone_wmarks(); 6563 } 6564 return 0; 6565 } 6566 6567 static int watermark_scale_factor_sysctl_handler(const struct ctl_table *table, int write, 6568 void *buffer, size_t *length, loff_t *ppos) 6569 { 6570 int rc; 6571 6572 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 6573 if (rc) 6574 return rc; 6575 6576 if (write) 6577 setup_per_zone_wmarks(); 6578 6579 return 0; 6580 } 6581 6582 #ifdef CONFIG_NUMA 6583 static void setup_min_unmapped_ratio(void) 6584 { 6585 pg_data_t *pgdat; 6586 struct zone *zone; 6587 6588 for_each_online_pgdat(pgdat) 6589 pgdat->min_unmapped_pages = 0; 6590 6591 for_each_zone(zone) 6592 zone->zone_pgdat->min_unmapped_pages += (zone_managed_pages(zone) * 6593 sysctl_min_unmapped_ratio) / 100; 6594 } 6595 6596 6597 static int sysctl_min_unmapped_ratio_sysctl_handler(const struct ctl_table *table, int write, 6598 void *buffer, size_t *length, loff_t *ppos) 6599 { 6600 int rc; 6601 6602 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 6603 if (rc) 6604 return rc; 6605 6606 setup_min_unmapped_ratio(); 6607 6608 return 0; 6609 } 6610 6611 static void setup_min_slab_ratio(void) 6612 { 6613 pg_data_t *pgdat; 6614 struct zone *zone; 6615 6616 for_each_online_pgdat(pgdat) 6617 pgdat->min_slab_pages = 0; 6618 6619 for_each_zone(zone) 6620 zone->zone_pgdat->min_slab_pages += (zone_managed_pages(zone) * 6621 sysctl_min_slab_ratio) / 100; 6622 } 6623 6624 static int sysctl_min_slab_ratio_sysctl_handler(const struct ctl_table *table, int write, 6625 void *buffer, size_t *length, loff_t *ppos) 6626 { 6627 int rc; 6628 6629 rc = proc_dointvec_minmax(table, write, buffer, length, ppos); 6630 if (rc) 6631 return rc; 6632 6633 setup_min_slab_ratio(); 6634 6635 return 0; 6636 } 6637 #endif 6638 6639 /* 6640 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around 6641 * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve() 6642 * whenever sysctl_lowmem_reserve_ratio changes. 6643 * 6644 * The reserve ratio obviously has absolutely no relation with the 6645 * minimum watermarks. The lowmem reserve ratio can only make sense 6646 * if in function of the boot time zone sizes. 6647 */ 6648 static int lowmem_reserve_ratio_sysctl_handler(const struct ctl_table *table, 6649 int write, void *buffer, size_t *length, loff_t *ppos) 6650 { 6651 int i; 6652 6653 proc_dointvec_minmax(table, write, buffer, length, ppos); 6654 6655 for (i = 0; i < MAX_NR_ZONES; i++) { 6656 if (sysctl_lowmem_reserve_ratio[i] < 1) 6657 sysctl_lowmem_reserve_ratio[i] = 0; 6658 } 6659 6660 setup_per_zone_lowmem_reserve(); 6661 return 0; 6662 } 6663 6664 /* 6665 * percpu_pagelist_high_fraction - changes the pcp->high for each zone on each 6666 * cpu. It is the fraction of total pages in each zone that a hot per cpu 6667 * pagelist can have before it gets flushed back to buddy allocator. 6668 */ 6669 static int percpu_pagelist_high_fraction_sysctl_handler(const struct ctl_table *table, 6670 int write, void *buffer, size_t *length, loff_t *ppos) 6671 { 6672 struct zone *zone; 6673 int old_percpu_pagelist_high_fraction; 6674 int ret; 6675 6676 /* 6677 * Avoid using pcp_batch_high_lock for reads as the value is read 6678 * atomically and a race with offlining is harmless. 6679 */ 6680 6681 if (!write) 6682 return proc_dointvec_minmax(table, write, buffer, length, ppos); 6683 6684 mutex_lock(&pcp_batch_high_lock); 6685 old_percpu_pagelist_high_fraction = percpu_pagelist_high_fraction; 6686 6687 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 6688 if (ret < 0) 6689 goto out; 6690 6691 /* Sanity checking to avoid pcp imbalance */ 6692 if (percpu_pagelist_high_fraction && 6693 percpu_pagelist_high_fraction < MIN_PERCPU_PAGELIST_HIGH_FRACTION) { 6694 percpu_pagelist_high_fraction = old_percpu_pagelist_high_fraction; 6695 ret = -EINVAL; 6696 goto out; 6697 } 6698 6699 /* No change? */ 6700 if (percpu_pagelist_high_fraction == old_percpu_pagelist_high_fraction) 6701 goto out; 6702 6703 for_each_populated_zone(zone) 6704 zone_set_pageset_high_and_batch(zone, 0); 6705 out: 6706 mutex_unlock(&pcp_batch_high_lock); 6707 return ret; 6708 } 6709 6710 static const struct ctl_table page_alloc_sysctl_table[] = { 6711 { 6712 .procname = "min_free_kbytes", 6713 .data = &min_free_kbytes, 6714 .maxlen = sizeof(min_free_kbytes), 6715 .mode = 0644, 6716 .proc_handler = min_free_kbytes_sysctl_handler, 6717 .extra1 = SYSCTL_ZERO, 6718 }, 6719 { 6720 .procname = "watermark_boost_factor", 6721 .data = &watermark_boost_factor, 6722 .maxlen = sizeof(watermark_boost_factor), 6723 .mode = 0644, 6724 .proc_handler = proc_dointvec_minmax, 6725 .extra1 = SYSCTL_ZERO, 6726 }, 6727 { 6728 .procname = "watermark_scale_factor", 6729 .data = &watermark_scale_factor, 6730 .maxlen = sizeof(watermark_scale_factor), 6731 .mode = 0644, 6732 .proc_handler = watermark_scale_factor_sysctl_handler, 6733 .extra1 = SYSCTL_ONE, 6734 .extra2 = SYSCTL_THREE_THOUSAND, 6735 }, 6736 { 6737 .procname = "defrag_mode", 6738 .data = &defrag_mode, 6739 .maxlen = sizeof(defrag_mode), 6740 .mode = 0644, 6741 .proc_handler = proc_dointvec_minmax, 6742 .extra1 = SYSCTL_ZERO, 6743 .extra2 = SYSCTL_ONE, 6744 }, 6745 { 6746 .procname = "percpu_pagelist_high_fraction", 6747 .data = &percpu_pagelist_high_fraction, 6748 .maxlen = sizeof(percpu_pagelist_high_fraction), 6749 .mode = 0644, 6750 .proc_handler = percpu_pagelist_high_fraction_sysctl_handler, 6751 .extra1 = SYSCTL_ZERO, 6752 }, 6753 { 6754 .procname = "lowmem_reserve_ratio", 6755 .data = &sysctl_lowmem_reserve_ratio, 6756 .maxlen = sizeof(sysctl_lowmem_reserve_ratio), 6757 .mode = 0644, 6758 .proc_handler = lowmem_reserve_ratio_sysctl_handler, 6759 }, 6760 #ifdef CONFIG_NUMA 6761 { 6762 .procname = "numa_zonelist_order", 6763 .data = &numa_zonelist_order, 6764 .maxlen = NUMA_ZONELIST_ORDER_LEN, 6765 .mode = 0644, 6766 .proc_handler = numa_zonelist_order_handler, 6767 }, 6768 { 6769 .procname = "min_unmapped_ratio", 6770 .data = &sysctl_min_unmapped_ratio, 6771 .maxlen = sizeof(sysctl_min_unmapped_ratio), 6772 .mode = 0644, 6773 .proc_handler = sysctl_min_unmapped_ratio_sysctl_handler, 6774 .extra1 = SYSCTL_ZERO, 6775 .extra2 = SYSCTL_ONE_HUNDRED, 6776 }, 6777 { 6778 .procname = "min_slab_ratio", 6779 .data = &sysctl_min_slab_ratio, 6780 .maxlen = sizeof(sysctl_min_slab_ratio), 6781 .mode = 0644, 6782 .proc_handler = sysctl_min_slab_ratio_sysctl_handler, 6783 .extra1 = SYSCTL_ZERO, 6784 .extra2 = SYSCTL_ONE_HUNDRED, 6785 }, 6786 #endif 6787 }; 6788 6789 void __init page_alloc_sysctl_init(void) 6790 { 6791 register_sysctl_init("vm", page_alloc_sysctl_table); 6792 } 6793 6794 #ifdef CONFIG_CONTIG_ALLOC 6795 /* Usage: See admin-guide/dynamic-debug-howto.rst */ 6796 static void alloc_contig_dump_pages(struct list_head *page_list) 6797 { 6798 DEFINE_DYNAMIC_DEBUG_METADATA(descriptor, "migrate failure"); 6799 6800 if (DYNAMIC_DEBUG_BRANCH(descriptor)) { 6801 struct page *page; 6802 6803 dump_stack(); 6804 list_for_each_entry(page, page_list, lru) 6805 dump_page(page, "migration failure"); 6806 } 6807 } 6808 6809 /* [start, end) must belong to a single zone. */ 6810 static int __alloc_contig_migrate_range(struct compact_control *cc, 6811 unsigned long start, unsigned long end) 6812 { 6813 /* This function is based on compact_zone() from compaction.c. */ 6814 unsigned int nr_reclaimed; 6815 unsigned long pfn = start; 6816 unsigned int tries = 0; 6817 int ret = 0; 6818 struct migration_target_control mtc = { 6819 .nid = zone_to_nid(cc->zone), 6820 .gfp_mask = cc->gfp_mask, 6821 .reason = MR_CONTIG_RANGE, 6822 }; 6823 6824 lru_cache_disable(); 6825 6826 while (pfn < end || !list_empty(&cc->migratepages)) { 6827 if (fatal_signal_pending(current)) { 6828 ret = -EINTR; 6829 break; 6830 } 6831 6832 if (list_empty(&cc->migratepages)) { 6833 cc->nr_migratepages = 0; 6834 ret = isolate_migratepages_range(cc, pfn, end); 6835 if (ret && ret != -EAGAIN) 6836 break; 6837 pfn = cc->migrate_pfn; 6838 tries = 0; 6839 } else if (++tries == 5) { 6840 ret = -EBUSY; 6841 break; 6842 } 6843 6844 nr_reclaimed = reclaim_clean_pages_from_list(cc->zone, 6845 &cc->migratepages); 6846 cc->nr_migratepages -= nr_reclaimed; 6847 6848 ret = migrate_pages(&cc->migratepages, alloc_migration_target, 6849 NULL, (unsigned long)&mtc, cc->mode, MR_CONTIG_RANGE, NULL); 6850 6851 /* 6852 * On -ENOMEM, migrate_pages() bails out right away. It is pointless 6853 * to retry again over this error, so do the same here. 6854 */ 6855 if (ret == -ENOMEM) 6856 break; 6857 } 6858 6859 lru_cache_enable(); 6860 if (ret < 0) { 6861 if (!(cc->gfp_mask & __GFP_NOWARN) && ret == -EBUSY) 6862 alloc_contig_dump_pages(&cc->migratepages); 6863 putback_movable_pages(&cc->migratepages); 6864 } 6865 6866 return (ret < 0) ? ret : 0; 6867 } 6868 6869 static void split_free_pages(struct list_head *list, gfp_t gfp_mask) 6870 { 6871 int order; 6872 6873 for (order = 0; order < NR_PAGE_ORDERS; order++) { 6874 struct page *page, *next; 6875 int nr_pages = 1 << order; 6876 6877 list_for_each_entry_safe(page, next, &list[order], lru) { 6878 int i; 6879 6880 post_alloc_hook(page, order, gfp_mask); 6881 set_page_refcounted(page); 6882 if (!order) 6883 continue; 6884 6885 split_page(page, order); 6886 6887 /* Add all subpages to the order-0 head, in sequence. */ 6888 list_del(&page->lru); 6889 for (i = 0; i < nr_pages; i++) 6890 list_add_tail(&page[i].lru, &list[0]); 6891 } 6892 } 6893 } 6894 6895 static int __alloc_contig_verify_gfp_mask(gfp_t gfp_mask, gfp_t *gfp_cc_mask) 6896 { 6897 const gfp_t reclaim_mask = __GFP_IO | __GFP_FS | __GFP_RECLAIM; 6898 const gfp_t action_mask = __GFP_COMP | __GFP_RETRY_MAYFAIL | __GFP_NOWARN | 6899 __GFP_ZERO | __GFP_ZEROTAGS | __GFP_SKIP_ZERO | 6900 __GFP_SKIP_KASAN; 6901 const gfp_t cc_action_mask = __GFP_RETRY_MAYFAIL | __GFP_NOWARN; 6902 6903 /* 6904 * We are given the range to allocate; node, mobility and placement 6905 * hints are irrelevant at this point. We'll simply ignore them. 6906 */ 6907 gfp_mask &= ~(GFP_ZONEMASK | __GFP_RECLAIMABLE | __GFP_WRITE | 6908 __GFP_HARDWALL | __GFP_THISNODE | __GFP_MOVABLE); 6909 6910 /* 6911 * We only support most reclaim flags (but not NOFAIL/NORETRY), and 6912 * selected action flags. 6913 */ 6914 if (gfp_mask & ~(reclaim_mask | action_mask)) 6915 return -EINVAL; 6916 6917 /* 6918 * Flags to control page compaction/migration/reclaim, to free up our 6919 * page range. Migratable pages are movable, __GFP_MOVABLE is implied 6920 * for them. 6921 * 6922 * Traditionally we always had __GFP_RETRY_MAYFAIL set, keep doing that 6923 * to not degrade callers. 6924 */ 6925 *gfp_cc_mask = (gfp_mask & (reclaim_mask | cc_action_mask)) | 6926 __GFP_MOVABLE | __GFP_RETRY_MAYFAIL; 6927 return 0; 6928 } 6929 6930 /** 6931 * alloc_contig_range() -- tries to allocate given range of pages 6932 * @start: start PFN to allocate 6933 * @end: one-past-the-last PFN to allocate 6934 * @alloc_flags: allocation information 6935 * @gfp_mask: GFP mask. Node/zone/placement hints are ignored; only some 6936 * action and reclaim modifiers are supported. Reclaim modifiers 6937 * control allocation behavior during compaction/migration/reclaim. 6938 * 6939 * The PFN range does not have to be pageblock aligned. The PFN range must 6940 * belong to a single zone. 6941 * 6942 * The first thing this routine does is attempt to MIGRATE_ISOLATE all 6943 * pageblocks in the range. Once isolated, the pageblocks should not 6944 * be modified by others. 6945 * 6946 * Return: zero on success or negative error code. On success all 6947 * pages which PFN is in [start, end) are allocated for the caller and 6948 * need to be freed with free_contig_range(). 6949 */ 6950 int alloc_contig_range_noprof(unsigned long start, unsigned long end, 6951 acr_flags_t alloc_flags, gfp_t gfp_mask) 6952 { 6953 const unsigned int order = ilog2(end - start); 6954 unsigned long outer_start, outer_end; 6955 int ret = 0; 6956 6957 struct compact_control cc = { 6958 .nr_migratepages = 0, 6959 .order = -1, 6960 .zone = page_zone(pfn_to_page(start)), 6961 .mode = MIGRATE_SYNC, 6962 .ignore_skip_hint = true, 6963 .no_set_skip_hint = true, 6964 .alloc_contig = true, 6965 }; 6966 INIT_LIST_HEAD(&cc.migratepages); 6967 enum pb_isolate_mode mode = (alloc_flags & ACR_FLAGS_CMA) ? 6968 PB_ISOLATE_MODE_CMA_ALLOC : 6969 PB_ISOLATE_MODE_OTHER; 6970 6971 /* 6972 * In contrast to the buddy, we allow for orders here that exceed 6973 * MAX_PAGE_ORDER, so we must manually make sure that we are not 6974 * exceeding the maximum folio order. 6975 */ 6976 if (WARN_ON_ONCE((gfp_mask & __GFP_COMP) && order > MAX_FOLIO_ORDER)) 6977 return -EINVAL; 6978 6979 gfp_mask = current_gfp_context(gfp_mask); 6980 if (__alloc_contig_verify_gfp_mask(gfp_mask, (gfp_t *)&cc.gfp_mask)) 6981 return -EINVAL; 6982 6983 /* 6984 * What we do here is we mark all pageblocks in range as 6985 * MIGRATE_ISOLATE. Because pageblock and max order pages may 6986 * have different sizes, and due to the way page allocator 6987 * work, start_isolate_page_range() has special handlings for this. 6988 * 6989 * Once the pageblocks are marked as MIGRATE_ISOLATE, we 6990 * migrate the pages from an unaligned range (ie. pages that 6991 * we are interested in). This will put all the pages in 6992 * range back to page allocator as MIGRATE_ISOLATE. 6993 * 6994 * When this is done, we take the pages in range from page 6995 * allocator removing them from the buddy system. This way 6996 * page allocator will never consider using them. 6997 * 6998 * This lets us mark the pageblocks back as 6999 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the 7000 * aligned range but not in the unaligned, original range are 7001 * put back to page allocator so that buddy can use them. 7002 */ 7003 7004 ret = start_isolate_page_range(start, end, mode); 7005 if (ret) 7006 goto done; 7007 7008 drain_all_pages(cc.zone); 7009 7010 /* 7011 * In case of -EBUSY, we'd like to know which page causes problem. 7012 * So, just fall through. test_pages_isolated() has a tracepoint 7013 * which will report the busy page. 7014 * 7015 * It is possible that busy pages could become available before 7016 * the call to test_pages_isolated, and the range will actually be 7017 * allocated. So, if we fall through be sure to clear ret so that 7018 * -EBUSY is not accidentally used or returned to caller. 7019 */ 7020 ret = __alloc_contig_migrate_range(&cc, start, end); 7021 if (ret && ret != -EBUSY) 7022 goto done; 7023 7024 /* 7025 * When in-use hugetlb pages are migrated, they may simply be released 7026 * back into the free hugepage pool instead of being returned to the 7027 * buddy system. After the migration of in-use huge pages is completed, 7028 * we will invoke replace_free_hugepage_folios() to ensure that these 7029 * hugepages are properly released to the buddy system. 7030 */ 7031 ret = replace_free_hugepage_folios(start, end); 7032 if (ret) 7033 goto done; 7034 7035 /* 7036 * Pages from [start, end) are within a pageblock_nr_pages 7037 * aligned blocks that are marked as MIGRATE_ISOLATE. What's 7038 * more, all pages in [start, end) are free in page allocator. 7039 * What we are going to do is to allocate all pages from 7040 * [start, end) (that is remove them from page allocator). 7041 * 7042 * The only problem is that pages at the beginning and at the 7043 * end of interesting range may be not aligned with pages that 7044 * page allocator holds, ie. they can be part of higher order 7045 * pages. Because of this, we reserve the bigger range and 7046 * once this is done free the pages we are not interested in. 7047 * 7048 * We don't have to hold zone->lock here because the pages are 7049 * isolated thus they won't get removed from buddy. 7050 */ 7051 outer_start = find_large_buddy(start); 7052 7053 /* Make sure the range is really isolated. */ 7054 if (test_pages_isolated(outer_start, end, mode)) { 7055 ret = -EBUSY; 7056 goto done; 7057 } 7058 7059 /* Grab isolated pages from freelists. */ 7060 outer_end = isolate_freepages_range(&cc, outer_start, end); 7061 if (!outer_end) { 7062 ret = -EBUSY; 7063 goto done; 7064 } 7065 7066 if (!(gfp_mask & __GFP_COMP)) { 7067 split_free_pages(cc.freepages, gfp_mask); 7068 7069 /* Free head and tail (if any) */ 7070 if (start != outer_start) 7071 free_contig_range(outer_start, start - outer_start); 7072 if (end != outer_end) 7073 free_contig_range(end, outer_end - end); 7074 } else if (start == outer_start && end == outer_end && is_power_of_2(end - start)) { 7075 struct page *head = pfn_to_page(start); 7076 7077 check_new_pages(head, order); 7078 prep_new_page(head, order, gfp_mask, 0); 7079 set_page_refcounted(head); 7080 } else { 7081 ret = -EINVAL; 7082 WARN(true, "PFN range: requested [%lu, %lu), allocated [%lu, %lu)\n", 7083 start, end, outer_start, outer_end); 7084 } 7085 done: 7086 undo_isolate_page_range(start, end); 7087 return ret; 7088 } 7089 EXPORT_SYMBOL(alloc_contig_range_noprof); 7090 7091 static int __alloc_contig_pages(unsigned long start_pfn, 7092 unsigned long nr_pages, gfp_t gfp_mask) 7093 { 7094 unsigned long end_pfn = start_pfn + nr_pages; 7095 7096 return alloc_contig_range_noprof(start_pfn, end_pfn, ACR_FLAGS_NONE, 7097 gfp_mask); 7098 } 7099 7100 static bool pfn_range_valid_contig(struct zone *z, unsigned long start_pfn, 7101 unsigned long nr_pages) 7102 { 7103 unsigned long i, end_pfn = start_pfn + nr_pages; 7104 struct page *page; 7105 7106 for (i = start_pfn; i < end_pfn; i++) { 7107 page = pfn_to_online_page(i); 7108 if (!page) 7109 return false; 7110 7111 if (page_zone(page) != z) 7112 return false; 7113 7114 if (PageReserved(page)) 7115 return false; 7116 7117 if (PageHuge(page)) 7118 return false; 7119 } 7120 return true; 7121 } 7122 7123 static bool zone_spans_last_pfn(const struct zone *zone, 7124 unsigned long start_pfn, unsigned long nr_pages) 7125 { 7126 unsigned long last_pfn = start_pfn + nr_pages - 1; 7127 7128 return zone_spans_pfn(zone, last_pfn); 7129 } 7130 7131 /** 7132 * alloc_contig_pages() -- tries to find and allocate contiguous range of pages 7133 * @nr_pages: Number of contiguous pages to allocate 7134 * @gfp_mask: GFP mask. Node/zone/placement hints limit the search; only some 7135 * action and reclaim modifiers are supported. Reclaim modifiers 7136 * control allocation behavior during compaction/migration/reclaim. 7137 * @nid: Target node 7138 * @nodemask: Mask for other possible nodes 7139 * 7140 * This routine is a wrapper around alloc_contig_range(). It scans over zones 7141 * on an applicable zonelist to find a contiguous pfn range which can then be 7142 * tried for allocation with alloc_contig_range(). This routine is intended 7143 * for allocation requests which can not be fulfilled with the buddy allocator. 7144 * 7145 * The allocated memory is always aligned to a page boundary. If nr_pages is a 7146 * power of two, then allocated range is also guaranteed to be aligned to same 7147 * nr_pages (e.g. 1GB request would be aligned to 1GB). 7148 * 7149 * Allocated pages can be freed with free_contig_range() or by manually calling 7150 * __free_page() on each allocated page. 7151 * 7152 * Return: pointer to contiguous pages on success, or NULL if not successful. 7153 */ 7154 struct page *alloc_contig_pages_noprof(unsigned long nr_pages, gfp_t gfp_mask, 7155 int nid, nodemask_t *nodemask) 7156 { 7157 unsigned long ret, pfn, flags; 7158 struct zonelist *zonelist; 7159 struct zone *zone; 7160 struct zoneref *z; 7161 7162 zonelist = node_zonelist(nid, gfp_mask); 7163 for_each_zone_zonelist_nodemask(zone, z, zonelist, 7164 gfp_zone(gfp_mask), nodemask) { 7165 spin_lock_irqsave(&zone->lock, flags); 7166 7167 pfn = ALIGN(zone->zone_start_pfn, nr_pages); 7168 while (zone_spans_last_pfn(zone, pfn, nr_pages)) { 7169 if (pfn_range_valid_contig(zone, pfn, nr_pages)) { 7170 /* 7171 * We release the zone lock here because 7172 * alloc_contig_range() will also lock the zone 7173 * at some point. If there's an allocation 7174 * spinning on this lock, it may win the race 7175 * and cause alloc_contig_range() to fail... 7176 */ 7177 spin_unlock_irqrestore(&zone->lock, flags); 7178 ret = __alloc_contig_pages(pfn, nr_pages, 7179 gfp_mask); 7180 if (!ret) 7181 return pfn_to_page(pfn); 7182 spin_lock_irqsave(&zone->lock, flags); 7183 } 7184 pfn += nr_pages; 7185 } 7186 spin_unlock_irqrestore(&zone->lock, flags); 7187 } 7188 return NULL; 7189 } 7190 #endif /* CONFIG_CONTIG_ALLOC */ 7191 7192 void free_contig_range(unsigned long pfn, unsigned long nr_pages) 7193 { 7194 unsigned long count = 0; 7195 struct folio *folio = pfn_folio(pfn); 7196 7197 if (folio_test_large(folio)) { 7198 int expected = folio_nr_pages(folio); 7199 7200 if (nr_pages == expected) 7201 folio_put(folio); 7202 else 7203 WARN(true, "PFN %lu: nr_pages %lu != expected %d\n", 7204 pfn, nr_pages, expected); 7205 return; 7206 } 7207 7208 for (; nr_pages--; pfn++) { 7209 struct page *page = pfn_to_page(pfn); 7210 7211 count += page_count(page) != 1; 7212 __free_page(page); 7213 } 7214 WARN(count != 0, "%lu pages are still in use!\n", count); 7215 } 7216 EXPORT_SYMBOL(free_contig_range); 7217 7218 /* 7219 * Effectively disable pcplists for the zone by setting the high limit to 0 7220 * and draining all cpus. A concurrent page freeing on another CPU that's about 7221 * to put the page on pcplist will either finish before the drain and the page 7222 * will be drained, or observe the new high limit and skip the pcplist. 7223 * 7224 * Must be paired with a call to zone_pcp_enable(). 7225 */ 7226 void zone_pcp_disable(struct zone *zone) 7227 { 7228 mutex_lock(&pcp_batch_high_lock); 7229 __zone_set_pageset_high_and_batch(zone, 0, 0, 1); 7230 __drain_all_pages(zone, true); 7231 } 7232 7233 void zone_pcp_enable(struct zone *zone) 7234 { 7235 __zone_set_pageset_high_and_batch(zone, zone->pageset_high_min, 7236 zone->pageset_high_max, zone->pageset_batch); 7237 mutex_unlock(&pcp_batch_high_lock); 7238 } 7239 7240 void zone_pcp_reset(struct zone *zone) 7241 { 7242 int cpu; 7243 struct per_cpu_zonestat *pzstats; 7244 7245 if (zone->per_cpu_pageset != &boot_pageset) { 7246 for_each_online_cpu(cpu) { 7247 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 7248 drain_zonestat(zone, pzstats); 7249 } 7250 free_percpu(zone->per_cpu_pageset); 7251 zone->per_cpu_pageset = &boot_pageset; 7252 if (zone->per_cpu_zonestats != &boot_zonestats) { 7253 free_percpu(zone->per_cpu_zonestats); 7254 zone->per_cpu_zonestats = &boot_zonestats; 7255 } 7256 } 7257 } 7258 7259 #ifdef CONFIG_MEMORY_HOTREMOVE 7260 /* 7261 * All pages in the range must be in a single zone, must not contain holes, 7262 * must span full sections, and must be isolated before calling this function. 7263 * 7264 * Returns the number of managed (non-PageOffline()) pages in the range: the 7265 * number of pages for which memory offlining code must adjust managed page 7266 * counters using adjust_managed_page_count(). 7267 */ 7268 unsigned long __offline_isolated_pages(unsigned long start_pfn, 7269 unsigned long end_pfn) 7270 { 7271 unsigned long already_offline = 0, flags; 7272 unsigned long pfn = start_pfn; 7273 struct page *page; 7274 struct zone *zone; 7275 unsigned int order; 7276 7277 offline_mem_sections(pfn, end_pfn); 7278 zone = page_zone(pfn_to_page(pfn)); 7279 spin_lock_irqsave(&zone->lock, flags); 7280 while (pfn < end_pfn) { 7281 page = pfn_to_page(pfn); 7282 /* 7283 * The HWPoisoned page may be not in buddy system, and 7284 * page_count() is not 0. 7285 */ 7286 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) { 7287 pfn++; 7288 continue; 7289 } 7290 /* 7291 * At this point all remaining PageOffline() pages have a 7292 * reference count of 0 and can simply be skipped. 7293 */ 7294 if (PageOffline(page)) { 7295 BUG_ON(page_count(page)); 7296 BUG_ON(PageBuddy(page)); 7297 already_offline++; 7298 pfn++; 7299 continue; 7300 } 7301 7302 BUG_ON(page_count(page)); 7303 BUG_ON(!PageBuddy(page)); 7304 VM_WARN_ON(get_pageblock_migratetype(page) != MIGRATE_ISOLATE); 7305 order = buddy_order(page); 7306 del_page_from_free_list(page, zone, order, MIGRATE_ISOLATE); 7307 pfn += (1 << order); 7308 } 7309 spin_unlock_irqrestore(&zone->lock, flags); 7310 7311 return end_pfn - start_pfn - already_offline; 7312 } 7313 #endif 7314 7315 /* 7316 * This function returns a stable result only if called under zone lock. 7317 */ 7318 bool is_free_buddy_page(const struct page *page) 7319 { 7320 unsigned long pfn = page_to_pfn(page); 7321 unsigned int order; 7322 7323 for (order = 0; order < NR_PAGE_ORDERS; order++) { 7324 const struct page *head = page - (pfn & ((1 << order) - 1)); 7325 7326 if (PageBuddy(head) && 7327 buddy_order_unsafe(head) >= order) 7328 break; 7329 } 7330 7331 return order <= MAX_PAGE_ORDER; 7332 } 7333 EXPORT_SYMBOL(is_free_buddy_page); 7334 7335 #ifdef CONFIG_MEMORY_FAILURE 7336 static inline void add_to_free_list(struct page *page, struct zone *zone, 7337 unsigned int order, int migratetype, 7338 bool tail) 7339 { 7340 __add_to_free_list(page, zone, order, migratetype, tail); 7341 account_freepages(zone, 1 << order, migratetype); 7342 } 7343 7344 /* 7345 * Break down a higher-order page in sub-pages, and keep our target out of 7346 * buddy allocator. 7347 */ 7348 static void break_down_buddy_pages(struct zone *zone, struct page *page, 7349 struct page *target, int low, int high, 7350 int migratetype) 7351 { 7352 unsigned long size = 1 << high; 7353 struct page *current_buddy; 7354 7355 while (high > low) { 7356 high--; 7357 size >>= 1; 7358 7359 if (target >= &page[size]) { 7360 current_buddy = page; 7361 page = page + size; 7362 } else { 7363 current_buddy = page + size; 7364 } 7365 7366 if (set_page_guard(zone, current_buddy, high)) 7367 continue; 7368 7369 add_to_free_list(current_buddy, zone, high, migratetype, false); 7370 set_buddy_order(current_buddy, high); 7371 } 7372 } 7373 7374 /* 7375 * Take a page that will be marked as poisoned off the buddy allocator. 7376 */ 7377 bool take_page_off_buddy(struct page *page) 7378 { 7379 struct zone *zone = page_zone(page); 7380 unsigned long pfn = page_to_pfn(page); 7381 unsigned long flags; 7382 unsigned int order; 7383 bool ret = false; 7384 7385 spin_lock_irqsave(&zone->lock, flags); 7386 for (order = 0; order < NR_PAGE_ORDERS; order++) { 7387 struct page *page_head = page - (pfn & ((1 << order) - 1)); 7388 int page_order = buddy_order(page_head); 7389 7390 if (PageBuddy(page_head) && page_order >= order) { 7391 unsigned long pfn_head = page_to_pfn(page_head); 7392 int migratetype = get_pfnblock_migratetype(page_head, 7393 pfn_head); 7394 7395 del_page_from_free_list(page_head, zone, page_order, 7396 migratetype); 7397 break_down_buddy_pages(zone, page_head, page, 0, 7398 page_order, migratetype); 7399 SetPageHWPoisonTakenOff(page); 7400 ret = true; 7401 break; 7402 } 7403 if (page_count(page_head) > 0) 7404 break; 7405 } 7406 spin_unlock_irqrestore(&zone->lock, flags); 7407 return ret; 7408 } 7409 7410 /* 7411 * Cancel takeoff done by take_page_off_buddy(). 7412 */ 7413 bool put_page_back_buddy(struct page *page) 7414 { 7415 struct zone *zone = page_zone(page); 7416 unsigned long flags; 7417 bool ret = false; 7418 7419 spin_lock_irqsave(&zone->lock, flags); 7420 if (put_page_testzero(page)) { 7421 unsigned long pfn = page_to_pfn(page); 7422 int migratetype = get_pfnblock_migratetype(page, pfn); 7423 7424 ClearPageHWPoisonTakenOff(page); 7425 __free_one_page(page, pfn, zone, 0, migratetype, FPI_NONE); 7426 if (TestClearPageHWPoison(page)) { 7427 ret = true; 7428 } 7429 } 7430 spin_unlock_irqrestore(&zone->lock, flags); 7431 7432 return ret; 7433 } 7434 #endif 7435 7436 #ifdef CONFIG_ZONE_DMA 7437 bool has_managed_dma(void) 7438 { 7439 struct pglist_data *pgdat; 7440 7441 for_each_online_pgdat(pgdat) { 7442 struct zone *zone = &pgdat->node_zones[ZONE_DMA]; 7443 7444 if (managed_zone(zone)) 7445 return true; 7446 } 7447 return false; 7448 } 7449 #endif /* CONFIG_ZONE_DMA */ 7450 7451 #ifdef CONFIG_UNACCEPTED_MEMORY 7452 7453 static bool lazy_accept = true; 7454 7455 static int __init accept_memory_parse(char *p) 7456 { 7457 if (!strcmp(p, "lazy")) { 7458 lazy_accept = true; 7459 return 0; 7460 } else if (!strcmp(p, "eager")) { 7461 lazy_accept = false; 7462 return 0; 7463 } else { 7464 return -EINVAL; 7465 } 7466 } 7467 early_param("accept_memory", accept_memory_parse); 7468 7469 static bool page_contains_unaccepted(struct page *page, unsigned int order) 7470 { 7471 phys_addr_t start = page_to_phys(page); 7472 7473 return range_contains_unaccepted_memory(start, PAGE_SIZE << order); 7474 } 7475 7476 static void __accept_page(struct zone *zone, unsigned long *flags, 7477 struct page *page) 7478 { 7479 list_del(&page->lru); 7480 account_freepages(zone, -MAX_ORDER_NR_PAGES, MIGRATE_MOVABLE); 7481 __mod_zone_page_state(zone, NR_UNACCEPTED, -MAX_ORDER_NR_PAGES); 7482 __ClearPageUnaccepted(page); 7483 spin_unlock_irqrestore(&zone->lock, *flags); 7484 7485 accept_memory(page_to_phys(page), PAGE_SIZE << MAX_PAGE_ORDER); 7486 7487 __free_pages_ok(page, MAX_PAGE_ORDER, FPI_TO_TAIL); 7488 } 7489 7490 void accept_page(struct page *page) 7491 { 7492 struct zone *zone = page_zone(page); 7493 unsigned long flags; 7494 7495 spin_lock_irqsave(&zone->lock, flags); 7496 if (!PageUnaccepted(page)) { 7497 spin_unlock_irqrestore(&zone->lock, flags); 7498 return; 7499 } 7500 7501 /* Unlocks zone->lock */ 7502 __accept_page(zone, &flags, page); 7503 } 7504 7505 static bool try_to_accept_memory_one(struct zone *zone) 7506 { 7507 unsigned long flags; 7508 struct page *page; 7509 7510 spin_lock_irqsave(&zone->lock, flags); 7511 page = list_first_entry_or_null(&zone->unaccepted_pages, 7512 struct page, lru); 7513 if (!page) { 7514 spin_unlock_irqrestore(&zone->lock, flags); 7515 return false; 7516 } 7517 7518 /* Unlocks zone->lock */ 7519 __accept_page(zone, &flags, page); 7520 7521 return true; 7522 } 7523 7524 static bool cond_accept_memory(struct zone *zone, unsigned int order, 7525 int alloc_flags) 7526 { 7527 long to_accept, wmark; 7528 bool ret = false; 7529 7530 if (list_empty(&zone->unaccepted_pages)) 7531 return false; 7532 7533 /* Bailout, since try_to_accept_memory_one() needs to take a lock */ 7534 if (alloc_flags & ALLOC_TRYLOCK) 7535 return false; 7536 7537 wmark = promo_wmark_pages(zone); 7538 7539 /* 7540 * Watermarks have not been initialized yet. 7541 * 7542 * Accepting one MAX_ORDER page to ensure progress. 7543 */ 7544 if (!wmark) 7545 return try_to_accept_memory_one(zone); 7546 7547 /* How much to accept to get to promo watermark? */ 7548 to_accept = wmark - 7549 (zone_page_state(zone, NR_FREE_PAGES) - 7550 __zone_watermark_unusable_free(zone, order, 0) - 7551 zone_page_state(zone, NR_UNACCEPTED)); 7552 7553 while (to_accept > 0) { 7554 if (!try_to_accept_memory_one(zone)) 7555 break; 7556 ret = true; 7557 to_accept -= MAX_ORDER_NR_PAGES; 7558 } 7559 7560 return ret; 7561 } 7562 7563 static bool __free_unaccepted(struct page *page) 7564 { 7565 struct zone *zone = page_zone(page); 7566 unsigned long flags; 7567 7568 if (!lazy_accept) 7569 return false; 7570 7571 spin_lock_irqsave(&zone->lock, flags); 7572 list_add_tail(&page->lru, &zone->unaccepted_pages); 7573 account_freepages(zone, MAX_ORDER_NR_PAGES, MIGRATE_MOVABLE); 7574 __mod_zone_page_state(zone, NR_UNACCEPTED, MAX_ORDER_NR_PAGES); 7575 __SetPageUnaccepted(page); 7576 spin_unlock_irqrestore(&zone->lock, flags); 7577 7578 return true; 7579 } 7580 7581 #else 7582 7583 static bool page_contains_unaccepted(struct page *page, unsigned int order) 7584 { 7585 return false; 7586 } 7587 7588 static bool cond_accept_memory(struct zone *zone, unsigned int order, 7589 int alloc_flags) 7590 { 7591 return false; 7592 } 7593 7594 static bool __free_unaccepted(struct page *page) 7595 { 7596 BUILD_BUG(); 7597 return false; 7598 } 7599 7600 #endif /* CONFIG_UNACCEPTED_MEMORY */ 7601 7602 struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order) 7603 { 7604 /* 7605 * Do not specify __GFP_DIRECT_RECLAIM, since direct claim is not allowed. 7606 * Do not specify __GFP_KSWAPD_RECLAIM either, since wake up of kswapd 7607 * is not safe in arbitrary context. 7608 * 7609 * These two are the conditions for gfpflags_allow_spinning() being true. 7610 * 7611 * Specify __GFP_NOWARN since failing alloc_pages_nolock() is not a reason 7612 * to warn. Also warn would trigger printk() which is unsafe from 7613 * various contexts. We cannot use printk_deferred_enter() to mitigate, 7614 * since the running context is unknown. 7615 * 7616 * Specify __GFP_ZERO to make sure that call to kmsan_alloc_page() below 7617 * is safe in any context. Also zeroing the page is mandatory for 7618 * BPF use cases. 7619 * 7620 * Though __GFP_NOMEMALLOC is not checked in the code path below, 7621 * specify it here to highlight that alloc_pages_nolock() 7622 * doesn't want to deplete reserves. 7623 */ 7624 gfp_t alloc_gfp = __GFP_NOWARN | __GFP_ZERO | __GFP_NOMEMALLOC | __GFP_COMP 7625 | gfp_flags; 7626 unsigned int alloc_flags = ALLOC_TRYLOCK; 7627 struct alloc_context ac = { }; 7628 struct page *page; 7629 7630 VM_WARN_ON_ONCE(gfp_flags & ~__GFP_ACCOUNT); 7631 /* 7632 * In PREEMPT_RT spin_trylock() will call raw_spin_lock() which is 7633 * unsafe in NMI. If spin_trylock() is called from hard IRQ the current 7634 * task may be waiting for one rt_spin_lock, but rt_spin_trylock() will 7635 * mark the task as the owner of another rt_spin_lock which will 7636 * confuse PI logic, so return immediately if called form hard IRQ or 7637 * NMI. 7638 * 7639 * Note, irqs_disabled() case is ok. This function can be called 7640 * from raw_spin_lock_irqsave region. 7641 */ 7642 if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq())) 7643 return NULL; 7644 7645 /* On UP, spin_trylock() always succeeds even when it is locked */ 7646 if (!IS_ENABLED(CONFIG_SMP) && in_nmi()) 7647 return NULL; 7648 7649 if (!pcp_allowed_order(order)) 7650 return NULL; 7651 7652 /* Bailout, since _deferred_grow_zone() needs to take a lock */ 7653 if (deferred_pages_enabled()) 7654 return NULL; 7655 7656 if (nid == NUMA_NO_NODE) 7657 nid = numa_node_id(); 7658 7659 prepare_alloc_pages(alloc_gfp, order, nid, NULL, &ac, 7660 &alloc_gfp, &alloc_flags); 7661 7662 /* 7663 * Best effort allocation from percpu free list. 7664 * If it's empty attempt to spin_trylock zone->lock. 7665 */ 7666 page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac); 7667 7668 /* Unlike regular alloc_pages() there is no __alloc_pages_slowpath(). */ 7669 7670 if (memcg_kmem_online() && page && (gfp_flags & __GFP_ACCOUNT) && 7671 unlikely(__memcg_kmem_charge_page(page, alloc_gfp, order) != 0)) { 7672 __free_frozen_pages(page, order, FPI_TRYLOCK); 7673 page = NULL; 7674 } 7675 trace_mm_page_alloc(page, order, alloc_gfp, ac.migratetype); 7676 kmsan_alloc_page(page, order, alloc_gfp); 7677 return page; 7678 } 7679 /** 7680 * alloc_pages_nolock - opportunistic reentrant allocation from any context 7681 * @gfp_flags: GFP flags. Only __GFP_ACCOUNT allowed. 7682 * @nid: node to allocate from 7683 * @order: allocation order size 7684 * 7685 * Allocates pages of a given order from the given node. This is safe to 7686 * call from any context (from atomic, NMI, and also reentrant 7687 * allocator -> tracepoint -> alloc_pages_nolock_noprof). 7688 * Allocation is best effort and to be expected to fail easily so nobody should 7689 * rely on the success. Failures are not reported via warn_alloc(). 7690 * See always fail conditions below. 7691 * 7692 * Return: allocated page or NULL on failure. NULL does not mean EBUSY or EAGAIN. 7693 * It means ENOMEM. There is no reason to call it again and expect !NULL. 7694 */ 7695 struct page *alloc_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order) 7696 { 7697 struct page *page; 7698 7699 page = alloc_frozen_pages_nolock_noprof(gfp_flags, nid, order); 7700 if (page) 7701 set_page_refcounted(page); 7702 return page; 7703 } 7704 EXPORT_SYMBOL_GPL(alloc_pages_nolock_noprof);