개념 설명 전체 · v6.18.37 / mm/slub.c
1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * SLUB: A slab allocator that limits cache line use instead of queuing 4 * objects in per cpu and per node lists. 5 * 6 * The allocator synchronizes using per slab locks or atomic operations 7 * and only uses a centralized lock to manage a pool of partial slabs. 8 * 9 * (C) 2007 SGI, Christoph Lameter 10 * (C) 2011 Linux Foundation, Christoph Lameter 11 */ 12 13 #include <linux/mm.h> 14 #include <linux/swap.h> /* mm_account_reclaimed_pages() */ 15 #include <linux/module.h> 16 #include <linux/bit_spinlock.h> 17 #include <linux/interrupt.h> 18 #include <linux/swab.h> 19 #include <linux/bitops.h> 20 #include <linux/slab.h> 21 #include "slab.h" 22 #include <linux/vmalloc.h> 23 #include <linux/proc_fs.h> 24 #include <linux/seq_file.h> 25 #include <linux/kasan.h> 26 #include <linux/node.h> 27 #include <linux/kmsan.h> 28 #include <linux/cpu.h> 29 #include <linux/cpuset.h> 30 #include <linux/mempolicy.h> 31 #include <linux/ctype.h> 32 #include <linux/stackdepot.h> 33 #include <linux/debugobjects.h> 34 #include <linux/kallsyms.h> 35 #include <linux/kfence.h> 36 #include <linux/memory.h> 37 #include <linux/math64.h> 38 #include <linux/fault-inject.h> 39 #include <linux/kmemleak.h> 40 #include <linux/stacktrace.h> 41 #include <linux/prefetch.h> 42 #include <linux/memcontrol.h> 43 #include <linux/random.h> 44 #include <linux/prandom.h> 45 #include <kunit/test.h> 46 #include <kunit/test-bug.h> 47 #include <linux/sort.h> 48 #include <linux/irq_work.h> 49 #include <linux/kprobes.h> 50 #include <linux/debugfs.h> 51 #include <trace/events/kmem.h> 52 53 #include "internal.h" 54 55 /* 56 * Lock order: 57 * 1. slab_mutex (Global Mutex) 58 * 2. node->list_lock (Spinlock) 59 * 3. kmem_cache->cpu_slab->lock (Local lock) 60 * 4. slab_lock(slab) (Only on some arches) 61 * 5. object_map_lock (Only for debugging) 62 * 63 * slab_mutex 64 * 65 * The role of the slab_mutex is to protect the list of all the slabs 66 * and to synchronize major metadata changes to slab cache structures. 67 * Also synchronizes memory hotplug callbacks. 68 * 69 * slab_lock 70 * 71 * The slab_lock is a wrapper around the page lock, thus it is a bit 72 * spinlock. 73 * 74 * The slab_lock is only used on arches that do not have the ability 75 * to do a cmpxchg_double. It only protects: 76 * 77 * A. slab->freelist -> List of free objects in a slab 78 * B. slab->inuse -> Number of objects in use 79 * C. slab->objects -> Number of objects in slab 80 * D. slab->frozen -> frozen state 81 * 82 * Frozen slabs 83 * 84 * If a slab is frozen then it is exempt from list management. It is 85 * the cpu slab which is actively allocated from by the processor that 86 * froze it and it is not on any list. The processor that froze the 87 * slab is the one who can perform list operations on the slab. Other 88 * processors may put objects onto the freelist but the processor that 89 * froze the slab is the only one that can retrieve the objects from the 90 * slab's freelist. 91 * 92 * CPU partial slabs 93 * 94 * The partially empty slabs cached on the CPU partial list are used 95 * for performance reasons, which speeds up the allocation process. 96 * These slabs are not frozen, but are also exempt from list management, 97 * by clearing the SL_partial flag when moving out of the node 98 * partial list. Please see __slab_free() for more details. 99 * 100 * To sum up, the current scheme is: 101 * - node partial slab: SL_partial && !frozen 102 * - cpu partial slab: !SL_partial && !frozen 103 * - cpu slab: !SL_partial && frozen 104 * - full slab: !SL_partial && !frozen 105 * 106 * list_lock 107 * 108 * The list_lock protects the partial and full list on each node and 109 * the partial slab counter. If taken then no new slabs may be added or 110 * removed from the lists nor make the number of partial slabs be modified. 111 * (Note that the total number of slabs is an atomic value that may be 112 * modified without taking the list lock). 113 * 114 * The list_lock is a centralized lock and thus we avoid taking it as 115 * much as possible. As long as SLUB does not have to handle partial 116 * slabs, operations can continue without any centralized lock. F.e. 117 * allocating a long series of objects that fill up slabs does not require 118 * the list lock. 119 * 120 * For debug caches, all allocations are forced to go through a list_lock 121 * protected region to serialize against concurrent validation. 122 * 123 * cpu_slab->lock local lock 124 * 125 * This locks protect slowpath manipulation of all kmem_cache_cpu fields 126 * except the stat counters. This is a percpu structure manipulated only by 127 * the local cpu, so the lock protects against being preempted or interrupted 128 * by an irq. Fast path operations rely on lockless operations instead. 129 * 130 * On PREEMPT_RT, the local lock neither disables interrupts nor preemption 131 * which means the lockless fastpath cannot be used as it might interfere with 132 * an in-progress slow path operations. In this case the local lock is always 133 * taken but it still utilizes the freelist for the common operations. 134 * 135 * lockless fastpaths 136 * 137 * The fast path allocation (slab_alloc_node()) and freeing (do_slab_free()) 138 * are fully lockless when satisfied from the percpu slab (and when 139 * cmpxchg_double is possible to use, otherwise slab_lock is taken). 140 * They also don't disable preemption or migration or irqs. They rely on 141 * the transaction id (tid) field to detect being preempted or moved to 142 * another cpu. 143 * 144 * irq, preemption, migration considerations 145 * 146 * Interrupts are disabled as part of list_lock or local_lock operations, or 147 * around the slab_lock operation, in order to make the slab allocator safe 148 * to use in the context of an irq. 149 * 150 * In addition, preemption (or migration on PREEMPT_RT) is disabled in the 151 * allocation slowpath, bulk allocation, and put_cpu_partial(), so that the 152 * local cpu doesn't change in the process and e.g. the kmem_cache_cpu pointer 153 * doesn't have to be revalidated in each section protected by the local lock. 154 * 155 * SLUB assigns one slab for allocation to each processor. 156 * Allocations only occur from these slabs called cpu slabs. 157 * 158 * Slabs with free elements are kept on a partial list and during regular 159 * operations no list for full slabs is used. If an object in a full slab is 160 * freed then the slab will show up again on the partial lists. 161 * We track full slabs for debugging purposes though because otherwise we 162 * cannot scan all objects. 163 * 164 * Slabs are freed when they become empty. Teardown and setup is 165 * minimal so we rely on the page allocators per cpu caches for 166 * fast frees and allocs. 167 * 168 * slab->frozen The slab is frozen and exempt from list processing. 169 * This means that the slab is dedicated to a purpose 170 * such as satisfying allocations for a specific 171 * processor. Objects may be freed in the slab while 172 * it is frozen but slab_free will then skip the usual 173 * list operations. It is up to the processor holding 174 * the slab to integrate the slab into the slab lists 175 * when the slab is no longer needed. 176 * 177 * One use of this flag is to mark slabs that are 178 * used for allocations. Then such a slab becomes a cpu 179 * slab. The cpu slab may be equipped with an additional 180 * freelist that allows lockless access to 181 * free objects in addition to the regular freelist 182 * that requires the slab lock. 183 * 184 * SLAB_DEBUG_FLAGS Slab requires special handling due to debug 185 * options set. This moves slab handling out of 186 * the fast path and disables lockless freelists. 187 */ 188 189 /** 190 * enum slab_flags - How the slab flags bits are used. 191 * @SL_locked: Is locked with slab_lock() 192 * @SL_partial: On the per-node partial list 193 * @SL_pfmemalloc: Was allocated from PF_MEMALLOC reserves 194 * 195 * The slab flags share space with the page flags but some bits have 196 * different interpretations. The high bits are used for information 197 * like zone/node/section. 198 */ 199 enum slab_flags { 200 SL_locked = PG_locked, 201 SL_partial = PG_workingset, /* Historical reasons for this bit */ 202 SL_pfmemalloc = PG_active, /* Historical reasons for this bit */ 203 }; 204 205 /* 206 * We could simply use migrate_disable()/enable() but as long as it's a 207 * function call even on !PREEMPT_RT, use inline preempt_disable() there. 208 */ 209 #ifndef CONFIG_PREEMPT_RT 210 #define slub_get_cpu_ptr(var) get_cpu_ptr(var) 211 #define slub_put_cpu_ptr(var) put_cpu_ptr(var) 212 #define USE_LOCKLESS_FAST_PATH() (true) 213 #else 214 #define slub_get_cpu_ptr(var) \ 215 ({ \ 216 migrate_disable(); \ 217 this_cpu_ptr(var); \ 218 }) 219 #define slub_put_cpu_ptr(var) \ 220 do { \ 221 (void)(var); \ 222 migrate_enable(); \ 223 } while (0) 224 #define USE_LOCKLESS_FAST_PATH() (false) 225 #endif 226 227 #ifndef CONFIG_SLUB_TINY 228 #define __fastpath_inline __always_inline 229 #else 230 #define __fastpath_inline 231 #endif 232 233 #ifdef CONFIG_SLUB_DEBUG 234 #ifdef CONFIG_SLUB_DEBUG_ON 235 DEFINE_STATIC_KEY_TRUE(slub_debug_enabled); 236 #else 237 DEFINE_STATIC_KEY_FALSE(slub_debug_enabled); 238 #endif 239 #endif /* CONFIG_SLUB_DEBUG */ 240 241 #ifdef CONFIG_NUMA 242 static DEFINE_STATIC_KEY_FALSE(strict_numa); 243 #endif 244 245 /* Structure holding parameters for get_partial() call chain */ 246 struct partial_context { 247 gfp_t flags; 248 unsigned int orig_size; 249 void *object; 250 }; 251 252 static inline bool kmem_cache_debug(struct kmem_cache *s) 253 { 254 return kmem_cache_debug_flags(s, SLAB_DEBUG_FLAGS); 255 } 256 257 void *fixup_red_left(struct kmem_cache *s, void *p) 258 { 259 if (kmem_cache_debug_flags(s, SLAB_RED_ZONE)) 260 p += s->red_left_pad; 261 262 return p; 263 } 264 265 static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s) 266 { 267 #ifdef CONFIG_SLUB_CPU_PARTIAL 268 return !kmem_cache_debug(s); 269 #else 270 return false; 271 #endif 272 } 273 274 /* 275 * Issues still to be resolved: 276 * 277 * - Support PAGE_ALLOC_DEBUG. Should be easy to do. 278 * 279 * - Variable sizing of the per node arrays 280 */ 281 282 /* Enable to log cmpxchg failures */ 283 #undef SLUB_DEBUG_CMPXCHG 284 285 #ifndef CONFIG_SLUB_TINY 286 /* 287 * Minimum number of partial slabs. These will be left on the partial 288 * lists even if they are empty. kmem_cache_shrink may reclaim them. 289 */ 290 #define MIN_PARTIAL 5 291 292 /* 293 * Maximum number of desirable partial slabs. 294 * The existence of more partial slabs makes kmem_cache_shrink 295 * sort the partial list by the number of objects in use. 296 */ 297 #define MAX_PARTIAL 10 298 #else 299 #define MIN_PARTIAL 0 300 #define MAX_PARTIAL 0 301 #endif 302 303 #define DEBUG_DEFAULT_FLAGS (SLAB_CONSISTENCY_CHECKS | SLAB_RED_ZONE | \ 304 SLAB_POISON | SLAB_STORE_USER) 305 306 /* 307 * These debug flags cannot use CMPXCHG because there might be consistency 308 * issues when checking or reading debug information 309 */ 310 #define SLAB_NO_CMPXCHG (SLAB_CONSISTENCY_CHECKS | SLAB_STORE_USER | \ 311 SLAB_TRACE) 312 313 314 /* 315 * Debugging flags that require metadata to be stored in the slab. These get 316 * disabled when slab_debug=O is used and a cache's min order increases with 317 * metadata. 318 */ 319 #define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER) 320 321 #define OO_SHIFT 16 322 #define OO_MASK ((1 << OO_SHIFT) - 1) 323 #define MAX_OBJS_PER_PAGE 32767 /* since slab.objects is u15 */ 324 325 /* Internal SLUB flags */ 326 /* Poison object */ 327 #define __OBJECT_POISON __SLAB_FLAG_BIT(_SLAB_OBJECT_POISON) 328 /* Use cmpxchg_double */ 329 330 #ifdef system_has_freelist_aba 331 #define __CMPXCHG_DOUBLE __SLAB_FLAG_BIT(_SLAB_CMPXCHG_DOUBLE) 332 #else 333 #define __CMPXCHG_DOUBLE __SLAB_FLAG_UNUSED 334 #endif 335 336 /* 337 * Tracking user of a slab. 338 */ 339 #define TRACK_ADDRS_COUNT 16 340 struct track { 341 unsigned long addr; /* Called from address */ 342 #ifdef CONFIG_STACKDEPOT 343 depot_stack_handle_t handle; 344 #endif 345 int cpu; /* Was running on cpu */ 346 int pid; /* Pid context */ 347 unsigned long when; /* When did the operation occur */ 348 }; 349 350 enum track_item { TRACK_ALLOC, TRACK_FREE }; 351 352 #ifdef SLAB_SUPPORTS_SYSFS 353 static int sysfs_slab_add(struct kmem_cache *); 354 static int sysfs_slab_alias(struct kmem_cache *, const char *); 355 #else 356 static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; } 357 static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p) 358 { return 0; } 359 #endif 360 361 #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG) 362 static void debugfs_slab_add(struct kmem_cache *); 363 #else 364 static inline void debugfs_slab_add(struct kmem_cache *s) { } 365 #endif 366 367 enum stat_item { 368 ALLOC_PCS, /* Allocation from percpu sheaf */ 369 ALLOC_FASTPATH, /* Allocation from cpu slab */ 370 ALLOC_SLOWPATH, /* Allocation by getting a new cpu slab */ 371 FREE_PCS, /* Free to percpu sheaf */ 372 FREE_RCU_SHEAF, /* Free to rcu_free sheaf */ 373 FREE_RCU_SHEAF_FAIL, /* Failed to free to a rcu_free sheaf */ 374 FREE_FASTPATH, /* Free to cpu slab */ 375 FREE_SLOWPATH, /* Freeing not to cpu slab */ 376 FREE_FROZEN, /* Freeing to frozen slab */ 377 FREE_ADD_PARTIAL, /* Freeing moves slab to partial list */ 378 FREE_REMOVE_PARTIAL, /* Freeing removes last object */ 379 ALLOC_FROM_PARTIAL, /* Cpu slab acquired from node partial list */ 380 ALLOC_SLAB, /* Cpu slab acquired from page allocator */ 381 ALLOC_REFILL, /* Refill cpu slab from slab freelist */ 382 ALLOC_NODE_MISMATCH, /* Switching cpu slab */ 383 FREE_SLAB, /* Slab freed to the page allocator */ 384 CPUSLAB_FLUSH, /* Abandoning of the cpu slab */ 385 DEACTIVATE_FULL, /* Cpu slab was full when deactivated */ 386 DEACTIVATE_EMPTY, /* Cpu slab was empty when deactivated */ 387 DEACTIVATE_TO_HEAD, /* Cpu slab was moved to the head of partials */ 388 DEACTIVATE_TO_TAIL, /* Cpu slab was moved to the tail of partials */ 389 DEACTIVATE_REMOTE_FREES,/* Slab contained remotely freed objects */ 390 DEACTIVATE_BYPASS, /* Implicit deactivation */ 391 ORDER_FALLBACK, /* Number of times fallback was necessary */ 392 CMPXCHG_DOUBLE_CPU_FAIL,/* Failures of this_cpu_cmpxchg_double */ 393 CMPXCHG_DOUBLE_FAIL, /* Failures of slab freelist update */ 394 CPU_PARTIAL_ALLOC, /* Used cpu partial on alloc */ 395 CPU_PARTIAL_FREE, /* Refill cpu partial on free */ 396 CPU_PARTIAL_NODE, /* Refill cpu partial from node partial */ 397 CPU_PARTIAL_DRAIN, /* Drain cpu partial to node partial */ 398 SHEAF_FLUSH, /* Objects flushed from a sheaf */ 399 SHEAF_REFILL, /* Objects refilled to a sheaf */ 400 SHEAF_ALLOC, /* Allocation of an empty sheaf */ 401 SHEAF_FREE, /* Freeing of an empty sheaf */ 402 BARN_GET, /* Got full sheaf from barn */ 403 BARN_GET_FAIL, /* Failed to get full sheaf from barn */ 404 BARN_PUT, /* Put full sheaf to barn */ 405 BARN_PUT_FAIL, /* Failed to put full sheaf to barn */ 406 SHEAF_PREFILL_FAST, /* Sheaf prefill grabbed the spare sheaf */ 407 SHEAF_PREFILL_SLOW, /* Sheaf prefill found no spare sheaf */ 408 SHEAF_PREFILL_OVERSIZE, /* Allocation of oversize sheaf for prefill */ 409 SHEAF_RETURN_FAST, /* Sheaf return reattached spare sheaf */ 410 SHEAF_RETURN_SLOW, /* Sheaf return could not reattach spare */ 411 NR_SLUB_STAT_ITEMS 412 }; 413 414 /* 415 * When changing the layout, make sure freelist and tid are still compatible 416 * with this_cpu_cmpxchg_double() alignment requirements. 417 */ 418 struct kmem_cache_cpu { 419 union { 420 struct { 421 void **freelist; /* Pointer to next available object */ 422 unsigned long tid; /* Globally unique transaction id */ 423 }; 424 freelist_aba_t freelist_tid; 425 }; 426 struct slab *slab; /* The slab from which we are allocating */ 427 #ifdef CONFIG_SLUB_CPU_PARTIAL 428 struct slab *partial; /* Partially allocated slabs */ 429 #endif 430 local_trylock_t lock; /* Protects the fields above */ 431 #ifdef CONFIG_SLUB_STATS 432 unsigned int stat[NR_SLUB_STAT_ITEMS]; 433 #endif 434 }; 435 436 static inline void stat(const struct kmem_cache *s, enum stat_item si) 437 { 438 #ifdef CONFIG_SLUB_STATS 439 /* 440 * The rmw is racy on a preemptible kernel but this is acceptable, so 441 * avoid this_cpu_add()'s irq-disable overhead. 442 */ 443 raw_cpu_inc(s->cpu_slab->stat[si]); 444 #endif 445 } 446 447 static inline 448 void stat_add(const struct kmem_cache *s, enum stat_item si, int v) 449 { 450 #ifdef CONFIG_SLUB_STATS 451 raw_cpu_add(s->cpu_slab->stat[si], v); 452 #endif 453 } 454 455 #define MAX_FULL_SHEAVES 10 456 #define MAX_EMPTY_SHEAVES 10 457 458 struct node_barn { 459 spinlock_t lock; 460 struct list_head sheaves_full; 461 struct list_head sheaves_empty; 462 unsigned int nr_full; 463 unsigned int nr_empty; 464 }; 465 466 struct slab_sheaf { 467 union { 468 struct rcu_head rcu_head; 469 struct list_head barn_list; 470 /* only used for prefilled sheafs */ 471 unsigned int capacity; 472 }; 473 struct kmem_cache *cache; 474 unsigned int size; 475 int node; /* only used for rcu_sheaf */ 476 void *objects[]; 477 }; 478 479 struct slub_percpu_sheaves { 480 local_trylock_t lock; 481 struct slab_sheaf *main; /* never NULL when unlocked */ 482 struct slab_sheaf *spare; /* empty or full, may be NULL */ 483 struct slab_sheaf *rcu_free; /* for batching kfree_rcu() */ 484 }; 485 486 /* 487 * The slab lists for all objects. 488 */ 489 struct kmem_cache_node { 490 spinlock_t list_lock; 491 unsigned long nr_partial; 492 struct list_head partial; 493 #ifdef CONFIG_SLUB_DEBUG 494 atomic_long_t nr_slabs; 495 atomic_long_t total_objects; 496 struct list_head full; 497 #endif 498 struct node_barn *barn; 499 }; 500 501 static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node) 502 { 503 return s->node[node]; 504 } 505 506 /* 507 * Get the barn of the current cpu's closest memory node. It may not exist on 508 * systems with memoryless nodes but without CONFIG_HAVE_MEMORYLESS_NODES 509 */ 510 static inline struct node_barn *get_barn(struct kmem_cache *s) 511 { 512 struct kmem_cache_node *n = get_node(s, numa_mem_id()); 513 514 if (!n) 515 return NULL; 516 517 return n->barn; 518 } 519 520 /* 521 * Iterator over all nodes. The body will be executed for each node that has 522 * a kmem_cache_node structure allocated (which is true for all online nodes) 523 */ 524 #define for_each_kmem_cache_node(__s, __node, __n) \ 525 for (__node = 0; __node < nr_node_ids; __node++) \ 526 if ((__n = get_node(__s, __node))) 527 528 /* 529 * Tracks for which NUMA nodes we have kmem_cache_nodes allocated. 530 * Corresponds to node_state[N_MEMORY], but can temporarily 531 * differ during memory hotplug/hotremove operations. 532 * Protected by slab_mutex. 533 */ 534 static nodemask_t slab_nodes; 535 536 /* 537 * Workqueue used for flush_cpu_slab(). 538 */ 539 static struct workqueue_struct *flushwq; 540 541 struct slub_flush_work { 542 struct work_struct work; 543 struct kmem_cache *s; 544 bool skip; 545 }; 546 547 static DEFINE_MUTEX(flush_lock); 548 static DEFINE_PER_CPU(struct slub_flush_work, slub_flush); 549 550 /******************************************************************** 551 * Core slab cache functions 552 *******************************************************************/ 553 554 /* 555 * Returns freelist pointer (ptr). With hardening, this is obfuscated 556 * with an XOR of the address where the pointer is held and a per-cache 557 * random number. 558 */ 559 static inline freeptr_t freelist_ptr_encode(const struct kmem_cache *s, 560 void *ptr, unsigned long ptr_addr) 561 { 562 unsigned long encoded; 563 564 #ifdef CONFIG_SLAB_FREELIST_HARDENED 565 encoded = (unsigned long)ptr ^ s->random ^ swab(ptr_addr); 566 #else 567 encoded = (unsigned long)ptr; 568 #endif 569 return (freeptr_t){.v = encoded}; 570 } 571 572 static inline void *freelist_ptr_decode(const struct kmem_cache *s, 573 freeptr_t ptr, unsigned long ptr_addr) 574 { 575 void *decoded; 576 577 #ifdef CONFIG_SLAB_FREELIST_HARDENED 578 decoded = (void *)(ptr.v ^ s->random ^ swab(ptr_addr)); 579 #else 580 decoded = (void *)ptr.v; 581 #endif 582 return decoded; 583 } 584 585 static inline void *get_freepointer(struct kmem_cache *s, void *object) 586 { 587 unsigned long ptr_addr; 588 freeptr_t p; 589 590 object = kasan_reset_tag(object); 591 ptr_addr = (unsigned long)object + s->offset; 592 p = *(freeptr_t *)(ptr_addr); 593 return freelist_ptr_decode(s, p, ptr_addr); 594 } 595 596 static void prefetch_freepointer(const struct kmem_cache *s, void *object) 597 { 598 prefetchw(object + s->offset); 599 } 600 601 /* 602 * When running under KMSAN, get_freepointer_safe() may return an uninitialized 603 * pointer value in the case the current thread loses the race for the next 604 * memory chunk in the freelist. In that case this_cpu_cmpxchg_double() in 605 * slab_alloc_node() will fail, so the uninitialized value won't be used, but 606 * KMSAN will still check all arguments of cmpxchg because of imperfect 607 * handling of inline assembly. 608 * To work around this problem, we apply __no_kmsan_checks to ensure that 609 * get_freepointer_safe() returns initialized memory. 610 */ 611 __no_kmsan_checks 612 static inline void *get_freepointer_safe(struct kmem_cache *s, void *object) 613 { 614 unsigned long freepointer_addr; 615 freeptr_t p; 616 617 if (!debug_pagealloc_enabled_static()) 618 return get_freepointer(s, object); 619 620 object = kasan_reset_tag(object); 621 freepointer_addr = (unsigned long)object + s->offset; 622 copy_from_kernel_nofault(&p, (freeptr_t *)freepointer_addr, sizeof(p)); 623 return freelist_ptr_decode(s, p, freepointer_addr); 624 } 625 626 static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp) 627 { 628 unsigned long freeptr_addr = (unsigned long)object + s->offset; 629 630 #ifdef CONFIG_SLAB_FREELIST_HARDENED 631 BUG_ON(object == fp); /* naive detection of double free or corruption */ 632 #endif 633 634 freeptr_addr = (unsigned long)kasan_reset_tag((void *)freeptr_addr); 635 *(freeptr_t *)freeptr_addr = freelist_ptr_encode(s, fp, freeptr_addr); 636 } 637 638 /* 639 * See comment in calculate_sizes(). 640 */ 641 static inline bool freeptr_outside_object(struct kmem_cache *s) 642 { 643 return s->offset >= s->inuse; 644 } 645 646 /* 647 * Return offset of the end of info block which is inuse + free pointer if 648 * not overlapping with object. 649 */ 650 static inline unsigned int get_info_end(struct kmem_cache *s) 651 { 652 if (freeptr_outside_object(s)) 653 return s->inuse + sizeof(void *); 654 else 655 return s->inuse; 656 } 657 658 /* Loop over all objects in a slab */ 659 #define for_each_object(__p, __s, __addr, __objects) \ 660 for (__p = fixup_red_left(__s, __addr); \ 661 __p < (__addr) + (__objects) * (__s)->size; \ 662 __p += (__s)->size) 663 664 static inline unsigned int order_objects(unsigned int order, unsigned int size) 665 { 666 return ((unsigned int)PAGE_SIZE << order) / size; 667 } 668 669 static inline struct kmem_cache_order_objects oo_make(unsigned int order, 670 unsigned int size) 671 { 672 struct kmem_cache_order_objects x = { 673 (order << OO_SHIFT) + order_objects(order, size) 674 }; 675 676 return x; 677 } 678 679 static inline unsigned int oo_order(struct kmem_cache_order_objects x) 680 { 681 return x.x >> OO_SHIFT; 682 } 683 684 static inline unsigned int oo_objects(struct kmem_cache_order_objects x) 685 { 686 return x.x & OO_MASK; 687 } 688 689 #ifdef CONFIG_SLUB_CPU_PARTIAL 690 static void slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects) 691 { 692 unsigned int nr_slabs; 693 694 s->cpu_partial = nr_objects; 695 696 /* 697 * We take the number of objects but actually limit the number of 698 * slabs on the per cpu partial list, in order to limit excessive 699 * growth of the list. For simplicity we assume that the slabs will 700 * be half-full. 701 */ 702 nr_slabs = DIV_ROUND_UP(nr_objects * 2, oo_objects(s->oo)); 703 s->cpu_partial_slabs = nr_slabs; 704 } 705 706 static inline unsigned int slub_get_cpu_partial(struct kmem_cache *s) 707 { 708 return s->cpu_partial_slabs; 709 } 710 #else 711 #ifdef SLAB_SUPPORTS_SYSFS 712 static inline void 713 slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects) 714 { 715 } 716 #endif 717 718 static inline unsigned int slub_get_cpu_partial(struct kmem_cache *s) 719 { 720 return 0; 721 } 722 #endif /* CONFIG_SLUB_CPU_PARTIAL */ 723 724 /* 725 * If network-based swap is enabled, slub must keep track of whether memory 726 * were allocated from pfmemalloc reserves. 727 */ 728 static inline bool slab_test_pfmemalloc(const struct slab *slab) 729 { 730 return test_bit(SL_pfmemalloc, &slab->flags.f); 731 } 732 733 static inline void slab_set_pfmemalloc(struct slab *slab) 734 { 735 set_bit(SL_pfmemalloc, &slab->flags.f); 736 } 737 738 static inline void __slab_clear_pfmemalloc(struct slab *slab) 739 { 740 __clear_bit(SL_pfmemalloc, &slab->flags.f); 741 } 742 743 /* 744 * Per slab locking using the pagelock 745 */ 746 static __always_inline void slab_lock(struct slab *slab) 747 { 748 bit_spin_lock(SL_locked, &slab->flags.f); 749 } 750 751 static __always_inline void slab_unlock(struct slab *slab) 752 { 753 bit_spin_unlock(SL_locked, &slab->flags.f); 754 } 755 756 static inline bool 757 __update_freelist_fast(struct slab *slab, 758 void *freelist_old, unsigned long counters_old, 759 void *freelist_new, unsigned long counters_new) 760 { 761 #ifdef system_has_freelist_aba 762 freelist_aba_t old = { .freelist = freelist_old, .counter = counters_old }; 763 freelist_aba_t new = { .freelist = freelist_new, .counter = counters_new }; 764 765 return try_cmpxchg_freelist(&slab->freelist_counter.full, &old.full, new.full); 766 #else 767 return false; 768 #endif 769 } 770 771 static inline bool 772 __update_freelist_slow(struct slab *slab, 773 void *freelist_old, unsigned long counters_old, 774 void *freelist_new, unsigned long counters_new) 775 { 776 bool ret = false; 777 778 slab_lock(slab); 779 if (slab->freelist == freelist_old && 780 slab->counters == counters_old) { 781 slab->freelist = freelist_new; 782 slab->counters = counters_new; 783 ret = true; 784 } 785 slab_unlock(slab); 786 787 return ret; 788 } 789 790 /* 791 * Interrupts must be disabled (for the fallback code to work right), typically 792 * by an _irqsave() lock variant. On PREEMPT_RT the preempt_disable(), which is 793 * part of bit_spin_lock(), is sufficient because the policy is not to allow any 794 * allocation/ free operation in hardirq context. Therefore nothing can 795 * interrupt the operation. 796 */ 797 static inline bool __slab_update_freelist(struct kmem_cache *s, struct slab *slab, 798 void *freelist_old, unsigned long counters_old, 799 void *freelist_new, unsigned long counters_new, 800 const char *n) 801 { 802 bool ret; 803 804 if (USE_LOCKLESS_FAST_PATH()) 805 lockdep_assert_irqs_disabled(); 806 807 if (s->flags & __CMPXCHG_DOUBLE) { 808 ret = __update_freelist_fast(slab, freelist_old, counters_old, 809 freelist_new, counters_new); 810 } else { 811 ret = __update_freelist_slow(slab, freelist_old, counters_old, 812 freelist_new, counters_new); 813 } 814 if (likely(ret)) 815 return true; 816 817 cpu_relax(); 818 stat(s, CMPXCHG_DOUBLE_FAIL); 819 820 #ifdef SLUB_DEBUG_CMPXCHG 821 pr_info("%s %s: cmpxchg double redo ", n, s->name); 822 #endif 823 824 return false; 825 } 826 827 static inline bool slab_update_freelist(struct kmem_cache *s, struct slab *slab, 828 void *freelist_old, unsigned long counters_old, 829 void *freelist_new, unsigned long counters_new, 830 const char *n) 831 { 832 bool ret; 833 834 if (s->flags & __CMPXCHG_DOUBLE) { 835 ret = __update_freelist_fast(slab, freelist_old, counters_old, 836 freelist_new, counters_new); 837 } else { 838 unsigned long flags; 839 840 local_irq_save(flags); 841 ret = __update_freelist_slow(slab, freelist_old, counters_old, 842 freelist_new, counters_new); 843 local_irq_restore(flags); 844 } 845 if (likely(ret)) 846 return true; 847 848 cpu_relax(); 849 stat(s, CMPXCHG_DOUBLE_FAIL); 850 851 #ifdef SLUB_DEBUG_CMPXCHG 852 pr_info("%s %s: cmpxchg double redo ", n, s->name); 853 #endif 854 855 return false; 856 } 857 858 /* 859 * kmalloc caches has fixed sizes (mostly power of 2), and kmalloc() API 860 * family will round up the real request size to these fixed ones, so 861 * there could be an extra area than what is requested. Save the original 862 * request size in the meta data area, for better debug and sanity check. 863 */ 864 static inline void set_orig_size(struct kmem_cache *s, 865 void *object, unsigned long orig_size) 866 { 867 void *p = kasan_reset_tag(object); 868 869 if (!slub_debug_orig_size(s)) 870 return; 871 872 p += get_info_end(s); 873 p += sizeof(struct track) * 2; 874 875 *(unsigned long *)p = orig_size; 876 } 877 878 static inline unsigned long get_orig_size(struct kmem_cache *s, void *object) 879 { 880 void *p = kasan_reset_tag(object); 881 882 if (is_kfence_address(object)) 883 return kfence_ksize(object); 884 885 if (!slub_debug_orig_size(s)) 886 return s->object_size; 887 888 p += get_info_end(s); 889 p += sizeof(struct track) * 2; 890 891 return *(unsigned long *)p; 892 } 893 894 #ifdef CONFIG_SLUB_DEBUG 895 896 /* 897 * For debugging context when we want to check if the struct slab pointer 898 * appears to be valid. 899 */ 900 static inline bool validate_slab_ptr(struct slab *slab) 901 { 902 return PageSlab(slab_page(slab)); 903 } 904 905 static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)]; 906 static DEFINE_SPINLOCK(object_map_lock); 907 908 static void __fill_map(unsigned long *obj_map, struct kmem_cache *s, 909 struct slab *slab) 910 { 911 void *addr = slab_address(slab); 912 void *p; 913 914 bitmap_zero(obj_map, slab->objects); 915 916 for (p = slab->freelist; p; p = get_freepointer(s, p)) 917 set_bit(__obj_to_index(s, addr, p), obj_map); 918 } 919 920 #if IS_ENABLED(CONFIG_KUNIT) 921 static bool slab_add_kunit_errors(void) 922 { 923 struct kunit_resource *resource; 924 925 if (!kunit_get_current_test()) 926 return false; 927 928 resource = kunit_find_named_resource(current->kunit_test, "slab_errors"); 929 if (!resource) 930 return false; 931 932 (*(int *)resource->data)++; 933 kunit_put_resource(resource); 934 return true; 935 } 936 937 bool slab_in_kunit_test(void) 938 { 939 struct kunit_resource *resource; 940 941 if (!kunit_get_current_test()) 942 return false; 943 944 resource = kunit_find_named_resource(current->kunit_test, "slab_errors"); 945 if (!resource) 946 return false; 947 948 kunit_put_resource(resource); 949 return true; 950 } 951 #else 952 static inline bool slab_add_kunit_errors(void) { return false; } 953 #endif 954 955 static inline unsigned int size_from_object(struct kmem_cache *s) 956 { 957 if (s->flags & SLAB_RED_ZONE) 958 return s->size - s->red_left_pad; 959 960 return s->size; 961 } 962 963 static inline void *restore_red_left(struct kmem_cache *s, void *p) 964 { 965 if (s->flags & SLAB_RED_ZONE) 966 p -= s->red_left_pad; 967 968 return p; 969 } 970 971 /* 972 * Debug settings: 973 */ 974 #if defined(CONFIG_SLUB_DEBUG_ON) 975 static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS; 976 #else 977 static slab_flags_t slub_debug; 978 #endif 979 980 static char *slub_debug_string; 981 static int disable_higher_order_debug; 982 983 /* 984 * slub is about to manipulate internal object metadata. This memory lies 985 * outside the range of the allocated object, so accessing it would normally 986 * be reported by kasan as a bounds error. metadata_access_enable() is used 987 * to tell kasan that these accesses are OK. 988 */ 989 static inline void metadata_access_enable(void) 990 { 991 kasan_disable_current(); 992 kmsan_disable_current(); 993 } 994 995 static inline void metadata_access_disable(void) 996 { 997 kmsan_enable_current(); 998 kasan_enable_current(); 999 } 1000 1001 /* 1002 * Object debugging 1003 */ 1004 1005 /* Verify that a pointer has an address that is valid within a slab page */ 1006 static inline int check_valid_pointer(struct kmem_cache *s, 1007 struct slab *slab, void *object) 1008 { 1009 void *base; 1010 1011 if (!object) 1012 return 1; 1013 1014 base = slab_address(slab); 1015 object = kasan_reset_tag(object); 1016 object = restore_red_left(s, object); 1017 if (object < base || object >= base + slab->objects * s->size || 1018 (object - base) % s->size) { 1019 return 0; 1020 } 1021 1022 return 1; 1023 } 1024 1025 static void print_section(char *level, char *text, u8 *addr, 1026 unsigned int length) 1027 { 1028 metadata_access_enable(); 1029 print_hex_dump(level, text, DUMP_PREFIX_ADDRESS, 1030 16, 1, kasan_reset_tag((void *)addr), length, 1); 1031 metadata_access_disable(); 1032 } 1033 1034 static struct track *get_track(struct kmem_cache *s, void *object, 1035 enum track_item alloc) 1036 { 1037 struct track *p; 1038 1039 p = object + get_info_end(s); 1040 1041 return kasan_reset_tag(p + alloc); 1042 } 1043 1044 #ifdef CONFIG_STACKDEPOT 1045 static noinline depot_stack_handle_t set_track_prepare(gfp_t gfp_flags) 1046 { 1047 depot_stack_handle_t handle; 1048 unsigned long entries[TRACK_ADDRS_COUNT]; 1049 unsigned int nr_entries; 1050 1051 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 3); 1052 handle = stack_depot_save(entries, nr_entries, gfp_flags); 1053 1054 return handle; 1055 } 1056 #else 1057 static inline depot_stack_handle_t set_track_prepare(gfp_t gfp_flags) 1058 { 1059 return 0; 1060 } 1061 #endif 1062 1063 static void set_track_update(struct kmem_cache *s, void *object, 1064 enum track_item alloc, unsigned long addr, 1065 depot_stack_handle_t handle) 1066 { 1067 struct track *p = get_track(s, object, alloc); 1068 1069 #ifdef CONFIG_STACKDEPOT 1070 p->handle = handle; 1071 #endif 1072 p->addr = addr; 1073 p->cpu = smp_processor_id(); 1074 p->pid = current->pid; 1075 p->when = jiffies; 1076 } 1077 1078 static __always_inline void set_track(struct kmem_cache *s, void *object, 1079 enum track_item alloc, unsigned long addr, gfp_t gfp_flags) 1080 { 1081 depot_stack_handle_t handle = set_track_prepare(gfp_flags); 1082 1083 set_track_update(s, object, alloc, addr, handle); 1084 } 1085 1086 static void init_tracking(struct kmem_cache *s, void *object) 1087 { 1088 struct track *p; 1089 1090 if (!(s->flags & SLAB_STORE_USER)) 1091 return; 1092 1093 p = get_track(s, object, TRACK_ALLOC); 1094 memset(p, 0, 2*sizeof(struct track)); 1095 } 1096 1097 static void print_track(const char *s, struct track *t, unsigned long pr_time) 1098 { 1099 depot_stack_handle_t handle __maybe_unused; 1100 1101 if (!t->addr) 1102 return; 1103 1104 pr_err("%s in %pS age=%lu cpu=%u pid=%d\n", 1105 s, (void *)t->addr, pr_time - t->when, t->cpu, t->pid); 1106 #ifdef CONFIG_STACKDEPOT 1107 handle = READ_ONCE(t->handle); 1108 if (handle) 1109 stack_depot_print(handle); 1110 else 1111 pr_err("object allocation/free stack trace missing\n"); 1112 #endif 1113 } 1114 1115 void print_tracking(struct kmem_cache *s, void *object) 1116 { 1117 unsigned long pr_time = jiffies; 1118 if (!(s->flags & SLAB_STORE_USER)) 1119 return; 1120 1121 print_track("Allocated", get_track(s, object, TRACK_ALLOC), pr_time); 1122 print_track("Freed", get_track(s, object, TRACK_FREE), pr_time); 1123 } 1124 1125 static void print_slab_info(const struct slab *slab) 1126 { 1127 pr_err("Slab 0x%p objects=%u used=%u fp=0x%p flags=%pGp\n", 1128 slab, slab->objects, slab->inuse, slab->freelist, 1129 &slab->flags.f); 1130 } 1131 1132 void skip_orig_size_check(struct kmem_cache *s, const void *object) 1133 { 1134 set_orig_size(s, (void *)object, s->object_size); 1135 } 1136 1137 static void __slab_bug(struct kmem_cache *s, const char *fmt, va_list argsp) 1138 { 1139 struct va_format vaf; 1140 va_list args; 1141 1142 va_copy(args, argsp); 1143 vaf.fmt = fmt; 1144 vaf.va = &args; 1145 pr_err("=============================================================================\n"); 1146 pr_err("BUG %s (%s): %pV\n", s ? s->name : "<unknown>", print_tainted(), &vaf); 1147 pr_err("-----------------------------------------------------------------------------\n\n"); 1148 va_end(args); 1149 } 1150 1151 static void slab_bug(struct kmem_cache *s, const char *fmt, ...) 1152 { 1153 va_list args; 1154 1155 va_start(args, fmt); 1156 __slab_bug(s, fmt, args); 1157 va_end(args); 1158 } 1159 1160 __printf(2, 3) 1161 static void slab_fix(struct kmem_cache *s, const char *fmt, ...) 1162 { 1163 struct va_format vaf; 1164 va_list args; 1165 1166 if (slab_add_kunit_errors()) 1167 return; 1168 1169 va_start(args, fmt); 1170 vaf.fmt = fmt; 1171 vaf.va = &args; 1172 pr_err("FIX %s: %pV\n", s->name, &vaf); 1173 va_end(args); 1174 } 1175 1176 static void print_trailer(struct kmem_cache *s, struct slab *slab, u8 *p) 1177 { 1178 unsigned int off; /* Offset of last byte */ 1179 u8 *addr = slab_address(slab); 1180 1181 print_tracking(s, p); 1182 1183 print_slab_info(slab); 1184 1185 pr_err("Object 0x%p @offset=%tu fp=0x%p\n\n", 1186 p, p - addr, get_freepointer(s, p)); 1187 1188 if (s->flags & SLAB_RED_ZONE) 1189 print_section(KERN_ERR, "Redzone ", p - s->red_left_pad, 1190 s->red_left_pad); 1191 else if (p > addr + 16) 1192 print_section(KERN_ERR, "Bytes b4 ", p - 16, 16); 1193 1194 print_section(KERN_ERR, "Object ", p, 1195 min_t(unsigned int, s->object_size, PAGE_SIZE)); 1196 if (s->flags & SLAB_RED_ZONE) 1197 print_section(KERN_ERR, "Redzone ", p + s->object_size, 1198 s->inuse - s->object_size); 1199 1200 off = get_info_end(s); 1201 1202 if (s->flags & SLAB_STORE_USER) 1203 off += 2 * sizeof(struct track); 1204 1205 if (slub_debug_orig_size(s)) 1206 off += sizeof(unsigned long); 1207 1208 off += kasan_metadata_size(s, false); 1209 1210 if (off != size_from_object(s)) 1211 /* Beginning of the filler is the free pointer */ 1212 print_section(KERN_ERR, "Padding ", p + off, 1213 size_from_object(s) - off); 1214 } 1215 1216 static void object_err(struct kmem_cache *s, struct slab *slab, 1217 u8 *object, const char *reason) 1218 { 1219 if (slab_add_kunit_errors()) 1220 return; 1221 1222 slab_bug(s, reason); 1223 if (!object || !check_valid_pointer(s, slab, object)) { 1224 print_slab_info(slab); 1225 pr_err("Invalid pointer 0x%p\n", object); 1226 } else { 1227 print_trailer(s, slab, object); 1228 } 1229 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 1230 1231 WARN_ON(1); 1232 } 1233 1234 static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab, 1235 void **freelist, void *nextfree) 1236 { 1237 if ((s->flags & SLAB_CONSISTENCY_CHECKS) && 1238 !check_valid_pointer(s, slab, nextfree) && freelist) { 1239 object_err(s, slab, *freelist, "Freechain corrupt"); 1240 *freelist = NULL; 1241 slab_fix(s, "Isolate corrupted freechain"); 1242 return true; 1243 } 1244 1245 return false; 1246 } 1247 1248 static void __slab_err(struct slab *slab) 1249 { 1250 if (slab_in_kunit_test()) 1251 return; 1252 1253 print_slab_info(slab); 1254 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 1255 1256 WARN_ON(1); 1257 } 1258 1259 static __printf(3, 4) void slab_err(struct kmem_cache *s, struct slab *slab, 1260 const char *fmt, ...) 1261 { 1262 va_list args; 1263 1264 if (slab_add_kunit_errors()) 1265 return; 1266 1267 va_start(args, fmt); 1268 __slab_bug(s, fmt, args); 1269 va_end(args); 1270 1271 __slab_err(slab); 1272 } 1273 1274 static void init_object(struct kmem_cache *s, void *object, u8 val) 1275 { 1276 u8 *p = kasan_reset_tag(object); 1277 unsigned int poison_size = s->object_size; 1278 1279 if (s->flags & SLAB_RED_ZONE) { 1280 /* 1281 * Here and below, avoid overwriting the KMSAN shadow. Keeping 1282 * the shadow makes it possible to distinguish uninit-value 1283 * from use-after-free. 1284 */ 1285 memset_no_sanitize_memory(p - s->red_left_pad, val, 1286 s->red_left_pad); 1287 1288 if (slub_debug_orig_size(s) && val == SLUB_RED_ACTIVE) { 1289 /* 1290 * Redzone the extra allocated space by kmalloc than 1291 * requested, and the poison size will be limited to 1292 * the original request size accordingly. 1293 */ 1294 poison_size = get_orig_size(s, object); 1295 } 1296 } 1297 1298 if (s->flags & __OBJECT_POISON) { 1299 memset_no_sanitize_memory(p, POISON_FREE, poison_size - 1); 1300 memset_no_sanitize_memory(p + poison_size - 1, POISON_END, 1); 1301 } 1302 1303 if (s->flags & SLAB_RED_ZONE) 1304 memset_no_sanitize_memory(p + poison_size, val, 1305 s->inuse - poison_size); 1306 } 1307 1308 static void restore_bytes(struct kmem_cache *s, const char *message, u8 data, 1309 void *from, void *to) 1310 { 1311 slab_fix(s, "Restoring %s 0x%p-0x%p=0x%x", message, from, to - 1, data); 1312 memset(from, data, to - from); 1313 } 1314 1315 #ifdef CONFIG_KMSAN 1316 #define pad_check_attributes noinline __no_kmsan_checks 1317 #else 1318 #define pad_check_attributes 1319 #endif 1320 1321 static pad_check_attributes int 1322 check_bytes_and_report(struct kmem_cache *s, struct slab *slab, 1323 u8 *object, const char *what, u8 *start, unsigned int value, 1324 unsigned int bytes, bool slab_obj_print) 1325 { 1326 u8 *fault; 1327 u8 *end; 1328 u8 *addr = slab_address(slab); 1329 1330 metadata_access_enable(); 1331 fault = memchr_inv(kasan_reset_tag(start), value, bytes); 1332 metadata_access_disable(); 1333 if (!fault) 1334 return 1; 1335 1336 end = start + bytes; 1337 while (end > fault && end[-1] == value) 1338 end--; 1339 1340 if (slab_add_kunit_errors()) 1341 goto skip_bug_print; 1342 1343 pr_err("[%s overwritten] 0x%p-0x%p @offset=%tu. First byte 0x%x instead of 0x%x\n", 1344 what, fault, end - 1, fault - addr, fault[0], value); 1345 1346 if (slab_obj_print) 1347 object_err(s, slab, object, "Object corrupt"); 1348 1349 skip_bug_print: 1350 restore_bytes(s, what, value, fault, end); 1351 return 0; 1352 } 1353 1354 /* 1355 * Object layout: 1356 * 1357 * object address 1358 * Bytes of the object to be managed. 1359 * If the freepointer may overlay the object then the free 1360 * pointer is at the middle of the object. 1361 * 1362 * Poisoning uses 0x6b (POISON_FREE) and the last byte is 1363 * 0xa5 (POISON_END) 1364 * 1365 * object + s->object_size 1366 * Padding to reach word boundary. This is also used for Redzoning. 1367 * Padding is extended by another word if Redzoning is enabled and 1368 * object_size == inuse. 1369 * 1370 * We fill with 0xbb (SLUB_RED_INACTIVE) for inactive objects and with 1371 * 0xcc (SLUB_RED_ACTIVE) for objects in use. 1372 * 1373 * object + s->inuse 1374 * Meta data starts here. 1375 * 1376 * A. Free pointer (if we cannot overwrite object on free) 1377 * B. Tracking data for SLAB_STORE_USER 1378 * C. Original request size for kmalloc object (SLAB_STORE_USER enabled) 1379 * D. Padding to reach required alignment boundary or at minimum 1380 * one word if debugging is on to be able to detect writes 1381 * before the word boundary. 1382 * 1383 * Padding is done using 0x5a (POISON_INUSE) 1384 * 1385 * object + s->size 1386 * Nothing is used beyond s->size. 1387 * 1388 * If slabcaches are merged then the object_size and inuse boundaries are mostly 1389 * ignored. And therefore no slab options that rely on these boundaries 1390 * may be used with merged slabcaches. 1391 */ 1392 1393 static int check_pad_bytes(struct kmem_cache *s, struct slab *slab, u8 *p) 1394 { 1395 unsigned long off = get_info_end(s); /* The end of info */ 1396 1397 if (s->flags & SLAB_STORE_USER) { 1398 /* We also have user information there */ 1399 off += 2 * sizeof(struct track); 1400 1401 if (s->flags & SLAB_KMALLOC) 1402 off += sizeof(unsigned long); 1403 } 1404 1405 off += kasan_metadata_size(s, false); 1406 1407 if (size_from_object(s) == off) 1408 return 1; 1409 1410 return check_bytes_and_report(s, slab, p, "Object padding", 1411 p + off, POISON_INUSE, size_from_object(s) - off, true); 1412 } 1413 1414 /* Check the pad bytes at the end of a slab page */ 1415 static pad_check_attributes void 1416 slab_pad_check(struct kmem_cache *s, struct slab *slab) 1417 { 1418 u8 *start; 1419 u8 *fault; 1420 u8 *end; 1421 u8 *pad; 1422 int length; 1423 int remainder; 1424 1425 if (!(s->flags & SLAB_POISON)) 1426 return; 1427 1428 start = slab_address(slab); 1429 length = slab_size(slab); 1430 end = start + length; 1431 remainder = length % s->size; 1432 if (!remainder) 1433 return; 1434 1435 pad = end - remainder; 1436 metadata_access_enable(); 1437 fault = memchr_inv(kasan_reset_tag(pad), POISON_INUSE, remainder); 1438 metadata_access_disable(); 1439 if (!fault) 1440 return; 1441 while (end > fault && end[-1] == POISON_INUSE) 1442 end--; 1443 1444 slab_bug(s, "Padding overwritten. 0x%p-0x%p @offset=%tu", 1445 fault, end - 1, fault - start); 1446 print_section(KERN_ERR, "Padding ", pad, remainder); 1447 __slab_err(slab); 1448 1449 restore_bytes(s, "slab padding", POISON_INUSE, fault, end); 1450 } 1451 1452 static int check_object(struct kmem_cache *s, struct slab *slab, 1453 void *object, u8 val) 1454 { 1455 u8 *p = object; 1456 u8 *endobject = object + s->object_size; 1457 unsigned int orig_size, kasan_meta_size; 1458 int ret = 1; 1459 1460 if (s->flags & SLAB_RED_ZONE) { 1461 if (!check_bytes_and_report(s, slab, object, "Left Redzone", 1462 object - s->red_left_pad, val, s->red_left_pad, ret)) 1463 ret = 0; 1464 1465 if (!check_bytes_and_report(s, slab, object, "Right Redzone", 1466 endobject, val, s->inuse - s->object_size, ret)) 1467 ret = 0; 1468 1469 if (slub_debug_orig_size(s) && val == SLUB_RED_ACTIVE) { 1470 orig_size = get_orig_size(s, object); 1471 1472 if (s->object_size > orig_size && 1473 !check_bytes_and_report(s, slab, object, 1474 "kmalloc Redzone", p + orig_size, 1475 val, s->object_size - orig_size, ret)) { 1476 ret = 0; 1477 } 1478 } 1479 } else { 1480 if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) { 1481 if (!check_bytes_and_report(s, slab, p, "Alignment padding", 1482 endobject, POISON_INUSE, 1483 s->inuse - s->object_size, ret)) 1484 ret = 0; 1485 } 1486 } 1487 1488 if (s->flags & SLAB_POISON) { 1489 if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON)) { 1490 /* 1491 * KASAN can save its free meta data inside of the 1492 * object at offset 0. Thus, skip checking the part of 1493 * the redzone that overlaps with the meta data. 1494 */ 1495 kasan_meta_size = kasan_metadata_size(s, true); 1496 if (kasan_meta_size < s->object_size - 1 && 1497 !check_bytes_and_report(s, slab, p, "Poison", 1498 p + kasan_meta_size, POISON_FREE, 1499 s->object_size - kasan_meta_size - 1, ret)) 1500 ret = 0; 1501 if (kasan_meta_size < s->object_size && 1502 !check_bytes_and_report(s, slab, p, "End Poison", 1503 p + s->object_size - 1, POISON_END, 1, ret)) 1504 ret = 0; 1505 } 1506 /* 1507 * check_pad_bytes cleans up on its own. 1508 */ 1509 if (!check_pad_bytes(s, slab, p)) 1510 ret = 0; 1511 } 1512 1513 /* 1514 * Cannot check freepointer while object is allocated if 1515 * object and freepointer overlap. 1516 */ 1517 if ((freeptr_outside_object(s) || val != SLUB_RED_ACTIVE) && 1518 !check_valid_pointer(s, slab, get_freepointer(s, p))) { 1519 object_err(s, slab, p, "Freepointer corrupt"); 1520 /* 1521 * No choice but to zap it and thus lose the remainder 1522 * of the free objects in this slab. May cause 1523 * another error because the object count is now wrong. 1524 */ 1525 set_freepointer(s, p, NULL); 1526 ret = 0; 1527 } 1528 1529 return ret; 1530 } 1531 1532 /* 1533 * Checks if the slab state looks sane. Assumes the struct slab pointer 1534 * was either obtained in a way that ensures it's valid, or validated 1535 * by validate_slab_ptr() 1536 */ 1537 static int check_slab(struct kmem_cache *s, struct slab *slab) 1538 { 1539 int maxobj; 1540 1541 maxobj = order_objects(slab_order(slab), s->size); 1542 if (slab->objects > maxobj) { 1543 slab_err(s, slab, "objects %u > max %u", 1544 slab->objects, maxobj); 1545 return 0; 1546 } 1547 if (slab->inuse > slab->objects) { 1548 slab_err(s, slab, "inuse %u > max %u", 1549 slab->inuse, slab->objects); 1550 return 0; 1551 } 1552 if (slab->frozen) { 1553 slab_err(s, slab, "Slab disabled since SLUB metadata consistency check failed"); 1554 return 0; 1555 } 1556 1557 /* Slab_pad_check fixes things up after itself */ 1558 slab_pad_check(s, slab); 1559 return 1; 1560 } 1561 1562 /* 1563 * Determine if a certain object in a slab is on the freelist. Must hold the 1564 * slab lock to guarantee that the chains are in a consistent state. 1565 */ 1566 static bool on_freelist(struct kmem_cache *s, struct slab *slab, void *search) 1567 { 1568 int nr = 0; 1569 void *fp; 1570 void *object = NULL; 1571 int max_objects; 1572 1573 fp = slab->freelist; 1574 while (fp && nr <= slab->objects) { 1575 if (fp == search) 1576 return true; 1577 if (!check_valid_pointer(s, slab, fp)) { 1578 if (object) { 1579 object_err(s, slab, object, 1580 "Freechain corrupt"); 1581 set_freepointer(s, object, NULL); 1582 break; 1583 } else { 1584 slab_err(s, slab, "Freepointer corrupt"); 1585 slab->freelist = NULL; 1586 slab->inuse = slab->objects; 1587 slab_fix(s, "Freelist cleared"); 1588 return false; 1589 } 1590 } 1591 object = fp; 1592 fp = get_freepointer(s, object); 1593 nr++; 1594 } 1595 1596 if (nr > slab->objects) { 1597 slab_err(s, slab, "Freelist cycle detected"); 1598 slab->freelist = NULL; 1599 slab->inuse = slab->objects; 1600 slab_fix(s, "Freelist cleared"); 1601 return false; 1602 } 1603 1604 max_objects = order_objects(slab_order(slab), s->size); 1605 if (max_objects > MAX_OBJS_PER_PAGE) 1606 max_objects = MAX_OBJS_PER_PAGE; 1607 1608 if (slab->objects != max_objects) { 1609 slab_err(s, slab, "Wrong number of objects. Found %d but should be %d", 1610 slab->objects, max_objects); 1611 slab->objects = max_objects; 1612 slab_fix(s, "Number of objects adjusted"); 1613 } 1614 if (slab->inuse != slab->objects - nr) { 1615 slab_err(s, slab, "Wrong object count. Counter is %d but counted were %d", 1616 slab->inuse, slab->objects - nr); 1617 slab->inuse = slab->objects - nr; 1618 slab_fix(s, "Object count adjusted"); 1619 } 1620 return search == NULL; 1621 } 1622 1623 static void trace(struct kmem_cache *s, struct slab *slab, void *object, 1624 int alloc) 1625 { 1626 if (s->flags & SLAB_TRACE) { 1627 pr_info("TRACE %s %s 0x%p inuse=%d fp=0x%p\n", 1628 s->name, 1629 alloc ? "alloc" : "free", 1630 object, slab->inuse, 1631 slab->freelist); 1632 1633 if (!alloc) 1634 print_section(KERN_INFO, "Object ", (void *)object, 1635 s->object_size); 1636 1637 dump_stack(); 1638 } 1639 } 1640 1641 /* 1642 * Tracking of fully allocated slabs for debugging purposes. 1643 */ 1644 static void add_full(struct kmem_cache *s, 1645 struct kmem_cache_node *n, struct slab *slab) 1646 { 1647 if (!(s->flags & SLAB_STORE_USER)) 1648 return; 1649 1650 lockdep_assert_held(&n->list_lock); 1651 list_add(&slab->slab_list, &n->full); 1652 } 1653 1654 static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct slab *slab) 1655 { 1656 if (!(s->flags & SLAB_STORE_USER)) 1657 return; 1658 1659 lockdep_assert_held(&n->list_lock); 1660 list_del(&slab->slab_list); 1661 } 1662 1663 static inline unsigned long node_nr_slabs(struct kmem_cache_node *n) 1664 { 1665 return atomic_long_read(&n->nr_slabs); 1666 } 1667 1668 static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects) 1669 { 1670 struct kmem_cache_node *n = get_node(s, node); 1671 1672 atomic_long_inc(&n->nr_slabs); 1673 atomic_long_add(objects, &n->total_objects); 1674 } 1675 static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects) 1676 { 1677 struct kmem_cache_node *n = get_node(s, node); 1678 1679 atomic_long_dec(&n->nr_slabs); 1680 atomic_long_sub(objects, &n->total_objects); 1681 } 1682 1683 /* Object debug checks for alloc/free paths */ 1684 static void setup_object_debug(struct kmem_cache *s, void *object) 1685 { 1686 if (!kmem_cache_debug_flags(s, SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON)) 1687 return; 1688 1689 init_object(s, object, SLUB_RED_INACTIVE); 1690 init_tracking(s, object); 1691 } 1692 1693 static 1694 void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr) 1695 { 1696 if (!kmem_cache_debug_flags(s, SLAB_POISON)) 1697 return; 1698 1699 metadata_access_enable(); 1700 memset(kasan_reset_tag(addr), POISON_INUSE, slab_size(slab)); 1701 metadata_access_disable(); 1702 } 1703 1704 static inline int alloc_consistency_checks(struct kmem_cache *s, 1705 struct slab *slab, void *object) 1706 { 1707 if (!check_slab(s, slab)) 1708 return 0; 1709 1710 if (!check_valid_pointer(s, slab, object)) { 1711 object_err(s, slab, object, "Freelist Pointer check fails"); 1712 return 0; 1713 } 1714 1715 if (!check_object(s, slab, object, SLUB_RED_INACTIVE)) 1716 return 0; 1717 1718 return 1; 1719 } 1720 1721 static noinline bool alloc_debug_processing(struct kmem_cache *s, 1722 struct slab *slab, void *object, int orig_size) 1723 { 1724 if (s->flags & SLAB_CONSISTENCY_CHECKS) { 1725 if (!alloc_consistency_checks(s, slab, object)) 1726 goto bad; 1727 } 1728 1729 /* Success. Perform special debug activities for allocs */ 1730 trace(s, slab, object, 1); 1731 set_orig_size(s, object, orig_size); 1732 init_object(s, object, SLUB_RED_ACTIVE); 1733 return true; 1734 1735 bad: 1736 /* 1737 * Let's do the best we can to avoid issues in the future. Marking all 1738 * objects as used avoids touching the remaining objects. 1739 */ 1740 slab_fix(s, "Marking all objects used"); 1741 slab->inuse = slab->objects; 1742 slab->freelist = NULL; 1743 slab->frozen = 1; /* mark consistency-failed slab as frozen */ 1744 1745 return false; 1746 } 1747 1748 static inline int free_consistency_checks(struct kmem_cache *s, 1749 struct slab *slab, void *object, unsigned long addr) 1750 { 1751 if (!check_valid_pointer(s, slab, object)) { 1752 slab_err(s, slab, "Invalid object pointer 0x%p", object); 1753 return 0; 1754 } 1755 1756 if (on_freelist(s, slab, object)) { 1757 object_err(s, slab, object, "Object already free"); 1758 return 0; 1759 } 1760 1761 if (!check_object(s, slab, object, SLUB_RED_ACTIVE)) 1762 return 0; 1763 1764 if (unlikely(s != slab->slab_cache)) { 1765 if (!slab->slab_cache) { 1766 slab_err(NULL, slab, "No slab cache for object 0x%p", 1767 object); 1768 } else { 1769 object_err(s, slab, object, 1770 "page slab pointer corrupt."); 1771 } 1772 return 0; 1773 } 1774 return 1; 1775 } 1776 1777 /* 1778 * Parse a block of slab_debug options. Blocks are delimited by ';' 1779 * 1780 * @str: start of block 1781 * @flags: returns parsed flags, or DEBUG_DEFAULT_FLAGS if none specified 1782 * @slabs: return start of list of slabs, or NULL when there's no list 1783 * @init: assume this is initial parsing and not per-kmem-create parsing 1784 * 1785 * returns the start of next block if there's any, or NULL 1786 */ 1787 static char * 1788 parse_slub_debug_flags(char *str, slab_flags_t *flags, char **slabs, bool init) 1789 { 1790 bool higher_order_disable = false; 1791 1792 /* Skip any completely empty blocks */ 1793 while (*str && *str == ';') 1794 str++; 1795 1796 if (*str == ',') { 1797 /* 1798 * No options but restriction on slabs. This means full 1799 * debugging for slabs matching a pattern. 1800 */ 1801 *flags = DEBUG_DEFAULT_FLAGS; 1802 goto check_slabs; 1803 } 1804 *flags = 0; 1805 1806 /* Determine which debug features should be switched on */ 1807 for (; *str && *str != ',' && *str != ';'; str++) { 1808 switch (tolower(*str)) { 1809 case '-': 1810 *flags = 0; 1811 break; 1812 case 'f': 1813 *flags |= SLAB_CONSISTENCY_CHECKS; 1814 break; 1815 case 'z': 1816 *flags |= SLAB_RED_ZONE; 1817 break; 1818 case 'p': 1819 *flags |= SLAB_POISON; 1820 break; 1821 case 'u': 1822 *flags |= SLAB_STORE_USER; 1823 break; 1824 case 't': 1825 *flags |= SLAB_TRACE; 1826 break; 1827 case 'a': 1828 *flags |= SLAB_FAILSLAB; 1829 break; 1830 case 'o': 1831 /* 1832 * Avoid enabling debugging on caches if its minimum 1833 * order would increase as a result. 1834 */ 1835 higher_order_disable = true; 1836 break; 1837 default: 1838 if (init) 1839 pr_err("slab_debug option '%c' unknown. skipped\n", *str); 1840 } 1841 } 1842 check_slabs: 1843 if (*str == ',') 1844 *slabs = ++str; 1845 else 1846 *slabs = NULL; 1847 1848 /* Skip over the slab list */ 1849 while (*str && *str != ';') 1850 str++; 1851 1852 /* Skip any completely empty blocks */ 1853 while (*str && *str == ';') 1854 str++; 1855 1856 if (init && higher_order_disable) 1857 disable_higher_order_debug = 1; 1858 1859 if (*str) 1860 return str; 1861 else 1862 return NULL; 1863 } 1864 1865 static int __init setup_slub_debug(char *str) 1866 { 1867 slab_flags_t flags; 1868 slab_flags_t global_flags; 1869 char *saved_str; 1870 char *slab_list; 1871 bool global_slub_debug_changed = false; 1872 bool slab_list_specified = false; 1873 1874 global_flags = DEBUG_DEFAULT_FLAGS; 1875 if (*str++ != '=' || !*str) 1876 /* 1877 * No options specified. Switch on full debugging. 1878 */ 1879 goto out; 1880 1881 saved_str = str; 1882 while (str) { 1883 str = parse_slub_debug_flags(str, &flags, &slab_list, true); 1884 1885 if (!slab_list) { 1886 global_flags = flags; 1887 global_slub_debug_changed = true; 1888 } else { 1889 slab_list_specified = true; 1890 if (flags & SLAB_STORE_USER) 1891 stack_depot_request_early_init(); 1892 } 1893 } 1894 1895 /* 1896 * For backwards compatibility, a single list of flags with list of 1897 * slabs means debugging is only changed for those slabs, so the global 1898 * slab_debug should be unchanged (0 or DEBUG_DEFAULT_FLAGS, depending 1899 * on CONFIG_SLUB_DEBUG_ON). We can extended that to multiple lists as 1900 * long as there is no option specifying flags without a slab list. 1901 */ 1902 if (slab_list_specified) { 1903 if (!global_slub_debug_changed) 1904 global_flags = slub_debug; 1905 slub_debug_string = saved_str; 1906 } 1907 out: 1908 slub_debug = global_flags; 1909 if (slub_debug & SLAB_STORE_USER) 1910 stack_depot_request_early_init(); 1911 if (slub_debug != 0 || slub_debug_string) 1912 static_branch_enable(&slub_debug_enabled); 1913 else 1914 static_branch_disable(&slub_debug_enabled); 1915 if ((static_branch_unlikely(&init_on_alloc) || 1916 static_branch_unlikely(&init_on_free)) && 1917 (slub_debug & SLAB_POISON)) 1918 pr_info("mem auto-init: SLAB_POISON will take precedence over init_on_alloc/init_on_free\n"); 1919 return 1; 1920 } 1921 1922 __setup("slab_debug", setup_slub_debug); 1923 __setup_param("slub_debug", slub_debug, setup_slub_debug, 0); 1924 1925 /* 1926 * kmem_cache_flags - apply debugging options to the cache 1927 * @flags: flags to set 1928 * @name: name of the cache 1929 * 1930 * Debug option(s) are applied to @flags. In addition to the debug 1931 * option(s), if a slab name (or multiple) is specified i.e. 1932 * slab_debug=<Debug-Options>,<slab name1>,<slab name2> ... 1933 * then only the select slabs will receive the debug option(s). 1934 */ 1935 slab_flags_t kmem_cache_flags(slab_flags_t flags, const char *name) 1936 { 1937 char *iter; 1938 size_t len; 1939 char *next_block; 1940 slab_flags_t block_flags; 1941 slab_flags_t slub_debug_local = slub_debug; 1942 1943 if (flags & SLAB_NO_USER_FLAGS) 1944 return flags; 1945 1946 /* 1947 * If the slab cache is for debugging (e.g. kmemleak) then 1948 * don't store user (stack trace) information by default, 1949 * but let the user enable it via the command line below. 1950 */ 1951 if (flags & SLAB_NOLEAKTRACE) 1952 slub_debug_local &= ~SLAB_STORE_USER; 1953 1954 len = strlen(name); 1955 next_block = slub_debug_string; 1956 /* Go through all blocks of debug options, see if any matches our slab's name */ 1957 while (next_block) { 1958 next_block = parse_slub_debug_flags(next_block, &block_flags, &iter, false); 1959 if (!iter) 1960 continue; 1961 /* Found a block that has a slab list, search it */ 1962 while (*iter) { 1963 char *end, *glob; 1964 size_t cmplen; 1965 1966 end = strchrnul(iter, ','); 1967 if (next_block && next_block < end) 1968 end = next_block - 1; 1969 1970 glob = strnchr(iter, end - iter, '*'); 1971 if (glob) 1972 cmplen = glob - iter; 1973 else 1974 cmplen = max_t(size_t, len, (end - iter)); 1975 1976 if (!strncmp(name, iter, cmplen)) { 1977 flags |= block_flags; 1978 return flags; 1979 } 1980 1981 if (!*end || *end == ';') 1982 break; 1983 iter = end + 1; 1984 } 1985 } 1986 1987 return flags | slub_debug_local; 1988 } 1989 #else /* !CONFIG_SLUB_DEBUG */ 1990 static inline void setup_object_debug(struct kmem_cache *s, void *object) {} 1991 static inline 1992 void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr) {} 1993 1994 static inline bool alloc_debug_processing(struct kmem_cache *s, 1995 struct slab *slab, void *object, int orig_size) { return true; } 1996 1997 static inline bool free_debug_processing(struct kmem_cache *s, 1998 struct slab *slab, void *head, void *tail, int *bulk_cnt, 1999 unsigned long addr, depot_stack_handle_t handle) { return true; } 2000 2001 static inline void slab_pad_check(struct kmem_cache *s, struct slab *slab) {} 2002 static inline int check_object(struct kmem_cache *s, struct slab *slab, 2003 void *object, u8 val) { return 1; } 2004 static inline depot_stack_handle_t set_track_prepare(gfp_t gfp_flags) { return 0; } 2005 static inline void set_track(struct kmem_cache *s, void *object, 2006 enum track_item alloc, unsigned long addr, gfp_t gfp_flags) {} 2007 static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n, 2008 struct slab *slab) {} 2009 static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, 2010 struct slab *slab) {} 2011 slab_flags_t kmem_cache_flags(slab_flags_t flags, const char *name) 2012 { 2013 return flags; 2014 } 2015 #define slub_debug 0 2016 2017 #define disable_higher_order_debug 0 2018 2019 static inline unsigned long node_nr_slabs(struct kmem_cache_node *n) 2020 { return 0; } 2021 static inline void inc_slabs_node(struct kmem_cache *s, int node, 2022 int objects) {} 2023 static inline void dec_slabs_node(struct kmem_cache *s, int node, 2024 int objects) {} 2025 static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab, 2026 void **freelist, void *nextfree) 2027 { 2028 return false; 2029 } 2030 #endif /* CONFIG_SLUB_DEBUG */ 2031 2032 #ifdef CONFIG_SLAB_OBJ_EXT 2033 2034 #ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG 2035 2036 static inline void mark_objexts_empty(struct slabobj_ext *obj_exts) 2037 { 2038 struct slabobj_ext *slab_exts; 2039 struct slab *obj_exts_slab; 2040 2041 obj_exts_slab = virt_to_slab(obj_exts); 2042 slab_exts = slab_obj_exts(obj_exts_slab); 2043 if (slab_exts) { 2044 unsigned int offs = obj_to_index(obj_exts_slab->slab_cache, 2045 obj_exts_slab, obj_exts); 2046 2047 if (unlikely(is_codetag_empty(&slab_exts[offs].ref))) 2048 return; 2049 2050 /* codetag should be NULL here */ 2051 WARN_ON(slab_exts[offs].ref.ct); 2052 set_codetag_empty(&slab_exts[offs].ref); 2053 } 2054 } 2055 2056 static inline bool mark_failed_objexts_alloc(struct slab *slab) 2057 { 2058 return cmpxchg(&slab->obj_exts, 0, OBJEXTS_ALLOC_FAIL) == 0; 2059 } 2060 2061 static inline void handle_failed_objexts_alloc(unsigned long obj_exts, 2062 struct slabobj_ext *vec, unsigned int objects) 2063 { 2064 /* 2065 * If vector previously failed to allocate then we have live 2066 * objects with no tag reference. Mark all references in this 2067 * vector as empty to avoid warnings later on. 2068 */ 2069 if (obj_exts == OBJEXTS_ALLOC_FAIL) { 2070 unsigned int i; 2071 2072 for (i = 0; i < objects; i++) 2073 set_codetag_empty(&vec[i].ref); 2074 } 2075 } 2076 2077 #else /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ 2078 2079 static inline void mark_objexts_empty(struct slabobj_ext *obj_exts) {} 2080 static inline bool mark_failed_objexts_alloc(struct slab *slab) { return false; } 2081 static inline void handle_failed_objexts_alloc(unsigned long obj_exts, 2082 struct slabobj_ext *vec, unsigned int objects) {} 2083 2084 #endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ 2085 2086 /* 2087 * The allocated objcg pointers array is not accounted directly. 2088 * Moreover, it should not come from DMA buffer and is not readily 2089 * reclaimable. So those GFP bits should be masked off. 2090 */ 2091 #define OBJCGS_CLEAR_MASK (__GFP_DMA | __GFP_RECLAIMABLE | \ 2092 __GFP_ACCOUNT | __GFP_NOFAIL) 2093 2094 static inline void init_slab_obj_exts(struct slab *slab) 2095 { 2096 slab->obj_exts = 0; 2097 } 2098 2099 /* 2100 * Calculate the allocation size for slabobj_ext array. 2101 * 2102 * When memory allocation profiling is enabled, the obj_exts array 2103 * could be allocated from the same slab cache it's being allocated for. 2104 * This would prevent the slab from ever being freed because it would 2105 * always contain at least one allocated object (its own obj_exts array). 2106 * 2107 * To avoid this, increase the allocation size when we detect the array 2108 * may come from the same cache, forcing it to use a different cache. 2109 */ 2110 static inline size_t obj_exts_alloc_size(struct kmem_cache *s, 2111 struct slab *slab, gfp_t gfp) 2112 { 2113 size_t sz = sizeof(struct slabobj_ext) * slab->objects; 2114 struct kmem_cache *obj_exts_cache; 2115 2116 if (sz > KMALLOC_MAX_CACHE_SIZE) 2117 return sz; 2118 2119 if (!is_kmalloc_normal(s)) 2120 return sz; 2121 2122 obj_exts_cache = kmalloc_slab(sz, NULL, gfp, 0); 2123 /* 2124 * We can't simply compare s with obj_exts_cache, because random kmalloc 2125 * caches have multiple caches per size, selected by caller address. 2126 * Since caller address may differ between kmalloc_slab() and actual 2127 * allocation, bump size when sizes are equal. 2128 */ 2129 if (s->object_size == obj_exts_cache->object_size) 2130 return obj_exts_cache->object_size + 1; 2131 2132 return sz; 2133 } 2134 2135 int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, 2136 gfp_t gfp, bool new_slab) 2137 { 2138 bool allow_spin = gfpflags_allow_spinning(gfp); 2139 unsigned int objects = objs_per_slab(s, slab); 2140 unsigned long new_exts; 2141 unsigned long old_exts; 2142 struct slabobj_ext *vec; 2143 size_t sz; 2144 2145 gfp &= ~OBJCGS_CLEAR_MASK; 2146 /* Prevent recursive extension vector allocation */ 2147 gfp |= __GFP_NO_OBJ_EXT; 2148 2149 sz = obj_exts_alloc_size(s, slab, gfp); 2150 2151 /* 2152 * Note that allow_spin may be false during early boot and its 2153 * restricted GFP_BOOT_MASK. Due to kmalloc_nolock() only supporting 2154 * architectures with cmpxchg16b, early obj_exts will be missing for 2155 * very early allocations on those. 2156 */ 2157 if (unlikely(!allow_spin)) 2158 vec = kmalloc_nolock(sz, __GFP_ZERO | __GFP_NO_OBJ_EXT, 2159 slab_nid(slab)); 2160 else 2161 vec = kmalloc_node(sz, gfp | __GFP_ZERO, slab_nid(slab)); 2162 2163 if (!vec) { 2164 /* 2165 * Try to mark vectors which failed to allocate. 2166 * If this operation fails, there may be a racing process 2167 * that has already completed the allocation. 2168 */ 2169 if (!mark_failed_objexts_alloc(slab) && 2170 slab_obj_exts(slab)) 2171 return 0; 2172 2173 return -ENOMEM; 2174 } 2175 2176 VM_WARN_ON_ONCE(virt_to_slab(vec) != NULL && 2177 virt_to_slab(vec)->slab_cache == s); 2178 2179 new_exts = (unsigned long)vec; 2180 if (unlikely(!allow_spin)) 2181 new_exts |= OBJEXTS_NOSPIN_ALLOC; 2182 #ifdef CONFIG_MEMCG 2183 new_exts |= MEMCG_DATA_OBJEXTS; 2184 #endif 2185 retry: 2186 old_exts = READ_ONCE(slab->obj_exts); 2187 handle_failed_objexts_alloc(old_exts, vec, objects); 2188 if (new_slab) { 2189 /* 2190 * If the slab is brand new and nobody can yet access its 2191 * obj_exts, no synchronization is required and obj_exts can 2192 * be simply assigned. 2193 */ 2194 slab->obj_exts = new_exts; 2195 } else if (old_exts & ~OBJEXTS_FLAGS_MASK) { 2196 /* 2197 * If the slab is already in use, somebody can allocate and 2198 * assign slabobj_exts in parallel. In this case the existing 2199 * objcg vector should be reused. 2200 */ 2201 mark_objexts_empty(vec); 2202 if (unlikely(!allow_spin)) 2203 kfree_nolock(vec); 2204 else 2205 kfree(vec); 2206 return 0; 2207 } else if (cmpxchg(&slab->obj_exts, old_exts, new_exts) != old_exts) { 2208 /* Retry if a racing thread changed slab->obj_exts from under us. */ 2209 goto retry; 2210 } 2211 2212 if (allow_spin) 2213 kmemleak_not_leak(vec); 2214 return 0; 2215 } 2216 2217 static inline void free_slab_obj_exts(struct slab *slab) 2218 { 2219 struct slabobj_ext *obj_exts; 2220 2221 obj_exts = slab_obj_exts(slab); 2222 if (!obj_exts) { 2223 /* 2224 * If obj_exts allocation failed, slab->obj_exts is set to 2225 * OBJEXTS_ALLOC_FAIL. In this case, we end up here and should 2226 * clear the flag. 2227 */ 2228 slab->obj_exts = 0; 2229 return; 2230 } 2231 2232 /* 2233 * obj_exts was created with __GFP_NO_OBJ_EXT flag, therefore its 2234 * corresponding extension will be NULL. alloc_tag_sub() will throw a 2235 * warning if slab has extensions but the extension of an object is 2236 * NULL, therefore replace NULL with CODETAG_EMPTY to indicate that 2237 * the extension for obj_exts is expected to be NULL. 2238 */ 2239 mark_objexts_empty(obj_exts); 2240 if (unlikely(READ_ONCE(slab->obj_exts) & OBJEXTS_NOSPIN_ALLOC)) 2241 kfree_nolock(obj_exts); 2242 else 2243 kfree(obj_exts); 2244 slab->obj_exts = 0; 2245 } 2246 2247 #else /* CONFIG_SLAB_OBJ_EXT */ 2248 2249 static inline void init_slab_obj_exts(struct slab *slab) 2250 { 2251 } 2252 2253 static int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, 2254 gfp_t gfp, bool new_slab) 2255 { 2256 return 0; 2257 } 2258 2259 static inline void free_slab_obj_exts(struct slab *slab) 2260 { 2261 } 2262 2263 #endif /* CONFIG_SLAB_OBJ_EXT */ 2264 2265 #ifdef CONFIG_MEM_ALLOC_PROFILING 2266 2267 static inline struct slabobj_ext * 2268 prepare_slab_obj_exts_hook(struct kmem_cache *s, gfp_t flags, void *p) 2269 { 2270 struct slab *slab; 2271 2272 slab = virt_to_slab(p); 2273 if (!slab_obj_exts(slab) && 2274 alloc_slab_obj_exts(slab, s, flags, false)) { 2275 pr_warn_once("%s, %s: Failed to create slab extension vector!\n", 2276 __func__, s->name); 2277 return NULL; 2278 } 2279 2280 return slab_obj_exts(slab) + obj_to_index(s, slab, p); 2281 } 2282 2283 /* Should be called only if mem_alloc_profiling_enabled() */ 2284 static noinline void 2285 __alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags) 2286 { 2287 struct slabobj_ext *obj_exts; 2288 2289 if (!object) 2290 return; 2291 2292 if (s->flags & (SLAB_NO_OBJ_EXT | SLAB_NOLEAKTRACE)) 2293 return; 2294 2295 if (flags & __GFP_NO_OBJ_EXT) 2296 return; 2297 2298 obj_exts = prepare_slab_obj_exts_hook(s, flags, object); 2299 /* 2300 * Currently obj_exts is used only for allocation profiling. 2301 * If other users appear then mem_alloc_profiling_enabled() 2302 * check should be added before alloc_tag_add(). 2303 */ 2304 if (likely(obj_exts)) 2305 alloc_tag_add(&obj_exts->ref, current->alloc_tag, s->size); 2306 else 2307 alloc_tag_set_inaccurate(current->alloc_tag); 2308 } 2309 2310 static inline void 2311 alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags) 2312 { 2313 if (mem_alloc_profiling_enabled()) 2314 __alloc_tagging_slab_alloc_hook(s, object, flags); 2315 } 2316 2317 /* Should be called only if mem_alloc_profiling_enabled() */ 2318 static noinline void 2319 __alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, 2320 int objects) 2321 { 2322 struct slabobj_ext *obj_exts; 2323 int i; 2324 2325 /* slab->obj_exts might not be NULL if it was created for MEMCG accounting. */ 2326 if (s->flags & (SLAB_NO_OBJ_EXT | SLAB_NOLEAKTRACE)) 2327 return; 2328 2329 obj_exts = slab_obj_exts(slab); 2330 if (!obj_exts) 2331 return; 2332 2333 for (i = 0; i < objects; i++) { 2334 unsigned int off = obj_to_index(s, slab, p[i]); 2335 2336 alloc_tag_sub(&obj_exts[off].ref, s->size); 2337 } 2338 } 2339 2340 static inline void 2341 alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, 2342 int objects) 2343 { 2344 if (mem_alloc_profiling_enabled()) 2345 __alloc_tagging_slab_free_hook(s, slab, p, objects); 2346 } 2347 2348 #else /* CONFIG_MEM_ALLOC_PROFILING */ 2349 2350 static inline void 2351 alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags) 2352 { 2353 } 2354 2355 static inline void 2356 alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, 2357 int objects) 2358 { 2359 } 2360 2361 #endif /* CONFIG_MEM_ALLOC_PROFILING */ 2362 2363 2364 #ifdef CONFIG_MEMCG 2365 2366 static void memcg_alloc_abort_single(struct kmem_cache *s, void *object); 2367 2368 static __fastpath_inline 2369 bool memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, 2370 gfp_t flags, size_t size, void **p) 2371 { 2372 if (likely(!memcg_kmem_online())) 2373 return true; 2374 2375 if (likely(!(flags & __GFP_ACCOUNT) && !(s->flags & SLAB_ACCOUNT))) 2376 return true; 2377 2378 if (likely(__memcg_slab_post_alloc_hook(s, lru, flags, size, p))) 2379 return true; 2380 2381 if (likely(size == 1)) { 2382 memcg_alloc_abort_single(s, *p); 2383 *p = NULL; 2384 } else { 2385 kmem_cache_free_bulk(s, size, p); 2386 } 2387 2388 return false; 2389 } 2390 2391 static __fastpath_inline 2392 void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, 2393 int objects) 2394 { 2395 struct slabobj_ext *obj_exts; 2396 2397 if (!memcg_kmem_online()) 2398 return; 2399 2400 obj_exts = slab_obj_exts(slab); 2401 if (likely(!obj_exts)) 2402 return; 2403 2404 __memcg_slab_free_hook(s, slab, p, objects, obj_exts); 2405 } 2406 2407 static __fastpath_inline 2408 bool memcg_slab_post_charge(void *p, gfp_t flags) 2409 { 2410 struct slabobj_ext *slab_exts; 2411 struct kmem_cache *s; 2412 struct folio *folio; 2413 struct slab *slab; 2414 unsigned long off; 2415 2416 folio = virt_to_folio(p); 2417 if (!folio_test_slab(folio)) { 2418 int size; 2419 2420 if (folio_memcg_kmem(folio)) 2421 return true; 2422 2423 if (__memcg_kmem_charge_page(folio_page(folio, 0), flags, 2424 folio_order(folio))) 2425 return false; 2426 2427 /* 2428 * This folio has already been accounted in the global stats but 2429 * not in the memcg stats. So, subtract from the global and use 2430 * the interface which adds to both global and memcg stats. 2431 */ 2432 size = folio_size(folio); 2433 node_stat_mod_folio(folio, NR_SLAB_UNRECLAIMABLE_B, -size); 2434 lruvec_stat_mod_folio(folio, NR_SLAB_UNRECLAIMABLE_B, size); 2435 return true; 2436 } 2437 2438 slab = folio_slab(folio); 2439 s = slab->slab_cache; 2440 2441 /* 2442 * Ignore KMALLOC_NORMAL cache to avoid possible circular dependency 2443 * of slab_obj_exts being allocated from the same slab and thus the slab 2444 * becoming effectively unfreeable. 2445 */ 2446 if (is_kmalloc_normal(s)) 2447 return true; 2448 2449 /* Ignore already charged objects. */ 2450 slab_exts = slab_obj_exts(slab); 2451 if (slab_exts) { 2452 off = obj_to_index(s, slab, p); 2453 if (unlikely(slab_exts[off].objcg)) 2454 return true; 2455 } 2456 2457 return __memcg_slab_post_alloc_hook(s, NULL, flags, 1, &p); 2458 } 2459 2460 #else /* CONFIG_MEMCG */ 2461 static inline bool memcg_slab_post_alloc_hook(struct kmem_cache *s, 2462 struct list_lru *lru, 2463 gfp_t flags, size_t size, 2464 void **p) 2465 { 2466 return true; 2467 } 2468 2469 static inline void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, 2470 void **p, int objects) 2471 { 2472 } 2473 2474 static inline bool memcg_slab_post_charge(void *p, gfp_t flags) 2475 { 2476 return true; 2477 } 2478 #endif /* CONFIG_MEMCG */ 2479 2480 #ifdef CONFIG_SLUB_RCU_DEBUG 2481 static void slab_free_after_rcu_debug(struct rcu_head *rcu_head); 2482 2483 struct rcu_delayed_free { 2484 struct rcu_head head; 2485 void *object; 2486 }; 2487 #endif 2488 2489 /* 2490 * Hooks for other subsystems that check memory allocations. In a typical 2491 * production configuration these hooks all should produce no code at all. 2492 * 2493 * Returns true if freeing of the object can proceed, false if its reuse 2494 * was delayed by CONFIG_SLUB_RCU_DEBUG or KASAN quarantine, or it was returned 2495 * to KFENCE. 2496 */ 2497 static __always_inline 2498 bool slab_free_hook(struct kmem_cache *s, void *x, bool init, 2499 bool after_rcu_delay) 2500 { 2501 /* Are the object contents still accessible? */ 2502 bool still_accessible = (s->flags & SLAB_TYPESAFE_BY_RCU) && !after_rcu_delay; 2503 2504 kmemleak_free_recursive(x, s->flags); 2505 kmsan_slab_free(s, x); 2506 2507 debug_check_no_locks_freed(x, s->object_size); 2508 2509 if (!(s->flags & SLAB_DEBUG_OBJECTS)) 2510 debug_check_no_obj_freed(x, s->object_size); 2511 2512 /* Use KCSAN to help debug racy use-after-free. */ 2513 if (!still_accessible) 2514 __kcsan_check_access(x, s->object_size, 2515 KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ASSERT); 2516 2517 if (kfence_free(x)) 2518 return false; 2519 2520 /* 2521 * Give KASAN a chance to notice an invalid free operation before we 2522 * modify the object. 2523 */ 2524 if (kasan_slab_pre_free(s, x)) 2525 return false; 2526 2527 #ifdef CONFIG_SLUB_RCU_DEBUG 2528 if (still_accessible) { 2529 struct rcu_delayed_free *delayed_free; 2530 2531 delayed_free = kmalloc(sizeof(*delayed_free), GFP_NOWAIT); 2532 if (delayed_free) { 2533 /* 2534 * Let KASAN track our call stack as a "related work 2535 * creation", just like if the object had been freed 2536 * normally via kfree_rcu(). 2537 * We have to do this manually because the rcu_head is 2538 * not located inside the object. 2539 */ 2540 kasan_record_aux_stack(x); 2541 2542 delayed_free->object = x; 2543 call_rcu(&delayed_free->head, slab_free_after_rcu_debug); 2544 return false; 2545 } 2546 } 2547 #endif /* CONFIG_SLUB_RCU_DEBUG */ 2548 2549 /* 2550 * As memory initialization might be integrated into KASAN, 2551 * kasan_slab_free and initialization memset's must be 2552 * kept together to avoid discrepancies in behavior. 2553 * 2554 * The initialization memset's clear the object and the metadata, 2555 * but don't touch the SLAB redzone. 2556 * 2557 * The object's freepointer is also avoided if stored outside the 2558 * object. 2559 */ 2560 if (unlikely(init)) { 2561 int rsize; 2562 unsigned int inuse, orig_size; 2563 2564 inuse = get_info_end(s); 2565 orig_size = get_orig_size(s, x); 2566 if (!kasan_has_integrated_init()) 2567 memset(kasan_reset_tag(x), 0, orig_size); 2568 rsize = (s->flags & SLAB_RED_ZONE) ? s->red_left_pad : 0; 2569 memset((char *)kasan_reset_tag(x) + inuse, 0, 2570 s->size - inuse - rsize); 2571 /* 2572 * Restore orig_size, otherwise kmalloc redzone overwritten 2573 * would be reported 2574 */ 2575 set_orig_size(s, x, orig_size); 2576 2577 } 2578 /* KASAN might put x into memory quarantine, delaying its reuse. */ 2579 return !kasan_slab_free(s, x, init, still_accessible, false); 2580 } 2581 2582 static __fastpath_inline 2583 bool slab_free_freelist_hook(struct kmem_cache *s, void **head, void **tail, 2584 int *cnt) 2585 { 2586 2587 void *object; 2588 void *next = *head; 2589 void *old_tail = *tail; 2590 bool init; 2591 2592 if (is_kfence_address(next)) { 2593 slab_free_hook(s, next, false, false); 2594 return false; 2595 } 2596 2597 /* Head and tail of the reconstructed freelist */ 2598 *head = NULL; 2599 *tail = NULL; 2600 2601 init = slab_want_init_on_free(s); 2602 2603 do { 2604 object = next; 2605 next = get_freepointer(s, object); 2606 2607 /* If object's reuse doesn't have to be delayed */ 2608 if (likely(slab_free_hook(s, object, init, false))) { 2609 /* Move object to the new freelist */ 2610 set_freepointer(s, object, *head); 2611 *head = object; 2612 if (!*tail) 2613 *tail = object; 2614 } else { 2615 /* 2616 * Adjust the reconstructed freelist depth 2617 * accordingly if object's reuse is delayed. 2618 */ 2619 --(*cnt); 2620 } 2621 } while (object != old_tail); 2622 2623 return *head != NULL; 2624 } 2625 2626 static void *setup_object(struct kmem_cache *s, void *object) 2627 { 2628 setup_object_debug(s, object); 2629 object = kasan_init_slab_obj(s, object); 2630 if (unlikely(s->ctor)) { 2631 kasan_unpoison_new_object(s, object); 2632 s->ctor(object); 2633 kasan_poison_new_object(s, object); 2634 } 2635 return object; 2636 } 2637 2638 static struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp) 2639 { 2640 struct slab_sheaf *sheaf = kzalloc(struct_size(sheaf, objects, 2641 s->sheaf_capacity), gfp); 2642 2643 if (unlikely(!sheaf)) 2644 return NULL; 2645 2646 sheaf->cache = s; 2647 2648 stat(s, SHEAF_ALLOC); 2649 2650 return sheaf; 2651 } 2652 2653 static void free_empty_sheaf(struct kmem_cache *s, struct slab_sheaf *sheaf) 2654 { 2655 kfree(sheaf); 2656 2657 stat(s, SHEAF_FREE); 2658 } 2659 2660 static int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, 2661 size_t size, void **p); 2662 2663 2664 static int refill_sheaf(struct kmem_cache *s, struct slab_sheaf *sheaf, 2665 gfp_t gfp) 2666 { 2667 int to_fill = s->sheaf_capacity - sheaf->size; 2668 int filled; 2669 2670 if (!to_fill) 2671 return 0; 2672 2673 filled = __kmem_cache_alloc_bulk(s, gfp, to_fill, 2674 &sheaf->objects[sheaf->size]); 2675 2676 sheaf->size += filled; 2677 2678 stat_add(s, SHEAF_REFILL, filled); 2679 2680 if (filled < to_fill) 2681 return -ENOMEM; 2682 2683 return 0; 2684 } 2685 2686 2687 static struct slab_sheaf *alloc_full_sheaf(struct kmem_cache *s, gfp_t gfp) 2688 { 2689 struct slab_sheaf *sheaf = alloc_empty_sheaf(s, gfp); 2690 2691 if (!sheaf) 2692 return NULL; 2693 2694 if (refill_sheaf(s, sheaf, gfp)) { 2695 free_empty_sheaf(s, sheaf); 2696 return NULL; 2697 } 2698 2699 return sheaf; 2700 } 2701 2702 /* 2703 * Maximum number of objects freed during a single flush of main pcs sheaf. 2704 * Translates directly to an on-stack array size. 2705 */ 2706 #define PCS_BATCH_MAX 32U 2707 2708 static void __kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p); 2709 2710 /* 2711 * Free all objects from the main sheaf. In order to perform 2712 * __kmem_cache_free_bulk() outside of cpu_sheaves->lock, work in batches where 2713 * object pointers are moved to a on-stack array under the lock. To bound the 2714 * stack usage, limit each batch to PCS_BATCH_MAX. 2715 * 2716 * Must be called with s->cpu_sheaves->lock locked, returns with the lock 2717 * unlocked. 2718 * 2719 * Returns how many objects are remaining to be flushed 2720 */ 2721 static unsigned int __sheaf_flush_main_batch(struct kmem_cache *s) 2722 { 2723 struct slub_percpu_sheaves *pcs; 2724 unsigned int batch, remaining; 2725 void *objects[PCS_BATCH_MAX]; 2726 struct slab_sheaf *sheaf; 2727 2728 lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); 2729 2730 pcs = this_cpu_ptr(s->cpu_sheaves); 2731 sheaf = pcs->main; 2732 2733 batch = min(PCS_BATCH_MAX, sheaf->size); 2734 2735 sheaf->size -= batch; 2736 memcpy(objects, sheaf->objects + sheaf->size, batch * sizeof(void *)); 2737 2738 remaining = sheaf->size; 2739 2740 local_unlock(&s->cpu_sheaves->lock); 2741 2742 __kmem_cache_free_bulk(s, batch, &objects[0]); 2743 2744 stat_add(s, SHEAF_FLUSH, batch); 2745 2746 return remaining; 2747 } 2748 2749 static void sheaf_flush_main(struct kmem_cache *s) 2750 { 2751 unsigned int remaining; 2752 2753 do { 2754 local_lock(&s->cpu_sheaves->lock); 2755 2756 remaining = __sheaf_flush_main_batch(s); 2757 2758 } while (remaining); 2759 } 2760 2761 /* 2762 * Returns true if the main sheaf was at least partially flushed. 2763 */ 2764 static bool sheaf_try_flush_main(struct kmem_cache *s) 2765 { 2766 unsigned int remaining; 2767 bool ret = false; 2768 2769 do { 2770 if (!local_trylock(&s->cpu_sheaves->lock)) 2771 return ret; 2772 2773 ret = true; 2774 remaining = __sheaf_flush_main_batch(s); 2775 2776 } while (remaining); 2777 2778 return ret; 2779 } 2780 2781 /* 2782 * Free all objects from a sheaf that's unused, i.e. not linked to any 2783 * cpu_sheaves, so we need no locking and batching. The locking is also not 2784 * necessary when flushing cpu's sheaves (both spare and main) during cpu 2785 * hotremove as the cpu is not executing anymore. 2786 */ 2787 static void sheaf_flush_unused(struct kmem_cache *s, struct slab_sheaf *sheaf) 2788 { 2789 if (!sheaf->size) 2790 return; 2791 2792 stat_add(s, SHEAF_FLUSH, sheaf->size); 2793 2794 __kmem_cache_free_bulk(s, sheaf->size, &sheaf->objects[0]); 2795 2796 sheaf->size = 0; 2797 } 2798 2799 static void __rcu_free_sheaf_prepare(struct kmem_cache *s, 2800 struct slab_sheaf *sheaf) 2801 { 2802 bool init = slab_want_init_on_free(s); 2803 void **p = &sheaf->objects[0]; 2804 unsigned int i = 0; 2805 2806 while (i < sheaf->size) { 2807 struct slab *slab = virt_to_slab(p[i]); 2808 2809 memcg_slab_free_hook(s, slab, p + i, 1); 2810 alloc_tagging_slab_free_hook(s, slab, p + i, 1); 2811 2812 if (unlikely(!slab_free_hook(s, p[i], init, true))) { 2813 p[i] = p[--sheaf->size]; 2814 continue; 2815 } 2816 2817 i++; 2818 } 2819 } 2820 2821 static void rcu_free_sheaf_nobarn(struct rcu_head *head) 2822 { 2823 struct slab_sheaf *sheaf; 2824 struct kmem_cache *s; 2825 2826 sheaf = container_of(head, struct slab_sheaf, rcu_head); 2827 s = sheaf->cache; 2828 2829 __rcu_free_sheaf_prepare(s, sheaf); 2830 2831 sheaf_flush_unused(s, sheaf); 2832 2833 free_empty_sheaf(s, sheaf); 2834 } 2835 2836 /* 2837 * Caller needs to make sure migration is disabled in order to fully flush 2838 * single cpu's sheaves 2839 * 2840 * must not be called from an irq 2841 * 2842 * flushing operations are rare so let's keep it simple and flush to slabs 2843 * directly, skipping the barn 2844 */ 2845 static void pcs_flush_all(struct kmem_cache *s) 2846 { 2847 struct slub_percpu_sheaves *pcs; 2848 struct slab_sheaf *spare, *rcu_free; 2849 2850 local_lock(&s->cpu_sheaves->lock); 2851 pcs = this_cpu_ptr(s->cpu_sheaves); 2852 2853 spare = pcs->spare; 2854 pcs->spare = NULL; 2855 2856 rcu_free = pcs->rcu_free; 2857 pcs->rcu_free = NULL; 2858 2859 local_unlock(&s->cpu_sheaves->lock); 2860 2861 if (spare) { 2862 sheaf_flush_unused(s, spare); 2863 free_empty_sheaf(s, spare); 2864 } 2865 2866 if (rcu_free) 2867 call_rcu(&rcu_free->rcu_head, rcu_free_sheaf_nobarn); 2868 2869 sheaf_flush_main(s); 2870 } 2871 2872 static void __pcs_flush_all_cpu(struct kmem_cache *s, unsigned int cpu) 2873 { 2874 struct slub_percpu_sheaves *pcs; 2875 2876 pcs = per_cpu_ptr(s->cpu_sheaves, cpu); 2877 2878 /* The cpu is not executing anymore so we don't need pcs->lock */ 2879 sheaf_flush_unused(s, pcs->main); 2880 if (pcs->spare) { 2881 sheaf_flush_unused(s, pcs->spare); 2882 free_empty_sheaf(s, pcs->spare); 2883 pcs->spare = NULL; 2884 } 2885 2886 if (pcs->rcu_free) { 2887 call_rcu(&pcs->rcu_free->rcu_head, rcu_free_sheaf_nobarn); 2888 pcs->rcu_free = NULL; 2889 } 2890 } 2891 2892 static void pcs_destroy(struct kmem_cache *s) 2893 { 2894 int cpu; 2895 2896 for_each_possible_cpu(cpu) { 2897 struct slub_percpu_sheaves *pcs; 2898 2899 pcs = per_cpu_ptr(s->cpu_sheaves, cpu); 2900 2901 /* can happen when unwinding failed create */ 2902 if (!pcs->main) 2903 continue; 2904 2905 /* 2906 * We have already passed __kmem_cache_shutdown() so everything 2907 * was flushed and there should be no objects allocated from 2908 * slabs, otherwise kmem_cache_destroy() would have aborted. 2909 * Therefore something would have to be really wrong if the 2910 * warnings here trigger, and we should rather leave objects and 2911 * sheaves to leak in that case. 2912 */ 2913 2914 WARN_ON(pcs->spare); 2915 WARN_ON(pcs->rcu_free); 2916 2917 if (!WARN_ON(pcs->main->size)) { 2918 free_empty_sheaf(s, pcs->main); 2919 pcs->main = NULL; 2920 } 2921 } 2922 2923 free_percpu(s->cpu_sheaves); 2924 s->cpu_sheaves = NULL; 2925 } 2926 2927 static struct slab_sheaf *barn_get_empty_sheaf(struct node_barn *barn) 2928 { 2929 struct slab_sheaf *empty = NULL; 2930 unsigned long flags; 2931 2932 if (!data_race(barn->nr_empty)) 2933 return NULL; 2934 2935 spin_lock_irqsave(&barn->lock, flags); 2936 2937 if (likely(barn->nr_empty)) { 2938 empty = list_first_entry(&barn->sheaves_empty, 2939 struct slab_sheaf, barn_list); 2940 list_del(&empty->barn_list); 2941 barn->nr_empty--; 2942 } 2943 2944 spin_unlock_irqrestore(&barn->lock, flags); 2945 2946 return empty; 2947 } 2948 2949 /* 2950 * The following two functions are used mainly in cases where we have to undo an 2951 * intended action due to a race or cpu migration. Thus they do not check the 2952 * empty or full sheaf limits for simplicity. 2953 */ 2954 2955 static void barn_put_empty_sheaf(struct node_barn *barn, struct slab_sheaf *sheaf) 2956 { 2957 unsigned long flags; 2958 2959 spin_lock_irqsave(&barn->lock, flags); 2960 2961 list_add(&sheaf->barn_list, &barn->sheaves_empty); 2962 barn->nr_empty++; 2963 2964 spin_unlock_irqrestore(&barn->lock, flags); 2965 } 2966 2967 static void barn_put_full_sheaf(struct node_barn *barn, struct slab_sheaf *sheaf) 2968 { 2969 unsigned long flags; 2970 2971 spin_lock_irqsave(&barn->lock, flags); 2972 2973 list_add(&sheaf->barn_list, &barn->sheaves_full); 2974 barn->nr_full++; 2975 2976 spin_unlock_irqrestore(&barn->lock, flags); 2977 } 2978 2979 static struct slab_sheaf *barn_get_full_or_empty_sheaf(struct node_barn *barn) 2980 { 2981 struct slab_sheaf *sheaf = NULL; 2982 unsigned long flags; 2983 2984 if (!data_race(barn->nr_full) && !data_race(barn->nr_empty)) 2985 return NULL; 2986 2987 spin_lock_irqsave(&barn->lock, flags); 2988 2989 if (barn->nr_full) { 2990 sheaf = list_first_entry(&barn->sheaves_full, struct slab_sheaf, 2991 barn_list); 2992 list_del(&sheaf->barn_list); 2993 barn->nr_full--; 2994 } else if (barn->nr_empty) { 2995 sheaf = list_first_entry(&barn->sheaves_empty, 2996 struct slab_sheaf, barn_list); 2997 list_del(&sheaf->barn_list); 2998 barn->nr_empty--; 2999 } 3000 3001 spin_unlock_irqrestore(&barn->lock, flags); 3002 3003 return sheaf; 3004 } 3005 3006 /* 3007 * If a full sheaf is available, return it and put the supplied empty one to 3008 * barn. We ignore the limit on empty sheaves as the number of sheaves doesn't 3009 * change. 3010 */ 3011 static struct slab_sheaf * 3012 barn_replace_empty_sheaf(struct node_barn *barn, struct slab_sheaf *empty) 3013 { 3014 struct slab_sheaf *full = NULL; 3015 unsigned long flags; 3016 3017 if (!data_race(barn->nr_full)) 3018 return NULL; 3019 3020 spin_lock_irqsave(&barn->lock, flags); 3021 3022 if (likely(barn->nr_full)) { 3023 full = list_first_entry(&barn->sheaves_full, struct slab_sheaf, 3024 barn_list); 3025 list_del(&full->barn_list); 3026 list_add(&empty->barn_list, &barn->sheaves_empty); 3027 barn->nr_full--; 3028 barn->nr_empty++; 3029 } 3030 3031 spin_unlock_irqrestore(&barn->lock, flags); 3032 3033 return full; 3034 } 3035 3036 /* 3037 * If an empty sheaf is available, return it and put the supplied full one to 3038 * barn. But if there are too many full sheaves, reject this with -E2BIG. 3039 */ 3040 static struct slab_sheaf * 3041 barn_replace_full_sheaf(struct node_barn *barn, struct slab_sheaf *full) 3042 { 3043 struct slab_sheaf *empty; 3044 unsigned long flags; 3045 3046 /* we don't repeat this check under barn->lock as it's not critical */ 3047 if (data_race(barn->nr_full) >= MAX_FULL_SHEAVES) 3048 return ERR_PTR(-E2BIG); 3049 if (!data_race(barn->nr_empty)) 3050 return ERR_PTR(-ENOMEM); 3051 3052 spin_lock_irqsave(&barn->lock, flags); 3053 3054 if (likely(barn->nr_empty)) { 3055 empty = list_first_entry(&barn->sheaves_empty, struct slab_sheaf, 3056 barn_list); 3057 list_del(&empty->barn_list); 3058 list_add(&full->barn_list, &barn->sheaves_full); 3059 barn->nr_empty--; 3060 barn->nr_full++; 3061 } else { 3062 empty = ERR_PTR(-ENOMEM); 3063 } 3064 3065 spin_unlock_irqrestore(&barn->lock, flags); 3066 3067 return empty; 3068 } 3069 3070 static void barn_init(struct node_barn *barn) 3071 { 3072 spin_lock_init(&barn->lock); 3073 INIT_LIST_HEAD(&barn->sheaves_full); 3074 INIT_LIST_HEAD(&barn->sheaves_empty); 3075 barn->nr_full = 0; 3076 barn->nr_empty = 0; 3077 } 3078 3079 static void barn_shrink(struct kmem_cache *s, struct node_barn *barn) 3080 { 3081 struct list_head empty_list; 3082 struct list_head full_list; 3083 struct slab_sheaf *sheaf, *sheaf2; 3084 unsigned long flags; 3085 3086 INIT_LIST_HEAD(&empty_list); 3087 INIT_LIST_HEAD(&full_list); 3088 3089 spin_lock_irqsave(&barn->lock, flags); 3090 3091 list_splice_init(&barn->sheaves_full, &full_list); 3092 barn->nr_full = 0; 3093 list_splice_init(&barn->sheaves_empty, &empty_list); 3094 barn->nr_empty = 0; 3095 3096 spin_unlock_irqrestore(&barn->lock, flags); 3097 3098 list_for_each_entry_safe(sheaf, sheaf2, &full_list, barn_list) { 3099 sheaf_flush_unused(s, sheaf); 3100 free_empty_sheaf(s, sheaf); 3101 } 3102 3103 list_for_each_entry_safe(sheaf, sheaf2, &empty_list, barn_list) 3104 free_empty_sheaf(s, sheaf); 3105 } 3106 3107 /* 3108 * Slab allocation and freeing 3109 */ 3110 static inline struct slab *alloc_slab_page(gfp_t flags, int node, 3111 struct kmem_cache_order_objects oo, 3112 bool allow_spin) 3113 { 3114 struct folio *folio; 3115 struct slab *slab; 3116 unsigned int order = oo_order(oo); 3117 3118 if (unlikely(!allow_spin)) 3119 folio = (struct folio *)alloc_frozen_pages_nolock(0/* __GFP_COMP is implied */, 3120 node, order); 3121 else if (node == NUMA_NO_NODE) 3122 folio = (struct folio *)alloc_frozen_pages(flags, order); 3123 else 3124 folio = (struct folio *)__alloc_frozen_pages(flags, order, node, NULL); 3125 3126 if (!folio) 3127 return NULL; 3128 3129 slab = folio_slab(folio); 3130 __folio_set_slab(folio); 3131 if (folio_is_pfmemalloc(folio)) 3132 slab_set_pfmemalloc(slab); 3133 3134 return slab; 3135 } 3136 3137 #ifdef CONFIG_SLAB_FREELIST_RANDOM 3138 /* Pre-initialize the random sequence cache */ 3139 static int init_cache_random_seq(struct kmem_cache *s) 3140 { 3141 unsigned int count = oo_objects(s->oo); 3142 int err; 3143 3144 /* Bailout if already initialised */ 3145 if (s->random_seq) 3146 return 0; 3147 3148 err = cache_random_seq_create(s, count, GFP_KERNEL); 3149 if (err) { 3150 pr_err("SLUB: Unable to initialize free list for %s\n", 3151 s->name); 3152 return err; 3153 } 3154 3155 /* Transform to an offset on the set of pages */ 3156 if (s->random_seq) { 3157 unsigned int i; 3158 3159 for (i = 0; i < count; i++) 3160 s->random_seq[i] *= s->size; 3161 } 3162 return 0; 3163 } 3164 3165 /* Initialize each random sequence freelist per cache */ 3166 static void __init init_freelist_randomization(void) 3167 { 3168 struct kmem_cache *s; 3169 3170 mutex_lock(&slab_mutex); 3171 3172 list_for_each_entry(s, &slab_caches, list) 3173 init_cache_random_seq(s); 3174 3175 mutex_unlock(&slab_mutex); 3176 } 3177 3178 /* Get the next entry on the pre-computed freelist randomized */ 3179 static void *next_freelist_entry(struct kmem_cache *s, 3180 unsigned long *pos, void *start, 3181 unsigned long page_limit, 3182 unsigned long freelist_count) 3183 { 3184 unsigned int idx; 3185 3186 /* 3187 * If the target page allocation failed, the number of objects on the 3188 * page might be smaller than the usual size defined by the cache. 3189 */ 3190 do { 3191 idx = s->random_seq[*pos]; 3192 *pos += 1; 3193 if (*pos >= freelist_count) 3194 *pos = 0; 3195 } while (unlikely(idx >= page_limit)); 3196 3197 return (char *)start + idx; 3198 } 3199 3200 static DEFINE_PER_CPU(struct rnd_state, slab_rnd_state); 3201 3202 /* Shuffle the single linked freelist based on a random pre-computed sequence */ 3203 static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab, 3204 bool allow_spin) 3205 { 3206 void *start; 3207 void *cur; 3208 void *next; 3209 unsigned long idx, pos, page_limit, freelist_count; 3210 3211 if (slab->objects < 2 || !s->random_seq) 3212 return false; 3213 3214 freelist_count = oo_objects(s->oo); 3215 if (allow_spin) { 3216 pos = get_random_u32_below(freelist_count); 3217 } else { 3218 struct rnd_state *state; 3219 3220 /* 3221 * An interrupt or NMI handler might interrupt and change 3222 * the state in the middle, but that's safe. 3223 */ 3224 state = &get_cpu_var(slab_rnd_state); 3225 pos = prandom_u32_state(state) % freelist_count; 3226 put_cpu_var(slab_rnd_state); 3227 } 3228 3229 page_limit = slab->objects * s->size; 3230 start = fixup_red_left(s, slab_address(slab)); 3231 3232 /* First entry is used as the base of the freelist */ 3233 cur = next_freelist_entry(s, &pos, start, page_limit, freelist_count); 3234 cur = setup_object(s, cur); 3235 slab->freelist = cur; 3236 3237 for (idx = 1; idx < slab->objects; idx++) { 3238 next = next_freelist_entry(s, &pos, start, page_limit, 3239 freelist_count); 3240 next = setup_object(s, next); 3241 set_freepointer(s, cur, next); 3242 cur = next; 3243 } 3244 set_freepointer(s, cur, NULL); 3245 3246 return true; 3247 } 3248 #else 3249 static inline int init_cache_random_seq(struct kmem_cache *s) 3250 { 3251 return 0; 3252 } 3253 static inline void init_freelist_randomization(void) { } 3254 static inline bool shuffle_freelist(struct kmem_cache *s, struct slab *slab, 3255 bool allow_spin) 3256 { 3257 return false; 3258 } 3259 #endif /* CONFIG_SLAB_FREELIST_RANDOM */ 3260 3261 static __always_inline void account_slab(struct slab *slab, int order, 3262 struct kmem_cache *s, gfp_t gfp) 3263 { 3264 if (memcg_kmem_online() && (s->flags & SLAB_ACCOUNT)) 3265 alloc_slab_obj_exts(slab, s, gfp, true); 3266 3267 mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s), 3268 PAGE_SIZE << order); 3269 } 3270 3271 static __always_inline void unaccount_slab(struct slab *slab, int order, 3272 struct kmem_cache *s) 3273 { 3274 /* 3275 * The slab object extensions should now be freed regardless of 3276 * whether mem_alloc_profiling_enabled() or not because profiling 3277 * might have been disabled after slab->obj_exts got allocated. 3278 */ 3279 free_slab_obj_exts(slab); 3280 3281 mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s), 3282 -(PAGE_SIZE << order)); 3283 } 3284 3285 static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) 3286 { 3287 bool allow_spin = gfpflags_allow_spinning(flags); 3288 struct slab *slab; 3289 struct kmem_cache_order_objects oo = s->oo; 3290 gfp_t alloc_gfp; 3291 void *start, *p, *next; 3292 int idx; 3293 bool shuffle; 3294 3295 flags &= gfp_allowed_mask; 3296 3297 flags |= s->allocflags; 3298 3299 /* 3300 * Let the initial higher-order allocation fail under memory pressure 3301 * so we fall-back to the minimum order allocation. 3302 */ 3303 alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL; 3304 if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min)) 3305 alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~__GFP_RECLAIM; 3306 3307 /* 3308 * __GFP_RECLAIM could be cleared on the first allocation attempt, 3309 * so pass allow_spin flag directly. 3310 */ 3311 slab = alloc_slab_page(alloc_gfp, node, oo, allow_spin); 3312 if (unlikely(!slab)) { 3313 oo = s->min; 3314 alloc_gfp = flags; 3315 /* 3316 * Allocation may have failed due to fragmentation. 3317 * Try a lower order alloc if possible 3318 */ 3319 slab = alloc_slab_page(alloc_gfp, node, oo, allow_spin); 3320 if (unlikely(!slab)) 3321 return NULL; 3322 stat(s, ORDER_FALLBACK); 3323 } 3324 3325 slab->objects = oo_objects(oo); 3326 slab->inuse = 0; 3327 slab->frozen = 0; 3328 init_slab_obj_exts(slab); 3329 3330 account_slab(slab, oo_order(oo), s, flags); 3331 3332 slab->slab_cache = s; 3333 3334 kasan_poison_slab(slab); 3335 3336 start = slab_address(slab); 3337 3338 setup_slab_debug(s, slab, start); 3339 3340 shuffle = shuffle_freelist(s, slab, allow_spin); 3341 3342 if (!shuffle) { 3343 start = fixup_red_left(s, start); 3344 start = setup_object(s, start); 3345 slab->freelist = start; 3346 for (idx = 0, p = start; idx < slab->objects - 1; idx++) { 3347 next = p + s->size; 3348 next = setup_object(s, next); 3349 set_freepointer(s, p, next); 3350 p = next; 3351 } 3352 set_freepointer(s, p, NULL); 3353 } 3354 3355 return slab; 3356 } 3357 3358 static struct slab *new_slab(struct kmem_cache *s, gfp_t flags, int node) 3359 { 3360 if (unlikely(flags & GFP_SLAB_BUG_MASK)) 3361 flags = kmalloc_fix_flags(flags); 3362 3363 WARN_ON_ONCE(s->ctor && (flags & __GFP_ZERO)); 3364 3365 return allocate_slab(s, 3366 flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node); 3367 } 3368 3369 static void __free_slab(struct kmem_cache *s, struct slab *slab) 3370 { 3371 struct folio *folio = slab_folio(slab); 3372 int order = folio_order(folio); 3373 int pages = 1 << order; 3374 3375 __slab_clear_pfmemalloc(slab); 3376 folio->mapping = NULL; 3377 __folio_clear_slab(folio); 3378 mm_account_reclaimed_pages(pages); 3379 unaccount_slab(slab, order, s); 3380 free_frozen_pages(&folio->page, order); 3381 } 3382 3383 static void rcu_free_slab(struct rcu_head *h) 3384 { 3385 struct slab *slab = container_of(h, struct slab, rcu_head); 3386 3387 __free_slab(slab->slab_cache, slab); 3388 } 3389 3390 static void free_slab(struct kmem_cache *s, struct slab *slab) 3391 { 3392 if (kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS)) { 3393 void *p; 3394 3395 slab_pad_check(s, slab); 3396 for_each_object(p, s, slab_address(slab), slab->objects) 3397 check_object(s, slab, p, SLUB_RED_INACTIVE); 3398 } 3399 3400 if (unlikely(s->flags & SLAB_TYPESAFE_BY_RCU)) 3401 call_rcu(&slab->rcu_head, rcu_free_slab); 3402 else 3403 __free_slab(s, slab); 3404 } 3405 3406 static void discard_slab(struct kmem_cache *s, struct slab *slab) 3407 { 3408 dec_slabs_node(s, slab_nid(slab), slab->objects); 3409 free_slab(s, slab); 3410 } 3411 3412 static inline bool slab_test_node_partial(const struct slab *slab) 3413 { 3414 return test_bit(SL_partial, &slab->flags.f); 3415 } 3416 3417 static inline void slab_set_node_partial(struct slab *slab) 3418 { 3419 set_bit(SL_partial, &slab->flags.f); 3420 } 3421 3422 static inline void slab_clear_node_partial(struct slab *slab) 3423 { 3424 clear_bit(SL_partial, &slab->flags.f); 3425 } 3426 3427 /* 3428 * Management of partially allocated slabs. 3429 */ 3430 static inline void 3431 __add_partial(struct kmem_cache_node *n, struct slab *slab, int tail) 3432 { 3433 n->nr_partial++; 3434 if (tail == DEACTIVATE_TO_TAIL) 3435 list_add_tail(&slab->slab_list, &n->partial); 3436 else 3437 list_add(&slab->slab_list, &n->partial); 3438 slab_set_node_partial(slab); 3439 } 3440 3441 static inline void add_partial(struct kmem_cache_node *n, 3442 struct slab *slab, int tail) 3443 { 3444 lockdep_assert_held(&n->list_lock); 3445 __add_partial(n, slab, tail); 3446 } 3447 3448 static inline void remove_partial(struct kmem_cache_node *n, 3449 struct slab *slab) 3450 { 3451 lockdep_assert_held(&n->list_lock); 3452 list_del(&slab->slab_list); 3453 slab_clear_node_partial(slab); 3454 n->nr_partial--; 3455 } 3456 3457 /* 3458 * Called only for kmem_cache_debug() caches instead of remove_partial(), with a 3459 * slab from the n->partial list. Remove only a single object from the slab, do 3460 * the alloc_debug_processing() checks and leave the slab on the list, or move 3461 * it to full list if it was the last free object. 3462 */ 3463 static void *alloc_single_from_partial(struct kmem_cache *s, 3464 struct kmem_cache_node *n, struct slab *slab, int orig_size) 3465 { 3466 void *object; 3467 3468 lockdep_assert_held(&n->list_lock); 3469 3470 #ifdef CONFIG_SLUB_DEBUG 3471 if (s->flags & SLAB_CONSISTENCY_CHECKS) { 3472 if (!validate_slab_ptr(slab)) { 3473 slab_err(s, slab, "Not a valid slab page"); 3474 return NULL; 3475 } 3476 } 3477 #endif 3478 3479 object = slab->freelist; 3480 slab->freelist = get_freepointer(s, object); 3481 slab->inuse++; 3482 3483 if (!alloc_debug_processing(s, slab, object, orig_size)) { 3484 remove_partial(n, slab); 3485 return NULL; 3486 } 3487 3488 if (slab->inuse == slab->objects) { 3489 remove_partial(n, slab); 3490 add_full(s, n, slab); 3491 } 3492 3493 return object; 3494 } 3495 3496 static void defer_deactivate_slab(struct slab *slab, void *flush_freelist); 3497 3498 /* 3499 * Called only for kmem_cache_debug() caches to allocate from a freshly 3500 * allocated slab. Allocate a single object instead of whole freelist 3501 * and put the slab to the partial (or full) list. 3502 */ 3503 static void *alloc_single_from_new_slab(struct kmem_cache *s, struct slab *slab, 3504 int orig_size, gfp_t gfpflags) 3505 { 3506 bool allow_spin = gfpflags_allow_spinning(gfpflags); 3507 int nid = slab_nid(slab); 3508 struct kmem_cache_node *n = get_node(s, nid); 3509 unsigned long flags; 3510 void *object; 3511 3512 if (!allow_spin && !spin_trylock_irqsave(&n->list_lock, flags)) { 3513 /* Unlucky, discard newly allocated slab */ 3514 defer_deactivate_slab(slab, NULL); 3515 return NULL; 3516 } 3517 3518 object = slab->freelist; 3519 slab->freelist = get_freepointer(s, object); 3520 slab->inuse = 1; 3521 3522 if (!alloc_debug_processing(s, slab, object, orig_size)) { 3523 /* 3524 * It's not really expected that this would fail on a 3525 * freshly allocated slab, but a concurrent memory 3526 * corruption in theory could cause that. 3527 * Leak memory of allocated slab. 3528 */ 3529 if (!allow_spin) 3530 spin_unlock_irqrestore(&n->list_lock, flags); 3531 return NULL; 3532 } 3533 3534 if (allow_spin) 3535 spin_lock_irqsave(&n->list_lock, flags); 3536 3537 if (slab->inuse == slab->objects) 3538 add_full(s, n, slab); 3539 else 3540 add_partial(n, slab, DEACTIVATE_TO_HEAD); 3541 3542 inc_slabs_node(s, nid, slab->objects); 3543 spin_unlock_irqrestore(&n->list_lock, flags); 3544 3545 return object; 3546 } 3547 3548 #ifdef CONFIG_SLUB_CPU_PARTIAL 3549 static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain); 3550 #else 3551 static inline void put_cpu_partial(struct kmem_cache *s, struct slab *slab, 3552 int drain) { } 3553 #endif 3554 static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags); 3555 3556 /* 3557 * Try to allocate a partial slab from a specific node. 3558 */ 3559 static struct slab *get_partial_node(struct kmem_cache *s, 3560 struct kmem_cache_node *n, 3561 struct partial_context *pc) 3562 { 3563 struct slab *slab, *slab2, *partial = NULL; 3564 unsigned long flags; 3565 unsigned int partial_slabs = 0; 3566 3567 /* 3568 * Racy check. If we mistakenly see no partial slabs then we 3569 * just allocate an empty slab. If we mistakenly try to get a 3570 * partial slab and there is none available then get_partial() 3571 * will return NULL. 3572 */ 3573 if (!n || !n->nr_partial) 3574 return NULL; 3575 3576 if (gfpflags_allow_spinning(pc->flags)) 3577 spin_lock_irqsave(&n->list_lock, flags); 3578 else if (!spin_trylock_irqsave(&n->list_lock, flags)) 3579 return NULL; 3580 list_for_each_entry_safe(slab, slab2, &n->partial, slab_list) { 3581 if (!pfmemalloc_match(slab, pc->flags)) 3582 continue; 3583 3584 if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) { 3585 void *object = alloc_single_from_partial(s, n, slab, 3586 pc->orig_size); 3587 if (object) { 3588 partial = slab; 3589 pc->object = object; 3590 break; 3591 } 3592 continue; 3593 } 3594 3595 remove_partial(n, slab); 3596 3597 if (!partial) { 3598 partial = slab; 3599 stat(s, ALLOC_FROM_PARTIAL); 3600 3601 if ((slub_get_cpu_partial(s) == 0)) { 3602 break; 3603 } 3604 } else { 3605 put_cpu_partial(s, slab, 0); 3606 stat(s, CPU_PARTIAL_NODE); 3607 3608 if (++partial_slabs > slub_get_cpu_partial(s) / 2) { 3609 break; 3610 } 3611 } 3612 } 3613 spin_unlock_irqrestore(&n->list_lock, flags); 3614 return partial; 3615 } 3616 3617 /* 3618 * Get a slab from somewhere. Search in increasing NUMA distances. 3619 */ 3620 static struct slab *get_any_partial(struct kmem_cache *s, 3621 struct partial_context *pc) 3622 { 3623 #ifdef CONFIG_NUMA 3624 struct zonelist *zonelist; 3625 struct zoneref *z; 3626 struct zone *zone; 3627 enum zone_type highest_zoneidx = gfp_zone(pc->flags); 3628 struct slab *slab; 3629 unsigned int cpuset_mems_cookie; 3630 bool allow_spin = gfpflags_allow_spinning(pc->flags); 3631 3632 /* 3633 * The defrag ratio allows a configuration of the tradeoffs between 3634 * inter node defragmentation and node local allocations. A lower 3635 * defrag_ratio increases the tendency to do local allocations 3636 * instead of attempting to obtain partial slabs from other nodes. 3637 * 3638 * If the defrag_ratio is set to 0 then kmalloc() always 3639 * returns node local objects. If the ratio is higher then kmalloc() 3640 * may return off node objects because partial slabs are obtained 3641 * from other nodes and filled up. 3642 * 3643 * If /sys/kernel/slab/xx/remote_node_defrag_ratio is set to 100 3644 * (which makes defrag_ratio = 1000) then every (well almost) 3645 * allocation will first attempt to defrag slab caches on other nodes. 3646 * This means scanning over all nodes to look for partial slabs which 3647 * may be expensive if we do it every time we are trying to find a slab 3648 * with available objects. 3649 */ 3650 if (!s->remote_node_defrag_ratio || 3651 get_cycles() % 1024 > s->remote_node_defrag_ratio) 3652 return NULL; 3653 3654 do { 3655 /* 3656 * read_mems_allowed_begin() accesses current->mems_allowed_seq, 3657 * a seqcount_spinlock_t that is not NMI-safe. Do not access 3658 * current->mems_allowed_seq and avoid retry when GFP flags 3659 * indicate spinning is not allowed. 3660 */ 3661 if (allow_spin) 3662 cpuset_mems_cookie = read_mems_allowed_begin(); 3663 3664 zonelist = node_zonelist(mempolicy_slab_node(), pc->flags); 3665 for_each_zone_zonelist(zone, z, zonelist, highest_zoneidx) { 3666 struct kmem_cache_node *n; 3667 3668 n = get_node(s, zone_to_nid(zone)); 3669 3670 if (n && cpuset_zone_allowed(zone, pc->flags) && 3671 n->nr_partial > s->min_partial) { 3672 slab = get_partial_node(s, n, pc); 3673 if (slab) { 3674 /* 3675 * Don't check read_mems_allowed_retry() 3676 * here - if mems_allowed was updated in 3677 * parallel, that was a harmless race 3678 * between allocation and the cpuset 3679 * update 3680 */ 3681 return slab; 3682 } 3683 } 3684 } 3685 } while (allow_spin && read_mems_allowed_retry(cpuset_mems_cookie)); 3686 #endif /* CONFIG_NUMA */ 3687 return NULL; 3688 } 3689 3690 /* 3691 * Get a partial slab, lock it and return it. 3692 */ 3693 static struct slab *get_partial(struct kmem_cache *s, int node, 3694 struct partial_context *pc) 3695 { 3696 struct slab *slab; 3697 int searchnode = node; 3698 3699 if (node == NUMA_NO_NODE) 3700 searchnode = numa_mem_id(); 3701 3702 slab = get_partial_node(s, get_node(s, searchnode), pc); 3703 if (slab || (node != NUMA_NO_NODE && (pc->flags & __GFP_THISNODE))) 3704 return slab; 3705 3706 return get_any_partial(s, pc); 3707 } 3708 3709 #ifdef CONFIG_PREEMPTION 3710 /* 3711 * Calculate the next globally unique transaction for disambiguation 3712 * during cmpxchg. The transactions start with the cpu number and are then 3713 * incremented by CONFIG_NR_CPUS. 3714 */ 3715 #define TID_STEP roundup_pow_of_two(CONFIG_NR_CPUS) 3716 #else 3717 /* 3718 * No preemption supported therefore also no need to check for 3719 * different cpus. 3720 */ 3721 #define TID_STEP 1 3722 #endif /* CONFIG_PREEMPTION */ 3723 3724 static inline unsigned long next_tid(unsigned long tid) 3725 { 3726 return tid + TID_STEP; 3727 } 3728 3729 #ifdef SLUB_DEBUG_CMPXCHG 3730 static inline unsigned int tid_to_cpu(unsigned long tid) 3731 { 3732 return tid % TID_STEP; 3733 } 3734 3735 static inline unsigned long tid_to_event(unsigned long tid) 3736 { 3737 return tid / TID_STEP; 3738 } 3739 #endif 3740 3741 static inline unsigned int init_tid(int cpu) 3742 { 3743 return cpu; 3744 } 3745 3746 static inline void note_cmpxchg_failure(const char *n, 3747 const struct kmem_cache *s, unsigned long tid) 3748 { 3749 #ifdef SLUB_DEBUG_CMPXCHG 3750 unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid); 3751 3752 pr_info("%s %s: cmpxchg redo ", n, s->name); 3753 3754 if (IS_ENABLED(CONFIG_PREEMPTION) && 3755 tid_to_cpu(tid) != tid_to_cpu(actual_tid)) { 3756 pr_warn("due to cpu change %d -> %d\n", 3757 tid_to_cpu(tid), tid_to_cpu(actual_tid)); 3758 } else if (tid_to_event(tid) != tid_to_event(actual_tid)) { 3759 pr_warn("due to cpu running other code. Event %ld->%ld\n", 3760 tid_to_event(tid), tid_to_event(actual_tid)); 3761 } else { 3762 pr_warn("for unknown reason: actual=%lx was=%lx target=%lx\n", 3763 actual_tid, tid, next_tid(tid)); 3764 } 3765 #endif 3766 stat(s, CMPXCHG_DOUBLE_CPU_FAIL); 3767 } 3768 3769 static void init_kmem_cache_cpus(struct kmem_cache *s) 3770 { 3771 #ifdef CONFIG_PREEMPT_RT 3772 /* 3773 * Register lockdep key for non-boot kmem caches to avoid 3774 * WARN_ON_ONCE(static_obj(key))) in lockdep_register_key() 3775 */ 3776 bool finegrain_lockdep = !init_section_contains(s, 1); 3777 #else 3778 /* 3779 * Don't bother with different lockdep classes for each 3780 * kmem_cache, since we only use local_trylock_irqsave(). 3781 */ 3782 bool finegrain_lockdep = false; 3783 #endif 3784 int cpu; 3785 struct kmem_cache_cpu *c; 3786 3787 if (finegrain_lockdep) 3788 lockdep_register_key(&s->lock_key); 3789 for_each_possible_cpu(cpu) { 3790 c = per_cpu_ptr(s->cpu_slab, cpu); 3791 local_trylock_init(&c->lock); 3792 if (finegrain_lockdep) 3793 lockdep_set_class(&c->lock, &s->lock_key); 3794 c->tid = init_tid(cpu); 3795 } 3796 } 3797 3798 /* 3799 * Finishes removing the cpu slab. Merges cpu's freelist with slab's freelist, 3800 * unfreezes the slabs and puts it on the proper list. 3801 * Assumes the slab has been already safely taken away from kmem_cache_cpu 3802 * by the caller. 3803 */ 3804 static void deactivate_slab(struct kmem_cache *s, struct slab *slab, 3805 void *freelist) 3806 { 3807 struct kmem_cache_node *n = get_node(s, slab_nid(slab)); 3808 int free_delta = 0; 3809 void *nextfree, *freelist_iter, *freelist_tail; 3810 int tail = DEACTIVATE_TO_HEAD; 3811 unsigned long flags = 0; 3812 struct slab new; 3813 struct slab old; 3814 3815 if (READ_ONCE(slab->freelist)) { 3816 stat(s, DEACTIVATE_REMOTE_FREES); 3817 tail = DEACTIVATE_TO_TAIL; 3818 } 3819 3820 /* 3821 * Stage one: Count the objects on cpu's freelist as free_delta and 3822 * remember the last object in freelist_tail for later splicing. 3823 */ 3824 freelist_tail = NULL; 3825 freelist_iter = freelist; 3826 while (freelist_iter) { 3827 nextfree = get_freepointer(s, freelist_iter); 3828 3829 /* 3830 * If 'nextfree' is invalid, it is possible that the object at 3831 * 'freelist_iter' is already corrupted. So isolate all objects 3832 * starting at 'freelist_iter' by skipping them. 3833 */ 3834 if (freelist_corrupted(s, slab, &freelist_iter, nextfree)) 3835 break; 3836 3837 freelist_tail = freelist_iter; 3838 free_delta++; 3839 3840 freelist_iter = nextfree; 3841 } 3842 3843 /* 3844 * Stage two: Unfreeze the slab while splicing the per-cpu 3845 * freelist to the head of slab's freelist. 3846 */ 3847 do { 3848 old.freelist = READ_ONCE(slab->freelist); 3849 old.counters = READ_ONCE(slab->counters); 3850 VM_BUG_ON(!old.frozen); 3851 3852 /* Determine target state of the slab */ 3853 new.counters = old.counters; 3854 new.frozen = 0; 3855 if (freelist_tail) { 3856 new.inuse -= free_delta; 3857 set_freepointer(s, freelist_tail, old.freelist); 3858 new.freelist = freelist; 3859 } else { 3860 new.freelist = old.freelist; 3861 } 3862 } while (!slab_update_freelist(s, slab, 3863 old.freelist, old.counters, 3864 new.freelist, new.counters, 3865 "unfreezing slab")); 3866 3867 /* 3868 * Stage three: Manipulate the slab list based on the updated state. 3869 */ 3870 if (!new.inuse && n->nr_partial >= s->min_partial) { 3871 stat(s, DEACTIVATE_EMPTY); 3872 discard_slab(s, slab); 3873 stat(s, FREE_SLAB); 3874 } else if (new.freelist) { 3875 spin_lock_irqsave(&n->list_lock, flags); 3876 add_partial(n, slab, tail); 3877 spin_unlock_irqrestore(&n->list_lock, flags); 3878 stat(s, tail); 3879 } else { 3880 stat(s, DEACTIVATE_FULL); 3881 } 3882 } 3883 3884 /* 3885 * ___slab_alloc()'s caller is supposed to check if kmem_cache::kmem_cache_cpu::lock 3886 * can be acquired without a deadlock before invoking the function. 3887 * 3888 * Without LOCKDEP we trust the code to be correct. kmalloc_nolock() is 3889 * using local_lock_is_locked() properly before calling local_lock_cpu_slab(), 3890 * and kmalloc() is not used in an unsupported context. 3891 * 3892 * With LOCKDEP, on PREEMPT_RT lockdep does its checking in local_lock_irqsave(). 3893 * On !PREEMPT_RT we use trylock to avoid false positives in NMI, but 3894 * lockdep_assert() will catch a bug in case: 3895 * #1 3896 * kmalloc() -> ___slab_alloc() -> irqsave -> NMI -> bpf -> kmalloc_nolock() 3897 * or 3898 * #2 3899 * kmalloc() -> ___slab_alloc() -> irqsave -> tracepoint/kprobe -> bpf -> kmalloc_nolock() 3900 * 3901 * On PREEMPT_RT an invocation is not possible from IRQ-off or preempt 3902 * disabled context. The lock will always be acquired and if needed it 3903 * block and sleep until the lock is available. 3904 * #1 is possible in !PREEMPT_RT only. 3905 * #2 is possible in both with a twist that irqsave is replaced with rt_spinlock: 3906 * kmalloc() -> ___slab_alloc() -> rt_spin_lock(kmem_cache_A) -> 3907 * tracepoint/kprobe -> bpf -> kmalloc_nolock() -> rt_spin_lock(kmem_cache_B) 3908 * 3909 * local_lock_is_locked() prevents the case kmem_cache_A == kmem_cache_B 3910 */ 3911 #if defined(CONFIG_PREEMPT_RT) || !defined(CONFIG_LOCKDEP) 3912 #define local_lock_cpu_slab(s, flags) \ 3913 local_lock_irqsave(&(s)->cpu_slab->lock, flags) 3914 #else 3915 #define local_lock_cpu_slab(s, flags) \ 3916 do { \ 3917 bool __l = local_trylock_irqsave(&(s)->cpu_slab->lock, flags); \ 3918 lockdep_assert(__l); \ 3919 } while (0) 3920 #endif 3921 3922 #define local_unlock_cpu_slab(s, flags) \ 3923 local_unlock_irqrestore(&(s)->cpu_slab->lock, flags) 3924 3925 #ifdef CONFIG_SLUB_CPU_PARTIAL 3926 static void __put_partials(struct kmem_cache *s, struct slab *partial_slab) 3927 { 3928 struct kmem_cache_node *n = NULL, *n2 = NULL; 3929 struct slab *slab, *slab_to_discard = NULL; 3930 unsigned long flags = 0; 3931 3932 while (partial_slab) { 3933 slab = partial_slab; 3934 partial_slab = slab->next; 3935 3936 n2 = get_node(s, slab_nid(slab)); 3937 if (n != n2) { 3938 if (n) 3939 spin_unlock_irqrestore(&n->list_lock, flags); 3940 3941 n = n2; 3942 spin_lock_irqsave(&n->list_lock, flags); 3943 } 3944 3945 if (unlikely(!slab->inuse && n->nr_partial >= s->min_partial)) { 3946 slab->next = slab_to_discard; 3947 slab_to_discard = slab; 3948 } else { 3949 add_partial(n, slab, DEACTIVATE_TO_TAIL); 3950 stat(s, FREE_ADD_PARTIAL); 3951 } 3952 } 3953 3954 if (n) 3955 spin_unlock_irqrestore(&n->list_lock, flags); 3956 3957 while (slab_to_discard) { 3958 slab = slab_to_discard; 3959 slab_to_discard = slab_to_discard->next; 3960 3961 stat(s, DEACTIVATE_EMPTY); 3962 discard_slab(s, slab); 3963 stat(s, FREE_SLAB); 3964 } 3965 } 3966 3967 /* 3968 * Put all the cpu partial slabs to the node partial list. 3969 */ 3970 static void put_partials(struct kmem_cache *s) 3971 { 3972 struct slab *partial_slab; 3973 unsigned long flags; 3974 3975 local_lock_irqsave(&s->cpu_slab->lock, flags); 3976 partial_slab = this_cpu_read(s->cpu_slab->partial); 3977 this_cpu_write(s->cpu_slab->partial, NULL); 3978 local_unlock_irqrestore(&s->cpu_slab->lock, flags); 3979 3980 if (partial_slab) 3981 __put_partials(s, partial_slab); 3982 } 3983 3984 static void put_partials_cpu(struct kmem_cache *s, 3985 struct kmem_cache_cpu *c) 3986 { 3987 struct slab *partial_slab; 3988 3989 partial_slab = slub_percpu_partial(c); 3990 c->partial = NULL; 3991 3992 if (partial_slab) 3993 __put_partials(s, partial_slab); 3994 } 3995 3996 /* 3997 * Put a slab into a partial slab slot if available. 3998 * 3999 * If we did not find a slot then simply move all the partials to the 4000 * per node partial list. 4001 */ 4002 static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain) 4003 { 4004 struct slab *oldslab; 4005 struct slab *slab_to_put = NULL; 4006 unsigned long flags; 4007 int slabs = 0; 4008 4009 local_lock_cpu_slab(s, flags); 4010 4011 oldslab = this_cpu_read(s->cpu_slab->partial); 4012 4013 if (oldslab) { 4014 if (drain && oldslab->slabs >= s->cpu_partial_slabs) { 4015 /* 4016 * Partial array is full. Move the existing set to the 4017 * per node partial list. Postpone the actual unfreezing 4018 * outside of the critical section. 4019 */ 4020 slab_to_put = oldslab; 4021 oldslab = NULL; 4022 } else { 4023 slabs = oldslab->slabs; 4024 } 4025 } 4026 4027 slabs++; 4028 4029 slab->slabs = slabs; 4030 slab->next = oldslab; 4031 4032 this_cpu_write(s->cpu_slab->partial, slab); 4033 4034 local_unlock_cpu_slab(s, flags); 4035 4036 if (slab_to_put) { 4037 __put_partials(s, slab_to_put); 4038 stat(s, CPU_PARTIAL_DRAIN); 4039 } 4040 } 4041 4042 #else /* CONFIG_SLUB_CPU_PARTIAL */ 4043 4044 static inline void put_partials(struct kmem_cache *s) { } 4045 static inline void put_partials_cpu(struct kmem_cache *s, 4046 struct kmem_cache_cpu *c) { } 4047 4048 #endif /* CONFIG_SLUB_CPU_PARTIAL */ 4049 4050 static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) 4051 { 4052 unsigned long flags; 4053 struct slab *slab; 4054 void *freelist; 4055 4056 local_lock_irqsave(&s->cpu_slab->lock, flags); 4057 4058 slab = c->slab; 4059 freelist = c->freelist; 4060 4061 c->slab = NULL; 4062 c->freelist = NULL; 4063 c->tid = next_tid(c->tid); 4064 4065 local_unlock_irqrestore(&s->cpu_slab->lock, flags); 4066 4067 if (slab) { 4068 deactivate_slab(s, slab, freelist); 4069 stat(s, CPUSLAB_FLUSH); 4070 } 4071 } 4072 4073 static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu) 4074 { 4075 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); 4076 void *freelist = c->freelist; 4077 struct slab *slab = c->slab; 4078 4079 c->slab = NULL; 4080 c->freelist = NULL; 4081 c->tid = next_tid(c->tid); 4082 4083 if (slab) { 4084 deactivate_slab(s, slab, freelist); 4085 stat(s, CPUSLAB_FLUSH); 4086 } 4087 4088 put_partials_cpu(s, c); 4089 } 4090 4091 static inline void flush_this_cpu_slab(struct kmem_cache *s) 4092 { 4093 struct kmem_cache_cpu *c = this_cpu_ptr(s->cpu_slab); 4094 4095 if (c->slab) 4096 flush_slab(s, c); 4097 4098 put_partials(s); 4099 } 4100 4101 static bool has_cpu_slab(int cpu, struct kmem_cache *s) 4102 { 4103 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); 4104 4105 return c->slab || slub_percpu_partial(c); 4106 } 4107 4108 static bool has_pcs_used(int cpu, struct kmem_cache *s) 4109 { 4110 struct slub_percpu_sheaves *pcs; 4111 4112 if (!s->cpu_sheaves) 4113 return false; 4114 4115 pcs = per_cpu_ptr(s->cpu_sheaves, cpu); 4116 4117 return (pcs->spare || pcs->rcu_free || pcs->main->size); 4118 } 4119 4120 /* 4121 * Flush cpu slab. 4122 * 4123 * Called from CPU work handler with migration disabled. 4124 */ 4125 static void flush_cpu_slab(struct work_struct *w) 4126 { 4127 struct kmem_cache *s; 4128 struct slub_flush_work *sfw; 4129 4130 sfw = container_of(w, struct slub_flush_work, work); 4131 4132 s = sfw->s; 4133 4134 if (s->cpu_sheaves) 4135 pcs_flush_all(s); 4136 4137 flush_this_cpu_slab(s); 4138 } 4139 4140 static void flush_all_cpus_locked(struct kmem_cache *s) 4141 { 4142 struct slub_flush_work *sfw; 4143 unsigned int cpu; 4144 4145 lockdep_assert_cpus_held(); 4146 mutex_lock(&flush_lock); 4147 4148 for_each_online_cpu(cpu) { 4149 sfw = &per_cpu(slub_flush, cpu); 4150 if (!has_cpu_slab(cpu, s) && !has_pcs_used(cpu, s)) { 4151 sfw->skip = true; 4152 continue; 4153 } 4154 INIT_WORK(&sfw->work, flush_cpu_slab); 4155 sfw->skip = false; 4156 sfw->s = s; 4157 queue_work_on(cpu, flushwq, &sfw->work); 4158 } 4159 4160 for_each_online_cpu(cpu) { 4161 sfw = &per_cpu(slub_flush, cpu); 4162 if (sfw->skip) 4163 continue; 4164 flush_work(&sfw->work); 4165 } 4166 4167 mutex_unlock(&flush_lock); 4168 } 4169 4170 static void flush_all(struct kmem_cache *s) 4171 { 4172 cpus_read_lock(); 4173 flush_all_cpus_locked(s); 4174 cpus_read_unlock(); 4175 } 4176 4177 static void flush_rcu_sheaf(struct work_struct *w) 4178 { 4179 struct slub_percpu_sheaves *pcs; 4180 struct slab_sheaf *rcu_free; 4181 struct slub_flush_work *sfw; 4182 struct kmem_cache *s; 4183 4184 sfw = container_of(w, struct slub_flush_work, work); 4185 s = sfw->s; 4186 4187 local_lock(&s->cpu_sheaves->lock); 4188 pcs = this_cpu_ptr(s->cpu_sheaves); 4189 4190 rcu_free = pcs->rcu_free; 4191 pcs->rcu_free = NULL; 4192 4193 local_unlock(&s->cpu_sheaves->lock); 4194 4195 if (rcu_free) 4196 call_rcu(&rcu_free->rcu_head, rcu_free_sheaf_nobarn); 4197 } 4198 4199 4200 /* needed for kvfree_rcu_barrier() */ 4201 void flush_rcu_sheaves_on_cache(struct kmem_cache *s) 4202 { 4203 struct slub_flush_work *sfw; 4204 unsigned int cpu; 4205 4206 lockdep_assert_cpus_held(); 4207 mutex_lock(&flush_lock); 4208 4209 for_each_online_cpu(cpu) { 4210 sfw = &per_cpu(slub_flush, cpu); 4211 4212 /* 4213 * we don't check if rcu_free sheaf exists - racing 4214 * __kfree_rcu_sheaf() might have just removed it. 4215 * by executing flush_rcu_sheaf() on the cpu we make 4216 * sure the __kfree_rcu_sheaf() finished its call_rcu() 4217 */ 4218 4219 INIT_WORK(&sfw->work, flush_rcu_sheaf); 4220 sfw->s = s; 4221 queue_work_on(cpu, flushwq, &sfw->work); 4222 } 4223 4224 for_each_online_cpu(cpu) { 4225 sfw = &per_cpu(slub_flush, cpu); 4226 flush_work(&sfw->work); 4227 } 4228 4229 mutex_unlock(&flush_lock); 4230 } 4231 4232 void flush_all_rcu_sheaves(void) 4233 { 4234 struct kmem_cache *s; 4235 4236 cpus_read_lock(); 4237 mutex_lock(&slab_mutex); 4238 4239 list_for_each_entry(s, &slab_caches, list) { 4240 if (!s->cpu_sheaves) 4241 continue; 4242 flush_rcu_sheaves_on_cache(s); 4243 } 4244 4245 mutex_unlock(&slab_mutex); 4246 cpus_read_unlock(); 4247 4248 rcu_barrier(); 4249 } 4250 4251 /* 4252 * Use the cpu notifier to insure that the cpu slabs are flushed when 4253 * necessary. 4254 */ 4255 static int slub_cpu_dead(unsigned int cpu) 4256 { 4257 struct kmem_cache *s; 4258 4259 mutex_lock(&slab_mutex); 4260 list_for_each_entry(s, &slab_caches, list) { 4261 __flush_cpu_slab(s, cpu); 4262 if (s->cpu_sheaves) 4263 __pcs_flush_all_cpu(s, cpu); 4264 } 4265 mutex_unlock(&slab_mutex); 4266 return 0; 4267 } 4268 4269 /* 4270 * Check if the objects in a per cpu structure fit numa 4271 * locality expectations. 4272 */ 4273 static inline int node_match(struct slab *slab, int node) 4274 { 4275 #ifdef CONFIG_NUMA 4276 if (node != NUMA_NO_NODE && slab_nid(slab) != node) 4277 return 0; 4278 #endif 4279 return 1; 4280 } 4281 4282 #ifdef CONFIG_SLUB_DEBUG 4283 static int count_free(struct slab *slab) 4284 { 4285 return slab->objects - slab->inuse; 4286 } 4287 4288 static inline unsigned long node_nr_objs(struct kmem_cache_node *n) 4289 { 4290 return atomic_long_read(&n->total_objects); 4291 } 4292 4293 /* Supports checking bulk free of a constructed freelist */ 4294 static inline bool free_debug_processing(struct kmem_cache *s, 4295 struct slab *slab, void *head, void *tail, int *bulk_cnt, 4296 unsigned long addr, depot_stack_handle_t handle) 4297 { 4298 bool checks_ok = false; 4299 void *object = head; 4300 int cnt = 0; 4301 4302 if (s->flags & SLAB_CONSISTENCY_CHECKS) { 4303 if (!check_slab(s, slab)) 4304 goto out; 4305 } 4306 4307 if (slab->inuse < *bulk_cnt) { 4308 slab_err(s, slab, "Slab has %d allocated objects but %d are to be freed\n", 4309 slab->inuse, *bulk_cnt); 4310 goto out; 4311 } 4312 4313 next_object: 4314 4315 if (++cnt > *bulk_cnt) 4316 goto out_cnt; 4317 4318 if (s->flags & SLAB_CONSISTENCY_CHECKS) { 4319 if (!free_consistency_checks(s, slab, object, addr)) 4320 goto out; 4321 } 4322 4323 if (s->flags & SLAB_STORE_USER) 4324 set_track_update(s, object, TRACK_FREE, addr, handle); 4325 trace(s, slab, object, 0); 4326 /* Freepointer not overwritten by init_object(), SLAB_POISON moved it */ 4327 init_object(s, object, SLUB_RED_INACTIVE); 4328 4329 /* Reached end of constructed freelist yet? */ 4330 if (object != tail) { 4331 object = get_freepointer(s, object); 4332 goto next_object; 4333 } 4334 checks_ok = true; 4335 4336 out_cnt: 4337 if (cnt != *bulk_cnt) { 4338 slab_err(s, slab, "Bulk free expected %d objects but found %d\n", 4339 *bulk_cnt, cnt); 4340 *bulk_cnt = cnt; 4341 } 4342 4343 out: 4344 4345 if (!checks_ok) 4346 slab_fix(s, "Object at 0x%p not freed", object); 4347 4348 return checks_ok; 4349 } 4350 #endif /* CONFIG_SLUB_DEBUG */ 4351 4352 #if defined(CONFIG_SLUB_DEBUG) || defined(SLAB_SUPPORTS_SYSFS) 4353 static unsigned long count_partial(struct kmem_cache_node *n, 4354 int (*get_count)(struct slab *)) 4355 { 4356 unsigned long flags; 4357 unsigned long x = 0; 4358 struct slab *slab; 4359 4360 spin_lock_irqsave(&n->list_lock, flags); 4361 list_for_each_entry(slab, &n->partial, slab_list) 4362 x += get_count(slab); 4363 spin_unlock_irqrestore(&n->list_lock, flags); 4364 return x; 4365 } 4366 #endif /* CONFIG_SLUB_DEBUG || SLAB_SUPPORTS_SYSFS */ 4367 4368 #ifdef CONFIG_SLUB_DEBUG 4369 #define MAX_PARTIAL_TO_SCAN 10000 4370 4371 static unsigned long count_partial_free_approx(struct kmem_cache_node *n) 4372 { 4373 unsigned long flags; 4374 unsigned long x = 0; 4375 struct slab *slab; 4376 4377 spin_lock_irqsave(&n->list_lock, flags); 4378 if (n->nr_partial <= MAX_PARTIAL_TO_SCAN) { 4379 list_for_each_entry(slab, &n->partial, slab_list) 4380 x += slab->objects - slab->inuse; 4381 } else { 4382 /* 4383 * For a long list, approximate the total count of objects in 4384 * it to meet the limit on the number of slabs to scan. 4385 * Scan from both the list's head and tail for better accuracy. 4386 */ 4387 unsigned long scanned = 0; 4388 4389 list_for_each_entry(slab, &n->partial, slab_list) { 4390 x += slab->objects - slab->inuse; 4391 if (++scanned == MAX_PARTIAL_TO_SCAN / 2) 4392 break; 4393 } 4394 list_for_each_entry_reverse(slab, &n->partial, slab_list) { 4395 x += slab->objects - slab->inuse; 4396 if (++scanned == MAX_PARTIAL_TO_SCAN) 4397 break; 4398 } 4399 x = mult_frac(x, n->nr_partial, scanned); 4400 x = min(x, node_nr_objs(n)); 4401 } 4402 spin_unlock_irqrestore(&n->list_lock, flags); 4403 return x; 4404 } 4405 4406 static noinline void 4407 slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) 4408 { 4409 static DEFINE_RATELIMIT_STATE(slub_oom_rs, DEFAULT_RATELIMIT_INTERVAL, 4410 DEFAULT_RATELIMIT_BURST); 4411 int cpu = raw_smp_processor_id(); 4412 int node; 4413 struct kmem_cache_node *n; 4414 4415 if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slub_oom_rs)) 4416 return; 4417 4418 pr_warn("SLUB: Unable to allocate memory on CPU %u (of node %d) on node %d, gfp=%#x(%pGg)\n", 4419 cpu, cpu_to_node(cpu), nid, gfpflags, &gfpflags); 4420 pr_warn(" cache: %s, object size: %u, buffer size: %u, default order: %u, min order: %u\n", 4421 s->name, s->object_size, s->size, oo_order(s->oo), 4422 oo_order(s->min)); 4423 4424 if (oo_order(s->min) > get_order(s->object_size)) 4425 pr_warn(" %s debugging increased min order, use slab_debug=O to disable.\n", 4426 s->name); 4427 4428 for_each_kmem_cache_node(s, node, n) { 4429 unsigned long nr_slabs; 4430 unsigned long nr_objs; 4431 unsigned long nr_free; 4432 4433 nr_free = count_partial_free_approx(n); 4434 nr_slabs = node_nr_slabs(n); 4435 nr_objs = node_nr_objs(n); 4436 4437 pr_warn(" node %d: slabs: %ld, objs: %ld, free: %ld\n", 4438 node, nr_slabs, nr_objs, nr_free); 4439 } 4440 } 4441 #else /* CONFIG_SLUB_DEBUG */ 4442 static inline void 4443 slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) { } 4444 #endif 4445 4446 static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags) 4447 { 4448 if (unlikely(slab_test_pfmemalloc(slab))) 4449 return gfp_pfmemalloc_allowed(gfpflags); 4450 4451 return true; 4452 } 4453 4454 static inline bool 4455 __update_cpu_freelist_fast(struct kmem_cache *s, 4456 void *freelist_old, void *freelist_new, 4457 unsigned long tid) 4458 { 4459 freelist_aba_t old = { .freelist = freelist_old, .counter = tid }; 4460 freelist_aba_t new = { .freelist = freelist_new, .counter = next_tid(tid) }; 4461 4462 return this_cpu_try_cmpxchg_freelist(s->cpu_slab->freelist_tid.full, 4463 &old.full, new.full); 4464 } 4465 4466 /* 4467 * Check the slab->freelist and either transfer the freelist to the 4468 * per cpu freelist or deactivate the slab. 4469 * 4470 * The slab is still frozen if the return value is not NULL. 4471 * 4472 * If this function returns NULL then the slab has been unfrozen. 4473 */ 4474 static inline void *get_freelist(struct kmem_cache *s, struct slab *slab) 4475 { 4476 struct slab new; 4477 unsigned long counters; 4478 void *freelist; 4479 4480 lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock)); 4481 4482 do { 4483 freelist = slab->freelist; 4484 counters = slab->counters; 4485 4486 new.counters = counters; 4487 4488 new.inuse = slab->objects; 4489 new.frozen = freelist != NULL; 4490 4491 } while (!__slab_update_freelist(s, slab, 4492 freelist, counters, 4493 NULL, new.counters, 4494 "get_freelist")); 4495 4496 return freelist; 4497 } 4498 4499 /* 4500 * Freeze the partial slab and return the pointer to the freelist. 4501 */ 4502 static inline void *freeze_slab(struct kmem_cache *s, struct slab *slab) 4503 { 4504 struct slab new; 4505 unsigned long counters; 4506 void *freelist; 4507 4508 do { 4509 freelist = slab->freelist; 4510 counters = slab->counters; 4511 4512 new.counters = counters; 4513 VM_BUG_ON(new.frozen); 4514 4515 new.inuse = slab->objects; 4516 new.frozen = 1; 4517 4518 } while (!slab_update_freelist(s, slab, 4519 freelist, counters, 4520 NULL, new.counters, 4521 "freeze_slab")); 4522 4523 return freelist; 4524 } 4525 4526 /* 4527 * Slow path. The lockless freelist is empty or we need to perform 4528 * debugging duties. 4529 * 4530 * Processing is still very fast if new objects have been freed to the 4531 * regular freelist. In that case we simply take over the regular freelist 4532 * as the lockless freelist and zap the regular freelist. 4533 * 4534 * If that is not working then we fall back to the partial lists. We take the 4535 * first element of the freelist as the object to allocate now and move the 4536 * rest of the freelist to the lockless freelist. 4537 * 4538 * And if we were unable to get a new slab from the partial slab lists then 4539 * we need to allocate a new slab. This is the slowest path since it involves 4540 * a call to the page allocator and the setup of a new slab. 4541 * 4542 * Version of __slab_alloc to use when we know that preemption is 4543 * already disabled (which is the case for bulk allocation). 4544 */ 4545 static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, 4546 unsigned long addr, struct kmem_cache_cpu *c, unsigned int orig_size) 4547 { 4548 bool allow_spin = gfpflags_allow_spinning(gfpflags); 4549 void *freelist; 4550 struct slab *slab; 4551 unsigned long flags; 4552 struct partial_context pc; 4553 bool try_thisnode = true; 4554 4555 stat(s, ALLOC_SLOWPATH); 4556 4557 reread_slab: 4558 4559 slab = READ_ONCE(c->slab); 4560 if (!slab) { 4561 /* 4562 * if the node is not online or has no normal memory, just 4563 * ignore the node constraint 4564 */ 4565 if (unlikely(node != NUMA_NO_NODE && 4566 !node_isset(node, slab_nodes))) 4567 node = NUMA_NO_NODE; 4568 goto new_slab; 4569 } 4570 4571 if (unlikely(!node_match(slab, node))) { 4572 /* 4573 * same as above but node_match() being false already 4574 * implies node != NUMA_NO_NODE. 4575 * 4576 * We don't strictly honor pfmemalloc and NUMA preferences 4577 * when !allow_spin because: 4578 * 4579 * 1. Most kmalloc() users allocate objects on the local node, 4580 * so kmalloc_nolock() tries not to interfere with them by 4581 * deactivating the cpu slab. 4582 * 4583 * 2. Deactivating due to NUMA or pfmemalloc mismatch may cause 4584 * unnecessary slab allocations even when n->partial list 4585 * is not empty. 4586 */ 4587 if (!node_isset(node, slab_nodes) || 4588 !allow_spin) { 4589 node = NUMA_NO_NODE; 4590 } else { 4591 stat(s, ALLOC_NODE_MISMATCH); 4592 goto deactivate_slab; 4593 } 4594 } 4595 4596 /* 4597 * By rights, we should be searching for a slab page that was 4598 * PFMEMALLOC but right now, we are losing the pfmemalloc 4599 * information when the page leaves the per-cpu allocator 4600 */ 4601 if (unlikely(!pfmemalloc_match(slab, gfpflags) && allow_spin)) 4602 goto deactivate_slab; 4603 4604 /* must check again c->slab in case we got preempted and it changed */ 4605 local_lock_cpu_slab(s, flags); 4606 4607 if (unlikely(slab != c->slab)) { 4608 local_unlock_cpu_slab(s, flags); 4609 goto reread_slab; 4610 } 4611 freelist = c->freelist; 4612 if (freelist) 4613 goto load_freelist; 4614 4615 freelist = get_freelist(s, slab); 4616 4617 if (!freelist) { 4618 c->slab = NULL; 4619 c->tid = next_tid(c->tid); 4620 local_unlock_cpu_slab(s, flags); 4621 stat(s, DEACTIVATE_BYPASS); 4622 goto new_slab; 4623 } 4624 4625 stat(s, ALLOC_REFILL); 4626 4627 load_freelist: 4628 4629 lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock)); 4630 4631 /* 4632 * freelist is pointing to the list of objects to be used. 4633 * slab is pointing to the slab from which the objects are obtained. 4634 * That slab must be frozen for per cpu allocations to work. 4635 */ 4636 VM_BUG_ON(!c->slab->frozen); 4637 c->freelist = get_freepointer(s, freelist); 4638 c->tid = next_tid(c->tid); 4639 local_unlock_cpu_slab(s, flags); 4640 return freelist; 4641 4642 deactivate_slab: 4643 4644 local_lock_cpu_slab(s, flags); 4645 if (slab != c->slab) { 4646 local_unlock_cpu_slab(s, flags); 4647 goto reread_slab; 4648 } 4649 freelist = c->freelist; 4650 c->slab = NULL; 4651 c->freelist = NULL; 4652 c->tid = next_tid(c->tid); 4653 local_unlock_cpu_slab(s, flags); 4654 deactivate_slab(s, slab, freelist); 4655 4656 new_slab: 4657 4658 #ifdef CONFIG_SLUB_CPU_PARTIAL 4659 while (slub_percpu_partial(c)) { 4660 local_lock_cpu_slab(s, flags); 4661 if (unlikely(c->slab)) { 4662 local_unlock_cpu_slab(s, flags); 4663 goto reread_slab; 4664 } 4665 if (unlikely(!slub_percpu_partial(c))) { 4666 local_unlock_cpu_slab(s, flags); 4667 /* we were preempted and partial list got empty */ 4668 goto new_objects; 4669 } 4670 4671 slab = slub_percpu_partial(c); 4672 slub_set_percpu_partial(c, slab); 4673 4674 if (likely(node_match(slab, node) && 4675 pfmemalloc_match(slab, gfpflags)) || 4676 !allow_spin) { 4677 c->slab = slab; 4678 freelist = get_freelist(s, slab); 4679 VM_BUG_ON(!freelist); 4680 stat(s, CPU_PARTIAL_ALLOC); 4681 goto load_freelist; 4682 } 4683 4684 local_unlock_cpu_slab(s, flags); 4685 4686 slab->next = NULL; 4687 __put_partials(s, slab); 4688 } 4689 #endif 4690 4691 new_objects: 4692 4693 pc.flags = gfpflags; 4694 /* 4695 * When a preferred node is indicated but no __GFP_THISNODE 4696 * 4697 * 1) try to get a partial slab from target node only by having 4698 * __GFP_THISNODE in pc.flags for get_partial() 4699 * 2) if 1) failed, try to allocate a new slab from target node with 4700 * GPF_NOWAIT | __GFP_THISNODE opportunistically 4701 * 3) if 2) failed, retry with original gfpflags which will allow 4702 * get_partial() try partial lists of other nodes before potentially 4703 * allocating new page from other nodes 4704 */ 4705 if (unlikely(node != NUMA_NO_NODE && !(gfpflags & __GFP_THISNODE) 4706 && try_thisnode)) { 4707 if (unlikely(!allow_spin)) 4708 /* Do not upgrade gfp to NOWAIT from more restrictive mode */ 4709 pc.flags = gfpflags | __GFP_THISNODE; 4710 else 4711 pc.flags = GFP_NOWAIT | __GFP_THISNODE; 4712 } 4713 4714 pc.orig_size = orig_size; 4715 slab = get_partial(s, node, &pc); 4716 if (slab) { 4717 if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) { 4718 freelist = pc.object; 4719 /* 4720 * For debug caches here we had to go through 4721 * alloc_single_from_partial() so just store the 4722 * tracking info and return the object. 4723 * 4724 * Due to disabled preemption we need to disallow 4725 * blocking. The flags are further adjusted by 4726 * gfp_nested_mask() in stack_depot itself. 4727 */ 4728 if (s->flags & SLAB_STORE_USER) 4729 set_track(s, freelist, TRACK_ALLOC, addr, 4730 gfpflags & ~(__GFP_DIRECT_RECLAIM)); 4731 4732 return freelist; 4733 } 4734 4735 freelist = freeze_slab(s, slab); 4736 goto retry_load_slab; 4737 } 4738 4739 slub_put_cpu_ptr(s->cpu_slab); 4740 slab = new_slab(s, pc.flags, node); 4741 c = slub_get_cpu_ptr(s->cpu_slab); 4742 4743 if (unlikely(!slab)) { 4744 if (node != NUMA_NO_NODE && !(gfpflags & __GFP_THISNODE) 4745 && try_thisnode) { 4746 try_thisnode = false; 4747 goto new_objects; 4748 } 4749 slab_out_of_memory(s, gfpflags, node); 4750 return NULL; 4751 } 4752 4753 stat(s, ALLOC_SLAB); 4754 4755 if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) { 4756 freelist = alloc_single_from_new_slab(s, slab, orig_size, gfpflags); 4757 4758 if (unlikely(!freelist)) { 4759 /* This could cause an endless loop. Fail instead. */ 4760 if (!allow_spin) 4761 return NULL; 4762 goto new_objects; 4763 } 4764 4765 if (s->flags & SLAB_STORE_USER) 4766 set_track(s, freelist, TRACK_ALLOC, addr, 4767 gfpflags & ~(__GFP_DIRECT_RECLAIM)); 4768 4769 return freelist; 4770 } 4771 4772 /* 4773 * No other reference to the slab yet so we can 4774 * muck around with it freely without cmpxchg 4775 */ 4776 freelist = slab->freelist; 4777 slab->freelist = NULL; 4778 slab->inuse = slab->objects; 4779 slab->frozen = 1; 4780 4781 inc_slabs_node(s, slab_nid(slab), slab->objects); 4782 4783 if (unlikely(!pfmemalloc_match(slab, gfpflags) && allow_spin)) { 4784 /* 4785 * For !pfmemalloc_match() case we don't load freelist so that 4786 * we don't make further mismatched allocations easier. 4787 */ 4788 deactivate_slab(s, slab, get_freepointer(s, freelist)); 4789 return freelist; 4790 } 4791 4792 retry_load_slab: 4793 4794 local_lock_cpu_slab(s, flags); 4795 if (unlikely(c->slab)) { 4796 void *flush_freelist = c->freelist; 4797 struct slab *flush_slab = c->slab; 4798 4799 c->slab = NULL; 4800 c->freelist = NULL; 4801 c->tid = next_tid(c->tid); 4802 4803 local_unlock_cpu_slab(s, flags); 4804 4805 if (unlikely(!allow_spin)) { 4806 /* Reentrant slub cannot take locks, defer */ 4807 defer_deactivate_slab(flush_slab, flush_freelist); 4808 } else { 4809 deactivate_slab(s, flush_slab, flush_freelist); 4810 } 4811 4812 stat(s, CPUSLAB_FLUSH); 4813 4814 goto retry_load_slab; 4815 } 4816 c->slab = slab; 4817 4818 goto load_freelist; 4819 } 4820 /* 4821 * We disallow kprobes in ___slab_alloc() to prevent reentrance 4822 * 4823 * kmalloc() -> ___slab_alloc() -> local_lock_cpu_slab() protected part of 4824 * ___slab_alloc() manipulating c->freelist -> kprobe -> bpf -> 4825 * kmalloc_nolock() or kfree_nolock() -> __update_cpu_freelist_fast() 4826 * manipulating c->freelist without lock. 4827 * 4828 * This does not prevent kprobe in functions called from ___slab_alloc() such as 4829 * local_lock_irqsave() itself, and that is fine, we only need to protect the 4830 * c->freelist manipulation in ___slab_alloc() itself. 4831 */ 4832 NOKPROBE_SYMBOL(___slab_alloc); 4833 4834 /* 4835 * A wrapper for ___slab_alloc() for contexts where preemption is not yet 4836 * disabled. Compensates for possible cpu changes by refetching the per cpu area 4837 * pointer. 4838 */ 4839 static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, 4840 unsigned long addr, struct kmem_cache_cpu *c, unsigned int orig_size) 4841 { 4842 void *p; 4843 4844 #ifdef CONFIG_PREEMPT_COUNT 4845 /* 4846 * We may have been preempted and rescheduled on a different 4847 * cpu before disabling preemption. Need to reload cpu area 4848 * pointer. 4849 */ 4850 c = slub_get_cpu_ptr(s->cpu_slab); 4851 #endif 4852 if (unlikely(!gfpflags_allow_spinning(gfpflags))) { 4853 if (local_lock_is_locked(&s->cpu_slab->lock)) { 4854 /* 4855 * EBUSY is an internal signal to kmalloc_nolock() to 4856 * retry a different bucket. It's not propagated 4857 * to the caller. 4858 */ 4859 p = ERR_PTR(-EBUSY); 4860 goto out; 4861 } 4862 } 4863 p = ___slab_alloc(s, gfpflags, node, addr, c, orig_size); 4864 out: 4865 #ifdef CONFIG_PREEMPT_COUNT 4866 slub_put_cpu_ptr(s->cpu_slab); 4867 #endif 4868 return p; 4869 } 4870 4871 static __always_inline void *__slab_alloc_node(struct kmem_cache *s, 4872 gfp_t gfpflags, int node, unsigned long addr, size_t orig_size) 4873 { 4874 struct kmem_cache_cpu *c; 4875 struct slab *slab; 4876 unsigned long tid; 4877 void *object; 4878 4879 redo: 4880 /* 4881 * Must read kmem_cache cpu data via this cpu ptr. Preemption is 4882 * enabled. We may switch back and forth between cpus while 4883 * reading from one cpu area. That does not matter as long 4884 * as we end up on the original cpu again when doing the cmpxchg. 4885 * 4886 * We must guarantee that tid and kmem_cache_cpu are retrieved on the 4887 * same cpu. We read first the kmem_cache_cpu pointer and use it to read 4888 * the tid. If we are preempted and switched to another cpu between the 4889 * two reads, it's OK as the two are still associated with the same cpu 4890 * and cmpxchg later will validate the cpu. 4891 */ 4892 c = raw_cpu_ptr(s->cpu_slab); 4893 tid = READ_ONCE(c->tid); 4894 4895 /* 4896 * Irqless object alloc/free algorithm used here depends on sequence 4897 * of fetching cpu_slab's data. tid should be fetched before anything 4898 * on c to guarantee that object and slab associated with previous tid 4899 * won't be used with current tid. If we fetch tid first, object and 4900 * slab could be one associated with next tid and our alloc/free 4901 * request will be failed. In this case, we will retry. So, no problem. 4902 */ 4903 barrier(); 4904 4905 /* 4906 * The transaction ids are globally unique per cpu and per operation on 4907 * a per cpu queue. Thus they can be guarantee that the cmpxchg_double 4908 * occurs on the right processor and that there was no operation on the 4909 * linked list in between. 4910 */ 4911 4912 object = c->freelist; 4913 slab = c->slab; 4914 4915 #ifdef CONFIG_NUMA 4916 if (static_branch_unlikely(&strict_numa) && 4917 node == NUMA_NO_NODE) { 4918 4919 struct mempolicy *mpol = current->mempolicy; 4920 4921 if (mpol) { 4922 /* 4923 * Special BIND rule support. If existing slab 4924 * is in permitted set then do not redirect 4925 * to a particular node. 4926 * Otherwise we apply the memory policy to get 4927 * the node we need to allocate on. 4928 */ 4929 if (mpol->mode != MPOL_BIND || !slab || 4930 !node_isset(slab_nid(slab), mpol->nodes)) 4931 4932 node = mempolicy_slab_node(); 4933 } 4934 } 4935 #endif 4936 4937 if (!USE_LOCKLESS_FAST_PATH() || 4938 unlikely(!object || !slab || !node_match(slab, node))) { 4939 object = __slab_alloc(s, gfpflags, node, addr, c, orig_size); 4940 } else { 4941 void *next_object = get_freepointer_safe(s, object); 4942 4943 /* 4944 * The cmpxchg will only match if there was no additional 4945 * operation and if we are on the right processor. 4946 * 4947 * The cmpxchg does the following atomically (without lock 4948 * semantics!) 4949 * 1. Relocate first pointer to the current per cpu area. 4950 * 2. Verify that tid and freelist have not been changed 4951 * 3. If they were not changed replace tid and freelist 4952 * 4953 * Since this is without lock semantics the protection is only 4954 * against code executing on this cpu *not* from access by 4955 * other cpus. 4956 */ 4957 if (unlikely(!__update_cpu_freelist_fast(s, object, next_object, tid))) { 4958 note_cmpxchg_failure("slab_alloc", s, tid); 4959 goto redo; 4960 } 4961 prefetch_freepointer(s, next_object); 4962 stat(s, ALLOC_FASTPATH); 4963 } 4964 4965 return object; 4966 } 4967 4968 /* 4969 * If the object has been wiped upon free, make sure it's fully initialized by 4970 * zeroing out freelist pointer. 4971 * 4972 * Note that we also wipe custom freelist pointers. 4973 */ 4974 static __always_inline void maybe_wipe_obj_freeptr(struct kmem_cache *s, 4975 void *obj) 4976 { 4977 if (unlikely(slab_want_init_on_free(s)) && obj && 4978 !freeptr_outside_object(s)) 4979 memset((void *)((char *)kasan_reset_tag(obj) + s->offset), 4980 0, sizeof(void *)); 4981 } 4982 4983 static __fastpath_inline 4984 struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags) 4985 { 4986 flags &= gfp_allowed_mask; 4987 4988 might_alloc(flags); 4989 4990 if (unlikely(should_failslab(s, flags))) 4991 return NULL; 4992 4993 return s; 4994 } 4995 4996 static __fastpath_inline 4997 bool slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, 4998 gfp_t flags, size_t size, void **p, bool init, 4999 unsigned int orig_size) 5000 { 5001 unsigned int zero_size = s->object_size; 5002 bool kasan_init = init; 5003 size_t i; 5004 gfp_t init_flags = flags & gfp_allowed_mask; 5005 5006 /* 5007 * For kmalloc object, the allocated memory size(object_size) is likely 5008 * larger than the requested size(orig_size). If redzone check is 5009 * enabled for the extra space, don't zero it, as it will be redzoned 5010 * soon. The redzone operation for this extra space could be seen as a 5011 * replacement of current poisoning under certain debug option, and 5012 * won't break other sanity checks. 5013 */ 5014 if (kmem_cache_debug_flags(s, SLAB_STORE_USER | SLAB_RED_ZONE) && 5015 (s->flags & SLAB_KMALLOC)) 5016 zero_size = orig_size; 5017 5018 /* 5019 * When slab_debug is enabled, avoid memory initialization integrated 5020 * into KASAN and instead zero out the memory via the memset below with 5021 * the proper size. Otherwise, KASAN might overwrite SLUB redzones and 5022 * cause false-positive reports. This does not lead to a performance 5023 * penalty on production builds, as slab_debug is not intended to be 5024 * enabled there. 5025 */ 5026 if (__slub_debug_enabled()) 5027 kasan_init = false; 5028 5029 /* 5030 * As memory initialization might be integrated into KASAN, 5031 * kasan_slab_alloc and initialization memset must be 5032 * kept together to avoid discrepancies in behavior. 5033 * 5034 * As p[i] might get tagged, memset and kmemleak hook come after KASAN. 5035 */ 5036 for (i = 0; i < size; i++) { 5037 p[i] = kasan_slab_alloc(s, p[i], init_flags, kasan_init); 5038 if (p[i] && init && (!kasan_init || 5039 !kasan_has_integrated_init())) 5040 memset(p[i], 0, zero_size); 5041 if (gfpflags_allow_spinning(flags)) 5042 kmemleak_alloc_recursive(p[i], s->object_size, 1, 5043 s->flags, init_flags); 5044 kmsan_slab_alloc(s, p[i], init_flags); 5045 alloc_tagging_slab_alloc_hook(s, p[i], flags); 5046 } 5047 5048 return memcg_slab_post_alloc_hook(s, lru, flags, size, p); 5049 } 5050 5051 /* 5052 * Replace the empty main sheaf with a (at least partially) full sheaf. 5053 * 5054 * Must be called with the cpu_sheaves local lock locked. If successful, returns 5055 * the pcs pointer and the local lock locked (possibly on a different cpu than 5056 * initially called). If not successful, returns NULL and the local lock 5057 * unlocked. 5058 */ 5059 static struct slub_percpu_sheaves * 5060 __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, gfp_t gfp) 5061 { 5062 struct slab_sheaf *empty = NULL; 5063 struct slab_sheaf *full; 5064 struct node_barn *barn; 5065 bool can_alloc; 5066 5067 lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); 5068 5069 if (pcs->spare && pcs->spare->size > 0) { 5070 swap(pcs->main, pcs->spare); 5071 return pcs; 5072 } 5073 5074 barn = get_barn(s); 5075 if (!barn) { 5076 local_unlock(&s->cpu_sheaves->lock); 5077 return NULL; 5078 } 5079 5080 full = barn_replace_empty_sheaf(barn, pcs->main); 5081 5082 if (full) { 5083 stat(s, BARN_GET); 5084 pcs->main = full; 5085 return pcs; 5086 } 5087 5088 stat(s, BARN_GET_FAIL); 5089 5090 can_alloc = gfpflags_allow_blocking(gfp); 5091 5092 if (can_alloc) { 5093 if (pcs->spare) { 5094 empty = pcs->spare; 5095 pcs->spare = NULL; 5096 } else { 5097 empty = barn_get_empty_sheaf(barn); 5098 } 5099 } 5100 5101 local_unlock(&s->cpu_sheaves->lock); 5102 5103 if (!can_alloc) 5104 return NULL; 5105 5106 if (empty) { 5107 if (!refill_sheaf(s, empty, gfp)) { 5108 full = empty; 5109 } else { 5110 /* 5111 * we must be very low on memory so don't bother 5112 * with the barn 5113 */ 5114 free_empty_sheaf(s, empty); 5115 } 5116 } else { 5117 full = alloc_full_sheaf(s, gfp); 5118 } 5119 5120 if (!full) 5121 return NULL; 5122 5123 /* 5124 * we can reach here only when gfpflags_allow_blocking 5125 * so this must not be an irq 5126 */ 5127 local_lock(&s->cpu_sheaves->lock); 5128 pcs = this_cpu_ptr(s->cpu_sheaves); 5129 5130 /* 5131 * If we are returning empty sheaf, we either got it from the 5132 * barn or had to allocate one. If we are returning a full 5133 * sheaf, it's due to racing or being migrated to a different 5134 * cpu. Breaching the barn's sheaf limits should be thus rare 5135 * enough so just ignore them to simplify the recovery. 5136 */ 5137 5138 if (pcs->main->size == 0) { 5139 barn_put_empty_sheaf(barn, pcs->main); 5140 pcs->main = full; 5141 return pcs; 5142 } 5143 5144 if (!pcs->spare) { 5145 pcs->spare = full; 5146 return pcs; 5147 } 5148 5149 if (pcs->spare->size == 0) { 5150 barn_put_empty_sheaf(barn, pcs->spare); 5151 pcs->spare = full; 5152 return pcs; 5153 } 5154 5155 barn_put_full_sheaf(barn, full); 5156 stat(s, BARN_PUT); 5157 5158 return pcs; 5159 } 5160 5161 static __fastpath_inline 5162 void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, int node) 5163 { 5164 struct slub_percpu_sheaves *pcs; 5165 bool node_requested; 5166 void *object; 5167 5168 #ifdef CONFIG_NUMA 5169 if (static_branch_unlikely(&strict_numa) && 5170 node == NUMA_NO_NODE) { 5171 5172 struct mempolicy *mpol = current->mempolicy; 5173 5174 if (mpol) { 5175 /* 5176 * Special BIND rule support. If the local node 5177 * is in permitted set then do not redirect 5178 * to a particular node. 5179 * Otherwise we apply the memory policy to get 5180 * the node we need to allocate on. 5181 */ 5182 if (mpol->mode != MPOL_BIND || 5183 !node_isset(numa_mem_id(), mpol->nodes)) 5184 5185 node = mempolicy_slab_node(); 5186 } 5187 } 5188 #endif 5189 5190 node_requested = IS_ENABLED(CONFIG_NUMA) && node != NUMA_NO_NODE; 5191 5192 /* 5193 * We assume the percpu sheaves contain only local objects although it's 5194 * not completely guaranteed, so we verify later. 5195 */ 5196 if (unlikely(node_requested && node != numa_mem_id())) 5197 return NULL; 5198 5199 if (!local_trylock(&s->cpu_sheaves->lock)) 5200 return NULL; 5201 5202 pcs = this_cpu_ptr(s->cpu_sheaves); 5203 5204 if (unlikely(pcs->main->size == 0)) { 5205 pcs = __pcs_replace_empty_main(s, pcs, gfp); 5206 if (unlikely(!pcs)) 5207 return NULL; 5208 } 5209 5210 object = pcs->main->objects[pcs->main->size - 1]; 5211 5212 if (unlikely(node_requested)) { 5213 /* 5214 * Verify that the object was from the node we want. This could 5215 * be false because of cpu migration during an unlocked part of 5216 * the current allocation or previous freeing process. 5217 */ 5218 if (folio_nid(virt_to_folio(object)) != node) { 5219 local_unlock(&s->cpu_sheaves->lock); 5220 return NULL; 5221 } 5222 } 5223 5224 pcs->main->size--; 5225 5226 local_unlock(&s->cpu_sheaves->lock); 5227 5228 stat(s, ALLOC_PCS); 5229 5230 return object; 5231 } 5232 5233 static __fastpath_inline 5234 unsigned int alloc_from_pcs_bulk(struct kmem_cache *s, size_t size, void **p) 5235 { 5236 struct slub_percpu_sheaves *pcs; 5237 struct slab_sheaf *main; 5238 unsigned int allocated = 0; 5239 unsigned int batch; 5240 5241 next_batch: 5242 if (!local_trylock(&s->cpu_sheaves->lock)) 5243 return allocated; 5244 5245 pcs = this_cpu_ptr(s->cpu_sheaves); 5246 5247 if (unlikely(pcs->main->size == 0)) { 5248 5249 struct slab_sheaf *full; 5250 struct node_barn *barn; 5251 5252 if (pcs->spare && pcs->spare->size > 0) { 5253 swap(pcs->main, pcs->spare); 5254 goto do_alloc; 5255 } 5256 5257 barn = get_barn(s); 5258 if (!barn) { 5259 local_unlock(&s->cpu_sheaves->lock); 5260 return allocated; 5261 } 5262 5263 full = barn_replace_empty_sheaf(barn, pcs->main); 5264 5265 if (full) { 5266 stat(s, BARN_GET); 5267 pcs->main = full; 5268 goto do_alloc; 5269 } 5270 5271 stat(s, BARN_GET_FAIL); 5272 5273 local_unlock(&s->cpu_sheaves->lock); 5274 5275 /* 5276 * Once full sheaves in barn are depleted, let the bulk 5277 * allocation continue from slab pages, otherwise we would just 5278 * be copying arrays of pointers twice. 5279 */ 5280 return allocated; 5281 } 5282 5283 do_alloc: 5284 5285 main = pcs->main; 5286 batch = min(size, main->size); 5287 5288 main->size -= batch; 5289 memcpy(p, main->objects + main->size, batch * sizeof(void *)); 5290 5291 local_unlock(&s->cpu_sheaves->lock); 5292 5293 stat_add(s, ALLOC_PCS, batch); 5294 5295 allocated += batch; 5296 5297 if (batch < size) { 5298 p += batch; 5299 size -= batch; 5300 goto next_batch; 5301 } 5302 5303 return allocated; 5304 } 5305 5306 5307 /* 5308 * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc) 5309 * have the fastpath folded into their functions. So no function call 5310 * overhead for requests that can be satisfied on the fastpath. 5311 * 5312 * The fastpath works by first checking if the lockless freelist can be used. 5313 * If not then __slab_alloc is called for slow processing. 5314 * 5315 * Otherwise we can simply pick the next object from the lockless free list. 5316 */ 5317 static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list_lru *lru, 5318 gfp_t gfpflags, int node, unsigned long addr, size_t orig_size) 5319 { 5320 void *object; 5321 bool init = false; 5322 5323 s = slab_pre_alloc_hook(s, gfpflags); 5324 if (unlikely(!s)) 5325 return NULL; 5326 5327 object = kfence_alloc(s, orig_size, gfpflags); 5328 if (unlikely(object)) 5329 goto out; 5330 5331 if (s->cpu_sheaves) 5332 object = alloc_from_pcs(s, gfpflags, node); 5333 5334 if (!object) 5335 object = __slab_alloc_node(s, gfpflags, node, addr, orig_size); 5336 5337 maybe_wipe_obj_freeptr(s, object); 5338 init = slab_want_init_on_alloc(gfpflags, s); 5339 5340 out: 5341 /* 5342 * When init equals 'true', like for kzalloc() family, only 5343 * @orig_size bytes might be zeroed instead of s->object_size 5344 * In case this fails due to memcg_slab_post_alloc_hook(), 5345 * object is set to NULL 5346 */ 5347 slab_post_alloc_hook(s, lru, gfpflags, 1, &object, init, orig_size); 5348 5349 return object; 5350 } 5351 5352 void *kmem_cache_alloc_noprof(struct kmem_cache *s, gfp_t gfpflags) 5353 { 5354 void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE, _RET_IP_, 5355 s->object_size); 5356 5357 trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, NUMA_NO_NODE); 5358 5359 return ret; 5360 } 5361 EXPORT_SYMBOL(kmem_cache_alloc_noprof); 5362 5363 void *kmem_cache_alloc_lru_noprof(struct kmem_cache *s, struct list_lru *lru, 5364 gfp_t gfpflags) 5365 { 5366 void *ret = slab_alloc_node(s, lru, gfpflags, NUMA_NO_NODE, _RET_IP_, 5367 s->object_size); 5368 5369 trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, NUMA_NO_NODE); 5370 5371 return ret; 5372 } 5373 EXPORT_SYMBOL(kmem_cache_alloc_lru_noprof); 5374 5375 bool kmem_cache_charge(void *objp, gfp_t gfpflags) 5376 { 5377 if (!memcg_kmem_online()) 5378 return true; 5379 5380 return memcg_slab_post_charge(objp, gfpflags); 5381 } 5382 EXPORT_SYMBOL(kmem_cache_charge); 5383 5384 /** 5385 * kmem_cache_alloc_node - Allocate an object on the specified node 5386 * @s: The cache to allocate from. 5387 * @gfpflags: See kmalloc(). 5388 * @node: node number of the target node. 5389 * 5390 * Identical to kmem_cache_alloc but it will allocate memory on the given 5391 * node, which can improve the performance for cpu bound structures. 5392 * 5393 * Fallback to other node is possible if __GFP_THISNODE is not set. 5394 * 5395 * Return: pointer to the new object or %NULL in case of error 5396 */ 5397 void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t gfpflags, int node) 5398 { 5399 void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, s->object_size); 5400 5401 trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, node); 5402 5403 return ret; 5404 } 5405 EXPORT_SYMBOL(kmem_cache_alloc_node_noprof); 5406 5407 /* 5408 * returns a sheaf that has at least the requested size 5409 * when prefilling is needed, do so with given gfp flags 5410 * 5411 * return NULL if sheaf allocation or prefilling failed 5412 */ 5413 struct slab_sheaf * 5414 kmem_cache_prefill_sheaf(struct kmem_cache *s, gfp_t gfp, unsigned int size) 5415 { 5416 struct slub_percpu_sheaves *pcs; 5417 struct slab_sheaf *sheaf = NULL; 5418 struct node_barn *barn; 5419 5420 if (unlikely(size > s->sheaf_capacity)) { 5421 5422 /* 5423 * slab_debug disables cpu sheaves intentionally so all 5424 * prefilled sheaves become "oversize" and we give up on 5425 * performance for the debugging. Same with SLUB_TINY. 5426 * Creating a cache without sheaves and then requesting a 5427 * prefilled sheaf is however not expected, so warn. 5428 */ 5429 WARN_ON_ONCE(s->sheaf_capacity == 0 && 5430 !IS_ENABLED(CONFIG_SLUB_TINY) && 5431 !(s->flags & SLAB_DEBUG_FLAGS)); 5432 5433 sheaf = kzalloc(struct_size(sheaf, objects, size), gfp); 5434 if (!sheaf) 5435 return NULL; 5436 5437 stat(s, SHEAF_PREFILL_OVERSIZE); 5438 sheaf->cache = s; 5439 sheaf->capacity = size; 5440 5441 if (!__kmem_cache_alloc_bulk(s, gfp, size, 5442 &sheaf->objects[0])) { 5443 kfree(sheaf); 5444 return NULL; 5445 } 5446 5447 sheaf->size = size; 5448 5449 return sheaf; 5450 } 5451 5452 local_lock(&s->cpu_sheaves->lock); 5453 pcs = this_cpu_ptr(s->cpu_sheaves); 5454 5455 if (pcs->spare) { 5456 sheaf = pcs->spare; 5457 pcs->spare = NULL; 5458 stat(s, SHEAF_PREFILL_FAST); 5459 } else { 5460 barn = get_barn(s); 5461 5462 stat(s, SHEAF_PREFILL_SLOW); 5463 if (barn) 5464 sheaf = barn_get_full_or_empty_sheaf(barn); 5465 if (sheaf && sheaf->size) 5466 stat(s, BARN_GET); 5467 else 5468 stat(s, BARN_GET_FAIL); 5469 } 5470 5471 local_unlock(&s->cpu_sheaves->lock); 5472 5473 5474 if (!sheaf) 5475 sheaf = alloc_empty_sheaf(s, gfp); 5476 5477 if (sheaf && sheaf->size < size) { 5478 if (refill_sheaf(s, sheaf, gfp)) { 5479 sheaf_flush_unused(s, sheaf); 5480 free_empty_sheaf(s, sheaf); 5481 sheaf = NULL; 5482 } 5483 } 5484 5485 if (sheaf) 5486 sheaf->capacity = s->sheaf_capacity; 5487 5488 return sheaf; 5489 } 5490 5491 /* 5492 * Use this to return a sheaf obtained by kmem_cache_prefill_sheaf() 5493 * 5494 * If the sheaf cannot simply become the percpu spare sheaf, but there's space 5495 * for a full sheaf in the barn, we try to refill the sheaf back to the cache's 5496 * sheaf_capacity to avoid handling partially full sheaves. 5497 * 5498 * If the refill fails because gfp is e.g. GFP_NOWAIT, or the barn is full, the 5499 * sheaf is instead flushed and freed. 5500 */ 5501 void kmem_cache_return_sheaf(struct kmem_cache *s, gfp_t gfp, 5502 struct slab_sheaf *sheaf) 5503 { 5504 struct slub_percpu_sheaves *pcs; 5505 struct node_barn *barn; 5506 5507 if (unlikely(sheaf->capacity != s->sheaf_capacity)) { 5508 sheaf_flush_unused(s, sheaf); 5509 kfree(sheaf); 5510 return; 5511 } 5512 5513 local_lock(&s->cpu_sheaves->lock); 5514 pcs = this_cpu_ptr(s->cpu_sheaves); 5515 barn = get_barn(s); 5516 5517 if (!pcs->spare) { 5518 pcs->spare = sheaf; 5519 sheaf = NULL; 5520 stat(s, SHEAF_RETURN_FAST); 5521 } 5522 5523 local_unlock(&s->cpu_sheaves->lock); 5524 5525 if (!sheaf) 5526 return; 5527 5528 stat(s, SHEAF_RETURN_SLOW); 5529 5530 /* 5531 * If the barn has too many full sheaves or we fail to refill the sheaf, 5532 * simply flush and free it. 5533 */ 5534 if (!barn || data_race(barn->nr_full) >= MAX_FULL_SHEAVES || 5535 refill_sheaf(s, sheaf, gfp)) { 5536 sheaf_flush_unused(s, sheaf); 5537 free_empty_sheaf(s, sheaf); 5538 return; 5539 } 5540 5541 barn_put_full_sheaf(barn, sheaf); 5542 stat(s, BARN_PUT); 5543 } 5544 5545 /* 5546 * refill a sheaf previously returned by kmem_cache_prefill_sheaf to at least 5547 * the given size 5548 * 5549 * the sheaf might be replaced by a new one when requesting more than 5550 * s->sheaf_capacity objects if such replacement is necessary, but the refill 5551 * fails (returning -ENOMEM), the existing sheaf is left intact 5552 * 5553 * In practice we always refill to full sheaf's capacity. 5554 */ 5555 int kmem_cache_refill_sheaf(struct kmem_cache *s, gfp_t gfp, 5556 struct slab_sheaf **sheafp, unsigned int size) 5557 { 5558 struct slab_sheaf *sheaf; 5559 5560 /* 5561 * TODO: do we want to support *sheaf == NULL to be equivalent of 5562 * kmem_cache_prefill_sheaf() ? 5563 */ 5564 if (!sheafp || !(*sheafp)) 5565 return -EINVAL; 5566 5567 sheaf = *sheafp; 5568 if (sheaf->size >= size) 5569 return 0; 5570 5571 if (likely(sheaf->capacity >= size)) { 5572 if (likely(sheaf->capacity == s->sheaf_capacity)) 5573 return refill_sheaf(s, sheaf, gfp); 5574 5575 if (!__kmem_cache_alloc_bulk(s, gfp, sheaf->capacity - sheaf->size, 5576 &sheaf->objects[sheaf->size])) { 5577 return -ENOMEM; 5578 } 5579 sheaf->size = sheaf->capacity; 5580 5581 return 0; 5582 } 5583 5584 /* 5585 * We had a regular sized sheaf and need an oversize one, or we had an 5586 * oversize one already but need a larger one now. 5587 * This should be a very rare path so let's not complicate it. 5588 */ 5589 sheaf = kmem_cache_prefill_sheaf(s, gfp, size); 5590 if (!sheaf) 5591 return -ENOMEM; 5592 5593 kmem_cache_return_sheaf(s, gfp, *sheafp); 5594 *sheafp = sheaf; 5595 return 0; 5596 } 5597 5598 /* 5599 * Allocate from a sheaf obtained by kmem_cache_prefill_sheaf() 5600 * 5601 * Guaranteed not to fail as many allocations as was the requested size. 5602 * After the sheaf is emptied, it fails - no fallback to the slab cache itself. 5603 * 5604 * The gfp parameter is meant only to specify __GFP_ZERO or __GFP_ACCOUNT 5605 * memcg charging is forced over limit if necessary, to avoid failure. 5606 */ 5607 void * 5608 kmem_cache_alloc_from_sheaf_noprof(struct kmem_cache *s, gfp_t gfp, 5609 struct slab_sheaf *sheaf) 5610 { 5611 void *ret = NULL; 5612 bool init; 5613 5614 if (sheaf->size == 0) 5615 goto out; 5616 5617 ret = sheaf->objects[--sheaf->size]; 5618 5619 init = slab_want_init_on_alloc(gfp, s); 5620 5621 /* add __GFP_NOFAIL to force successful memcg charging */ 5622 slab_post_alloc_hook(s, NULL, gfp | __GFP_NOFAIL, 1, &ret, init, s->object_size); 5623 out: 5624 trace_kmem_cache_alloc(_RET_IP_, ret, s, gfp, NUMA_NO_NODE); 5625 5626 return ret; 5627 } 5628 5629 unsigned int kmem_cache_sheaf_size(struct slab_sheaf *sheaf) 5630 { 5631 return sheaf->size; 5632 } 5633 /* 5634 * To avoid unnecessary overhead, we pass through large allocation requests 5635 * directly to the page allocator. We use __GFP_COMP, because we will need to 5636 * know the allocation order to free the pages properly in kfree. 5637 */ 5638 static void *___kmalloc_large_node(size_t size, gfp_t flags, int node) 5639 { 5640 struct folio *folio; 5641 void *ptr = NULL; 5642 unsigned int order = get_order(size); 5643 5644 if (unlikely(flags & GFP_SLAB_BUG_MASK)) 5645 flags = kmalloc_fix_flags(flags); 5646 5647 flags |= __GFP_COMP; 5648 5649 if (node == NUMA_NO_NODE) 5650 folio = (struct folio *)alloc_frozen_pages_noprof(flags, order); 5651 else 5652 folio = (struct folio *)__alloc_frozen_pages_noprof(flags, order, node, NULL); 5653 5654 if (folio) { 5655 ptr = folio_address(folio); 5656 lruvec_stat_mod_folio(folio, NR_SLAB_UNRECLAIMABLE_B, 5657 PAGE_SIZE << order); 5658 __folio_set_large_kmalloc(folio); 5659 } 5660 5661 ptr = kasan_kmalloc_large(ptr, size, flags); 5662 /* As ptr might get tagged, call kmemleak hook after KASAN. */ 5663 kmemleak_alloc(ptr, size, 1, flags); 5664 kmsan_kmalloc_large(ptr, size, flags); 5665 5666 return ptr; 5667 } 5668 5669 void *__kmalloc_large_noprof(size_t size, gfp_t flags) 5670 { 5671 void *ret = ___kmalloc_large_node(size, flags, NUMA_NO_NODE); 5672 5673 trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << get_order(size), 5674 flags, NUMA_NO_NODE); 5675 return ret; 5676 } 5677 EXPORT_SYMBOL(__kmalloc_large_noprof); 5678 5679 void *__kmalloc_large_node_noprof(size_t size, gfp_t flags, int node) 5680 { 5681 void *ret = ___kmalloc_large_node(size, flags, node); 5682 5683 trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << get_order(size), 5684 flags, node); 5685 return ret; 5686 } 5687 EXPORT_SYMBOL(__kmalloc_large_node_noprof); 5688 5689 static __always_inline 5690 void *__do_kmalloc_node(size_t size, kmem_buckets *b, gfp_t flags, int node, 5691 unsigned long caller) 5692 { 5693 struct kmem_cache *s; 5694 void *ret; 5695 5696 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) { 5697 ret = __kmalloc_large_node_noprof(size, flags, node); 5698 trace_kmalloc(caller, ret, size, 5699 PAGE_SIZE << get_order(size), flags, node); 5700 return ret; 5701 } 5702 5703 if (unlikely(!size)) 5704 return ZERO_SIZE_PTR; 5705 5706 s = kmalloc_slab(size, b, flags, caller); 5707 5708 ret = slab_alloc_node(s, NULL, flags, node, caller, size); 5709 ret = kasan_kmalloc(s, ret, size, flags); 5710 trace_kmalloc(caller, ret, size, s->size, flags, node); 5711 return ret; 5712 } 5713 void *__kmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node) 5714 { 5715 return __do_kmalloc_node(size, PASS_BUCKET_PARAM(b), flags, node, _RET_IP_); 5716 } 5717 EXPORT_SYMBOL(__kmalloc_node_noprof); 5718 5719 void *__kmalloc_noprof(size_t size, gfp_t flags) 5720 { 5721 return __do_kmalloc_node(size, NULL, flags, NUMA_NO_NODE, _RET_IP_); 5722 } 5723 EXPORT_SYMBOL(__kmalloc_noprof); 5724 5725 /** 5726 * kmalloc_nolock - Allocate an object of given size from any context. 5727 * @size: size to allocate 5728 * @gfp_flags: GFP flags. Only __GFP_ACCOUNT, __GFP_ZERO, __GFP_NO_OBJ_EXT 5729 * allowed. 5730 * @node: node number of the target node. 5731 * 5732 * Return: pointer to the new object or NULL in case of error. 5733 * NULL does not mean EBUSY or EAGAIN. It means ENOMEM. 5734 * There is no reason to call it again and expect !NULL. 5735 */ 5736 void *kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node) 5737 { 5738 gfp_t alloc_gfp = __GFP_NOWARN | __GFP_NOMEMALLOC | gfp_flags; 5739 struct kmem_cache *s; 5740 bool can_retry = true; 5741 void *ret = ERR_PTR(-EBUSY); 5742 5743 VM_WARN_ON_ONCE(gfp_flags & ~(__GFP_ACCOUNT | __GFP_ZERO | 5744 __GFP_NO_OBJ_EXT)); 5745 5746 if (unlikely(!size)) 5747 return ZERO_SIZE_PTR; 5748 5749 if (IS_ENABLED(CONFIG_PREEMPT_RT) && !preemptible()) 5750 /* 5751 * kmalloc_nolock() in PREEMPT_RT is not supported from 5752 * non-preemptible context because local_lock becomes a 5753 * sleeping lock on RT. 5754 */ 5755 return NULL; 5756 5757 /* On UP, spin_trylock() always succeeds even when it is locked */ 5758 if (!IS_ENABLED(CONFIG_SMP) && in_nmi()) 5759 return NULL; 5760 5761 retry: 5762 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) 5763 return NULL; 5764 s = kmalloc_slab(size, NULL, alloc_gfp, _RET_IP_); 5765 5766 if (!(s->flags & __CMPXCHG_DOUBLE) && !kmem_cache_debug(s)) 5767 /* 5768 * kmalloc_nolock() is not supported on architectures that 5769 * don't implement cmpxchg16b, but debug caches don't use 5770 * per-cpu slab and per-cpu partial slabs. They rely on 5771 * kmem_cache_node->list_lock, so kmalloc_nolock() can 5772 * attempt to allocate from debug caches by 5773 * spin_trylock_irqsave(&n->list_lock, ...) 5774 */ 5775 return NULL; 5776 5777 /* 5778 * Do not call slab_alloc_node(), since trylock mode isn't 5779 * compatible with slab_pre_alloc_hook/should_failslab and 5780 * kfence_alloc. Hence call __slab_alloc_node() (at most twice) 5781 * and slab_post_alloc_hook() directly. 5782 * 5783 * In !PREEMPT_RT ___slab_alloc() manipulates (freelist,tid) pair 5784 * in irq saved region. It assumes that the same cpu will not 5785 * __update_cpu_freelist_fast() into the same (freelist,tid) pair. 5786 * Therefore use in_nmi() to check whether particular bucket is in 5787 * irq protected section. 5788 * 5789 * If in_nmi() && local_lock_is_locked(s->cpu_slab) then it means that 5790 * this cpu was interrupted somewhere inside ___slab_alloc() after 5791 * it did local_lock_irqsave(&s->cpu_slab->lock, flags). 5792 * In this case fast path with __update_cpu_freelist_fast() is not safe. 5793 */ 5794 if (!in_nmi() || !local_lock_is_locked(&s->cpu_slab->lock)) 5795 ret = __slab_alloc_node(s, alloc_gfp, node, _RET_IP_, size); 5796 5797 if (PTR_ERR(ret) == -EBUSY) { 5798 if (can_retry) { 5799 /* pick the next kmalloc bucket */ 5800 size = s->object_size + 1; 5801 /* 5802 * Another alternative is to 5803 * if (memcg) alloc_gfp &= ~__GFP_ACCOUNT; 5804 * else if (!memcg) alloc_gfp |= __GFP_ACCOUNT; 5805 * to retry from bucket of the same size. 5806 */ 5807 can_retry = false; 5808 goto retry; 5809 } 5810 ret = NULL; 5811 } 5812 5813 maybe_wipe_obj_freeptr(s, ret); 5814 slab_post_alloc_hook(s, NULL, alloc_gfp, 1, &ret, 5815 slab_want_init_on_alloc(alloc_gfp, s), size); 5816 5817 ret = kasan_kmalloc(s, ret, size, alloc_gfp); 5818 return ret; 5819 } 5820 EXPORT_SYMBOL_GPL(kmalloc_nolock_noprof); 5821 5822 void *__kmalloc_node_track_caller_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, 5823 int node, unsigned long caller) 5824 { 5825 return __do_kmalloc_node(size, PASS_BUCKET_PARAM(b), flags, node, caller); 5826 5827 } 5828 EXPORT_SYMBOL(__kmalloc_node_track_caller_noprof); 5829 5830 void *__kmalloc_cache_noprof(struct kmem_cache *s, gfp_t gfpflags, size_t size) 5831 { 5832 void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE, 5833 _RET_IP_, size); 5834 5835 trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, NUMA_NO_NODE); 5836 5837 ret = kasan_kmalloc(s, ret, size, gfpflags); 5838 return ret; 5839 } 5840 EXPORT_SYMBOL(__kmalloc_cache_noprof); 5841 5842 void *__kmalloc_cache_node_noprof(struct kmem_cache *s, gfp_t gfpflags, 5843 int node, size_t size) 5844 { 5845 void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, size); 5846 5847 trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, node); 5848 5849 ret = kasan_kmalloc(s, ret, size, gfpflags); 5850 return ret; 5851 } 5852 EXPORT_SYMBOL(__kmalloc_cache_node_noprof); 5853 5854 static noinline void free_to_partial_list( 5855 struct kmem_cache *s, struct slab *slab, 5856 void *head, void *tail, int bulk_cnt, 5857 unsigned long addr) 5858 { 5859 struct kmem_cache_node *n = get_node(s, slab_nid(slab)); 5860 struct slab *slab_free = NULL; 5861 int cnt = bulk_cnt; 5862 unsigned long flags; 5863 depot_stack_handle_t handle = 0; 5864 5865 /* 5866 * We cannot use GFP_NOWAIT as there are callsites where waking up 5867 * kswapd could deadlock 5868 */ 5869 if (s->flags & SLAB_STORE_USER) 5870 handle = set_track_prepare(__GFP_NOWARN); 5871 5872 spin_lock_irqsave(&n->list_lock, flags); 5873 5874 if (free_debug_processing(s, slab, head, tail, &cnt, addr, handle)) { 5875 void *prior = slab->freelist; 5876 5877 /* Perform the actual freeing while we still hold the locks */ 5878 slab->inuse -= cnt; 5879 set_freepointer(s, tail, prior); 5880 slab->freelist = head; 5881 5882 /* 5883 * If the slab is empty, and node's partial list is full, 5884 * it should be discarded anyway no matter it's on full or 5885 * partial list. 5886 */ 5887 if (slab->inuse == 0 && n->nr_partial >= s->min_partial) 5888 slab_free = slab; 5889 5890 if (!prior) { 5891 /* was on full list */ 5892 remove_full(s, n, slab); 5893 if (!slab_free) { 5894 add_partial(n, slab, DEACTIVATE_TO_TAIL); 5895 stat(s, FREE_ADD_PARTIAL); 5896 } 5897 } else if (slab_free) { 5898 remove_partial(n, slab); 5899 stat(s, FREE_REMOVE_PARTIAL); 5900 } 5901 } 5902 5903 if (slab_free) { 5904 /* 5905 * Update the counters while still holding n->list_lock to 5906 * prevent spurious validation warnings 5907 */ 5908 dec_slabs_node(s, slab_nid(slab_free), slab_free->objects); 5909 } 5910 5911 spin_unlock_irqrestore(&n->list_lock, flags); 5912 5913 if (slab_free) { 5914 stat(s, FREE_SLAB); 5915 free_slab(s, slab_free); 5916 } 5917 } 5918 5919 /* 5920 * Slow path handling. This may still be called frequently since objects 5921 * have a longer lifetime than the cpu slabs in most processing loads. 5922 * 5923 * So we still attempt to reduce cache line usage. Just take the slab 5924 * lock and free the item. If there is no additional partial slab 5925 * handling required then we can return immediately. 5926 */ 5927 static void __slab_free(struct kmem_cache *s, struct slab *slab, 5928 void *head, void *tail, int cnt, 5929 unsigned long addr) 5930 5931 { 5932 void *prior; 5933 int was_frozen; 5934 struct slab new; 5935 unsigned long counters; 5936 struct kmem_cache_node *n = NULL; 5937 unsigned long flags; 5938 bool on_node_partial; 5939 5940 stat(s, FREE_SLOWPATH); 5941 5942 if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) { 5943 free_to_partial_list(s, slab, head, tail, cnt, addr); 5944 return; 5945 } 5946 5947 do { 5948 if (unlikely(n)) { 5949 spin_unlock_irqrestore(&n->list_lock, flags); 5950 n = NULL; 5951 } 5952 prior = slab->freelist; 5953 counters = slab->counters; 5954 set_freepointer(s, tail, prior); 5955 new.counters = counters; 5956 was_frozen = new.frozen; 5957 new.inuse -= cnt; 5958 if ((!new.inuse || !prior) && !was_frozen) { 5959 /* Needs to be taken off a list */ 5960 if (!kmem_cache_has_cpu_partial(s) || prior) { 5961 5962 n = get_node(s, slab_nid(slab)); 5963 /* 5964 * Speculatively acquire the list_lock. 5965 * If the cmpxchg does not succeed then we may 5966 * drop the list_lock without any processing. 5967 * 5968 * Otherwise the list_lock will synchronize with 5969 * other processors updating the list of slabs. 5970 */ 5971 spin_lock_irqsave(&n->list_lock, flags); 5972 5973 on_node_partial = slab_test_node_partial(slab); 5974 } 5975 } 5976 5977 } while (!slab_update_freelist(s, slab, 5978 prior, counters, 5979 head, new.counters, 5980 "__slab_free")); 5981 5982 if (likely(!n)) { 5983 5984 if (likely(was_frozen)) { 5985 /* 5986 * The list lock was not taken therefore no list 5987 * activity can be necessary. 5988 */ 5989 stat(s, FREE_FROZEN); 5990 } else if (kmem_cache_has_cpu_partial(s) && !prior) { 5991 /* 5992 * If we started with a full slab then put it onto the 5993 * per cpu partial list. 5994 */ 5995 put_cpu_partial(s, slab, 1); 5996 stat(s, CPU_PARTIAL_FREE); 5997 } 5998 5999 return; 6000 } 6001 6002 /* 6003 * This slab was partially empty but not on the per-node partial list, 6004 * in which case we shouldn't manipulate its list, just return. 6005 */ 6006 if (prior && !on_node_partial) { 6007 spin_unlock_irqrestore(&n->list_lock, flags); 6008 return; 6009 } 6010 6011 if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) 6012 goto slab_empty; 6013 6014 /* 6015 * Objects left in the slab. If it was not on the partial list before 6016 * then add it. 6017 */ 6018 if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) { 6019 add_partial(n, slab, DEACTIVATE_TO_TAIL); 6020 stat(s, FREE_ADD_PARTIAL); 6021 } 6022 spin_unlock_irqrestore(&n->list_lock, flags); 6023 return; 6024 6025 slab_empty: 6026 if (prior) { 6027 /* 6028 * Slab on the partial list. 6029 */ 6030 remove_partial(n, slab); 6031 stat(s, FREE_REMOVE_PARTIAL); 6032 } 6033 6034 spin_unlock_irqrestore(&n->list_lock, flags); 6035 stat(s, FREE_SLAB); 6036 discard_slab(s, slab); 6037 } 6038 6039 /* 6040 * pcs is locked. We should have get rid of the spare sheaf and obtained an 6041 * empty sheaf, while the main sheaf is full. We want to install the empty sheaf 6042 * as a main sheaf, and make the current main sheaf a spare sheaf. 6043 * 6044 * However due to having relinquished the cpu_sheaves lock when obtaining 6045 * the empty sheaf, we need to handle some unlikely but possible cases. 6046 * 6047 * If we put any sheaf to barn here, it's because we were interrupted or have 6048 * been migrated to a different cpu, which should be rare enough so just ignore 6049 * the barn's limits to simplify the handling. 6050 * 6051 * An alternative scenario that gets us here is when we fail 6052 * barn_replace_full_sheaf(), because there's no empty sheaf available in the 6053 * barn, so we had to allocate it by alloc_empty_sheaf(). But because we saw the 6054 * limit on full sheaves was not exceeded, we assume it didn't change and just 6055 * put the full sheaf there. 6056 */ 6057 static void __pcs_install_empty_sheaf(struct kmem_cache *s, 6058 struct slub_percpu_sheaves *pcs, struct slab_sheaf *empty, 6059 struct node_barn *barn) 6060 { 6061 lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); 6062 6063 /* This is what we expect to find if nobody interrupted us. */ 6064 if (likely(!pcs->spare)) { 6065 pcs->spare = pcs->main; 6066 pcs->main = empty; 6067 return; 6068 } 6069 6070 /* 6071 * Unlikely because if the main sheaf had space, we would have just 6072 * freed to it. Get rid of our empty sheaf. 6073 */ 6074 if (pcs->main->size < s->sheaf_capacity) { 6075 barn_put_empty_sheaf(barn, empty); 6076 return; 6077 } 6078 6079 /* Also unlikely for the same reason */ 6080 if (pcs->spare->size < s->sheaf_capacity) { 6081 swap(pcs->main, pcs->spare); 6082 barn_put_empty_sheaf(barn, empty); 6083 return; 6084 } 6085 6086 /* 6087 * We probably failed barn_replace_full_sheaf() due to no empty sheaf 6088 * available there, but we allocated one, so finish the job. 6089 */ 6090 barn_put_full_sheaf(barn, pcs->main); 6091 stat(s, BARN_PUT); 6092 pcs->main = empty; 6093 } 6094 6095 /* 6096 * Replace the full main sheaf with a (at least partially) empty sheaf. 6097 * 6098 * Must be called with the cpu_sheaves local lock locked. If successful, returns 6099 * the pcs pointer and the local lock locked (possibly on a different cpu than 6100 * initially called). If not successful, returns NULL and the local lock 6101 * unlocked. 6102 */ 6103 static struct slub_percpu_sheaves * 6104 __pcs_replace_full_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs) 6105 { 6106 struct slab_sheaf *empty; 6107 struct node_barn *barn; 6108 bool put_fail; 6109 6110 restart: 6111 lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); 6112 6113 barn = get_barn(s); 6114 if (!barn) { 6115 local_unlock(&s->cpu_sheaves->lock); 6116 return NULL; 6117 } 6118 6119 put_fail = false; 6120 6121 if (!pcs->spare) { 6122 empty = barn_get_empty_sheaf(barn); 6123 if (empty) { 6124 pcs->spare = pcs->main; 6125 pcs->main = empty; 6126 return pcs; 6127 } 6128 goto alloc_empty; 6129 } 6130 6131 if (pcs->spare->size < s->sheaf_capacity) { 6132 swap(pcs->main, pcs->spare); 6133 return pcs; 6134 } 6135 6136 empty = barn_replace_full_sheaf(barn, pcs->main); 6137 6138 if (!IS_ERR(empty)) { 6139 stat(s, BARN_PUT); 6140 pcs->main = empty; 6141 return pcs; 6142 } 6143 6144 if (PTR_ERR(empty) == -E2BIG) { 6145 /* Since we got here, spare exists and is full */ 6146 struct slab_sheaf *to_flush = pcs->spare; 6147 6148 stat(s, BARN_PUT_FAIL); 6149 6150 pcs->spare = NULL; 6151 local_unlock(&s->cpu_sheaves->lock); 6152 6153 sheaf_flush_unused(s, to_flush); 6154 empty = to_flush; 6155 goto got_empty; 6156 } 6157 6158 /* 6159 * We could not replace full sheaf because barn had no empty 6160 * sheaves. We can still allocate it and put the full sheaf in 6161 * __pcs_install_empty_sheaf(), but if we fail to allocate it, 6162 * make sure to count the fail. 6163 */ 6164 put_fail = true; 6165 6166 alloc_empty: 6167 local_unlock(&s->cpu_sheaves->lock); 6168 6169 empty = alloc_empty_sheaf(s, GFP_NOWAIT); 6170 if (empty) 6171 goto got_empty; 6172 6173 if (put_fail) 6174 stat(s, BARN_PUT_FAIL); 6175 6176 if (!sheaf_try_flush_main(s)) 6177 return NULL; 6178 6179 if (!local_trylock(&s->cpu_sheaves->lock)) 6180 return NULL; 6181 6182 pcs = this_cpu_ptr(s->cpu_sheaves); 6183 6184 /* 6185 * we flushed the main sheaf so it should be empty now, 6186 * but in case we got preempted or migrated, we need to 6187 * check again 6188 */ 6189 if (pcs->main->size == s->sheaf_capacity) 6190 goto restart; 6191 6192 return pcs; 6193 6194 got_empty: 6195 if (!local_trylock(&s->cpu_sheaves->lock)) { 6196 barn_put_empty_sheaf(barn, empty); 6197 return NULL; 6198 } 6199 6200 pcs = this_cpu_ptr(s->cpu_sheaves); 6201 __pcs_install_empty_sheaf(s, pcs, empty, barn); 6202 6203 return pcs; 6204 } 6205 6206 /* 6207 * Free an object to the percpu sheaves. 6208 * The object is expected to have passed slab_free_hook() already. 6209 */ 6210 static __fastpath_inline 6211 bool free_to_pcs(struct kmem_cache *s, void *object) 6212 { 6213 struct slub_percpu_sheaves *pcs; 6214 6215 if (!local_trylock(&s->cpu_sheaves->lock)) 6216 return false; 6217 6218 pcs = this_cpu_ptr(s->cpu_sheaves); 6219 6220 if (unlikely(pcs->main->size == s->sheaf_capacity)) { 6221 6222 pcs = __pcs_replace_full_main(s, pcs); 6223 if (unlikely(!pcs)) 6224 return false; 6225 } 6226 6227 pcs->main->objects[pcs->main->size++] = object; 6228 6229 local_unlock(&s->cpu_sheaves->lock); 6230 6231 stat(s, FREE_PCS); 6232 6233 return true; 6234 } 6235 6236 static void rcu_free_sheaf(struct rcu_head *head) 6237 { 6238 struct kmem_cache_node *n; 6239 struct slab_sheaf *sheaf; 6240 struct node_barn *barn = NULL; 6241 struct kmem_cache *s; 6242 6243 sheaf = container_of(head, struct slab_sheaf, rcu_head); 6244 6245 s = sheaf->cache; 6246 6247 /* 6248 * This may remove some objects due to slab_free_hook() returning false, 6249 * so that the sheaf might no longer be completely full. But it's easier 6250 * to handle it as full (unless it became completely empty), as the code 6251 * handles it fine. The only downside is that sheaf will serve fewer 6252 * allocations when reused. It only happens due to debugging, which is a 6253 * performance hit anyway. 6254 */ 6255 __rcu_free_sheaf_prepare(s, sheaf); 6256 6257 n = get_node(s, sheaf->node); 6258 if (!n) 6259 goto flush; 6260 6261 barn = n->barn; 6262 6263 /* due to slab_free_hook() */ 6264 if (unlikely(sheaf->size == 0)) 6265 goto empty; 6266 6267 /* 6268 * Checking nr_full/nr_empty outside lock avoids contention in case the 6269 * barn is at the respective limit. Due to the race we might go over the 6270 * limit but that should be rare and harmless. 6271 */ 6272 6273 if (data_race(barn->nr_full) < MAX_FULL_SHEAVES) { 6274 stat(s, BARN_PUT); 6275 barn_put_full_sheaf(barn, sheaf); 6276 return; 6277 } 6278 6279 flush: 6280 stat(s, BARN_PUT_FAIL); 6281 sheaf_flush_unused(s, sheaf); 6282 6283 empty: 6284 if (barn && data_race(barn->nr_empty) < MAX_EMPTY_SHEAVES) { 6285 barn_put_empty_sheaf(barn, sheaf); 6286 return; 6287 } 6288 6289 free_empty_sheaf(s, sheaf); 6290 } 6291 6292 /* 6293 * kvfree_call_rcu() can be called while holding a raw_spinlock_t. Since 6294 * __kfree_rcu_sheaf() may acquire a spinlock_t (sleeping lock on PREEMPT_RT), 6295 * this would violate lock nesting rules. Therefore, kvfree_call_rcu() avoids 6296 * this problem by bypassing the sheaves layer entirely on PREEMPT_RT. 6297 * 6298 * However, lockdep still complains that it is invalid to acquire spinlock_t 6299 * while holding raw_spinlock_t, even on !PREEMPT_RT where spinlock_t is a 6300 * spinning lock. Tell lockdep that acquiring spinlock_t is valid here 6301 * by temporarily raising the wait-type to LD_WAIT_CONFIG. 6302 */ 6303 static DEFINE_WAIT_OVERRIDE_MAP(kfree_rcu_sheaf_map, LD_WAIT_CONFIG); 6304 6305 bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) 6306 { 6307 struct slub_percpu_sheaves *pcs; 6308 struct slab_sheaf *rcu_sheaf; 6309 6310 if (WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_RT))) 6311 return false; 6312 6313 lock_map_acquire_try(&kfree_rcu_sheaf_map); 6314 6315 if (!local_trylock(&s->cpu_sheaves->lock)) 6316 goto fail; 6317 6318 pcs = this_cpu_ptr(s->cpu_sheaves); 6319 6320 if (unlikely(!pcs->rcu_free)) { 6321 6322 struct slab_sheaf *empty; 6323 struct node_barn *barn; 6324 6325 if (pcs->spare && pcs->spare->size == 0) { 6326 pcs->rcu_free = pcs->spare; 6327 pcs->spare = NULL; 6328 goto do_free; 6329 } 6330 6331 barn = get_barn(s); 6332 if (!barn) { 6333 local_unlock(&s->cpu_sheaves->lock); 6334 goto fail; 6335 } 6336 6337 empty = barn_get_empty_sheaf(barn); 6338 6339 if (empty) { 6340 pcs->rcu_free = empty; 6341 goto do_free; 6342 } 6343 6344 local_unlock(&s->cpu_sheaves->lock); 6345 6346 empty = alloc_empty_sheaf(s, GFP_NOWAIT); 6347 6348 if (!empty) 6349 goto fail; 6350 6351 if (!local_trylock(&s->cpu_sheaves->lock)) { 6352 barn_put_empty_sheaf(barn, empty); 6353 goto fail; 6354 } 6355 6356 pcs = this_cpu_ptr(s->cpu_sheaves); 6357 6358 if (unlikely(pcs->rcu_free)) 6359 barn_put_empty_sheaf(barn, empty); 6360 else 6361 pcs->rcu_free = empty; 6362 } 6363 6364 do_free: 6365 6366 rcu_sheaf = pcs->rcu_free; 6367 6368 /* 6369 * Since we flush immediately when size reaches capacity, we never reach 6370 * this with size already at capacity, so no OOB write is possible. 6371 */ 6372 rcu_sheaf->objects[rcu_sheaf->size++] = obj; 6373 6374 if (likely(rcu_sheaf->size < s->sheaf_capacity)) { 6375 rcu_sheaf = NULL; 6376 } else { 6377 pcs->rcu_free = NULL; 6378 rcu_sheaf->node = numa_mem_id(); 6379 } 6380 6381 /* 6382 * we flush before local_unlock to make sure a racing 6383 * flush_all_rcu_sheaves() doesn't miss this sheaf 6384 */ 6385 if (rcu_sheaf) 6386 call_rcu(&rcu_sheaf->rcu_head, rcu_free_sheaf); 6387 6388 local_unlock(&s->cpu_sheaves->lock); 6389 6390 stat(s, FREE_RCU_SHEAF); 6391 lock_map_release(&kfree_rcu_sheaf_map); 6392 return true; 6393 6394 fail: 6395 stat(s, FREE_RCU_SHEAF_FAIL); 6396 lock_map_release(&kfree_rcu_sheaf_map); 6397 return false; 6398 } 6399 6400 /* 6401 * Bulk free objects to the percpu sheaves. 6402 * Unlike free_to_pcs() this includes the calls to all necessary hooks 6403 * and the fallback to freeing to slab pages. 6404 */ 6405 static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p) 6406 { 6407 struct slub_percpu_sheaves *pcs; 6408 struct slab_sheaf *main, *empty; 6409 bool init = slab_want_init_on_free(s); 6410 unsigned int batch, i = 0; 6411 struct node_barn *barn; 6412 void *remote_objects[PCS_BATCH_MAX]; 6413 unsigned int remote_nr = 0; 6414 int node = numa_mem_id(); 6415 6416 next_remote_batch: 6417 while (i < size) { 6418 struct slab *slab = virt_to_slab(p[i]); 6419 6420 memcg_slab_free_hook(s, slab, p + i, 1); 6421 alloc_tagging_slab_free_hook(s, slab, p + i, 1); 6422 6423 if (unlikely(!slab_free_hook(s, p[i], init, false))) { 6424 p[i] = p[--size]; 6425 continue; 6426 } 6427 6428 if (unlikely(IS_ENABLED(CONFIG_NUMA) && slab_nid(slab) != node)) { 6429 remote_objects[remote_nr] = p[i]; 6430 p[i] = p[--size]; 6431 if (++remote_nr >= PCS_BATCH_MAX) 6432 goto flush_remote; 6433 continue; 6434 } 6435 6436 i++; 6437 } 6438 6439 if (!size) 6440 goto flush_remote; 6441 6442 next_batch: 6443 if (!local_trylock(&s->cpu_sheaves->lock)) 6444 goto fallback; 6445 6446 pcs = this_cpu_ptr(s->cpu_sheaves); 6447 6448 if (likely(pcs->main->size < s->sheaf_capacity)) 6449 goto do_free; 6450 6451 barn = get_barn(s); 6452 if (!barn) 6453 goto no_empty; 6454 6455 if (!pcs->spare) { 6456 empty = barn_get_empty_sheaf(barn); 6457 if (!empty) 6458 goto no_empty; 6459 6460 pcs->spare = pcs->main; 6461 pcs->main = empty; 6462 goto do_free; 6463 } 6464 6465 if (pcs->spare->size < s->sheaf_capacity) { 6466 swap(pcs->main, pcs->spare); 6467 goto do_free; 6468 } 6469 6470 empty = barn_replace_full_sheaf(barn, pcs->main); 6471 if (IS_ERR(empty)) { 6472 stat(s, BARN_PUT_FAIL); 6473 goto no_empty; 6474 } 6475 6476 stat(s, BARN_PUT); 6477 pcs->main = empty; 6478 6479 do_free: 6480 main = pcs->main; 6481 batch = min(size, s->sheaf_capacity - main->size); 6482 6483 memcpy(main->objects + main->size, p, batch * sizeof(void *)); 6484 main->size += batch; 6485 6486 local_unlock(&s->cpu_sheaves->lock); 6487 6488 stat_add(s, FREE_PCS, batch); 6489 6490 if (batch < size) { 6491 p += batch; 6492 size -= batch; 6493 goto next_batch; 6494 } 6495 6496 if (remote_nr) 6497 goto flush_remote; 6498 6499 return; 6500 6501 no_empty: 6502 local_unlock(&s->cpu_sheaves->lock); 6503 6504 /* 6505 * if we depleted all empty sheaves in the barn or there are too 6506 * many full sheaves, free the rest to slab pages 6507 */ 6508 fallback: 6509 __kmem_cache_free_bulk(s, size, p); 6510 6511 flush_remote: 6512 if (remote_nr) { 6513 __kmem_cache_free_bulk(s, remote_nr, &remote_objects[0]); 6514 if (i < size) { 6515 remote_nr = 0; 6516 goto next_remote_batch; 6517 } 6518 } 6519 } 6520 6521 struct defer_free { 6522 struct llist_head objects; 6523 struct llist_head slabs; 6524 struct irq_work work; 6525 }; 6526 6527 static void free_deferred_objects(struct irq_work *work); 6528 6529 static DEFINE_PER_CPU(struct defer_free, defer_free_objects) = { 6530 .objects = LLIST_HEAD_INIT(objects), 6531 .slabs = LLIST_HEAD_INIT(slabs), 6532 .work = IRQ_WORK_INIT(free_deferred_objects), 6533 }; 6534 6535 /* 6536 * In PREEMPT_RT irq_work runs in per-cpu kthread, so it's safe 6537 * to take sleeping spin_locks from __slab_free() and deactivate_slab(). 6538 * In !PREEMPT_RT irq_work will run after local_unlock_irqrestore(). 6539 */ 6540 static void free_deferred_objects(struct irq_work *work) 6541 { 6542 struct defer_free *df = container_of(work, struct defer_free, work); 6543 struct llist_head *objs = &df->objects; 6544 struct llist_head *slabs = &df->slabs; 6545 struct llist_node *llnode, *pos, *t; 6546 6547 if (llist_empty(objs) && llist_empty(slabs)) 6548 return; 6549 6550 llnode = llist_del_all(objs); 6551 llist_for_each_safe(pos, t, llnode) { 6552 struct kmem_cache *s; 6553 struct slab *slab; 6554 void *x = pos; 6555 6556 slab = virt_to_slab(x); 6557 s = slab->slab_cache; 6558 6559 /* Point 'x' back to the beginning of allocated object */ 6560 x -= s->offset; 6561 6562 /* 6563 * We used freepointer in 'x' to link 'x' into df->objects. 6564 * Clear it to NULL to avoid false positive detection 6565 * of "Freepointer corruption". 6566 */ 6567 set_freepointer(s, x, NULL); 6568 6569 __slab_free(s, slab, x, x, 1, _THIS_IP_); 6570 } 6571 6572 llnode = llist_del_all(slabs); 6573 llist_for_each_safe(pos, t, llnode) { 6574 struct slab *slab = container_of(pos, struct slab, llnode); 6575 6576 if (slab->frozen) 6577 deactivate_slab(slab->slab_cache, slab, slab->flush_freelist); 6578 else 6579 free_slab(slab->slab_cache, slab); 6580 } 6581 } 6582 6583 static void defer_free(struct kmem_cache *s, void *head) 6584 { 6585 struct defer_free *df; 6586 6587 guard(preempt)(); 6588 6589 head = kasan_reset_tag(head); 6590 6591 df = this_cpu_ptr(&defer_free_objects); 6592 if (llist_add(head + s->offset, &df->objects)) 6593 irq_work_queue(&df->work); 6594 } 6595 6596 static void defer_deactivate_slab(struct slab *slab, void *flush_freelist) 6597 { 6598 struct defer_free *df; 6599 6600 slab->flush_freelist = flush_freelist; 6601 6602 guard(preempt)(); 6603 6604 df = this_cpu_ptr(&defer_free_objects); 6605 if (llist_add(&slab->llnode, &df->slabs)) 6606 irq_work_queue(&df->work); 6607 } 6608 6609 void defer_free_barrier(void) 6610 { 6611 int cpu; 6612 6613 for_each_possible_cpu(cpu) 6614 irq_work_sync(&per_cpu_ptr(&defer_free_objects, cpu)->work); 6615 } 6616 6617 /* 6618 * Fastpath with forced inlining to produce a kfree and kmem_cache_free that 6619 * can perform fastpath freeing without additional function calls. 6620 * 6621 * The fastpath is only possible if we are freeing to the current cpu slab 6622 * of this processor. This typically the case if we have just allocated 6623 * the item before. 6624 * 6625 * If fastpath is not possible then fall back to __slab_free where we deal 6626 * with all sorts of special processing. 6627 * 6628 * Bulk free of a freelist with several objects (all pointing to the 6629 * same slab) possible by specifying head and tail ptr, plus objects 6630 * count (cnt). Bulk free indicated by tail pointer being set. 6631 */ 6632 static __always_inline void do_slab_free(struct kmem_cache *s, 6633 struct slab *slab, void *head, void *tail, 6634 int cnt, unsigned long addr) 6635 { 6636 /* cnt == 0 signals that it's called from kfree_nolock() */ 6637 bool allow_spin = cnt; 6638 struct kmem_cache_cpu *c; 6639 unsigned long tid; 6640 void **freelist; 6641 6642 redo: 6643 /* 6644 * Determine the currently cpus per cpu slab. 6645 * The cpu may change afterward. However that does not matter since 6646 * data is retrieved via this pointer. If we are on the same cpu 6647 * during the cmpxchg then the free will succeed. 6648 */ 6649 c = raw_cpu_ptr(s->cpu_slab); 6650 tid = READ_ONCE(c->tid); 6651 6652 /* Same with comment on barrier() in __slab_alloc_node() */ 6653 barrier(); 6654 6655 if (unlikely(slab != c->slab)) { 6656 if (unlikely(!allow_spin)) { 6657 /* 6658 * __slab_free() can locklessly cmpxchg16 into a slab, 6659 * but then it might need to take spin_lock or local_lock 6660 * in put_cpu_partial() for further processing. 6661 * Avoid the complexity and simply add to a deferred list. 6662 */ 6663 defer_free(s, head); 6664 } else { 6665 __slab_free(s, slab, head, tail, cnt, addr); 6666 } 6667 return; 6668 } 6669 6670 if (unlikely(!allow_spin)) { 6671 if ((in_nmi() || !USE_LOCKLESS_FAST_PATH()) && 6672 local_lock_is_locked(&s->cpu_slab->lock)) { 6673 defer_free(s, head); 6674 return; 6675 } 6676 cnt = 1; /* restore cnt. kfree_nolock() frees one object at a time */ 6677 } 6678 6679 if (USE_LOCKLESS_FAST_PATH()) { 6680 freelist = READ_ONCE(c->freelist); 6681 6682 set_freepointer(s, tail, freelist); 6683 6684 if (unlikely(!__update_cpu_freelist_fast(s, freelist, head, tid))) { 6685 note_cmpxchg_failure("slab_free", s, tid); 6686 goto redo; 6687 } 6688 } else { 6689 __maybe_unused unsigned long flags = 0; 6690 6691 /* Update the free list under the local lock */ 6692 local_lock_cpu_slab(s, flags); 6693 c = this_cpu_ptr(s->cpu_slab); 6694 if (unlikely(slab != c->slab)) { 6695 local_unlock_cpu_slab(s, flags); 6696 goto redo; 6697 } 6698 tid = c->tid; 6699 freelist = c->freelist; 6700 6701 set_freepointer(s, tail, freelist); 6702 c->freelist = head; 6703 c->tid = next_tid(tid); 6704 6705 local_unlock_cpu_slab(s, flags); 6706 } 6707 stat_add(s, FREE_FASTPATH, cnt); 6708 } 6709 6710 static __fastpath_inline 6711 void slab_free(struct kmem_cache *s, struct slab *slab, void *object, 6712 unsigned long addr) 6713 { 6714 memcg_slab_free_hook(s, slab, &object, 1); 6715 alloc_tagging_slab_free_hook(s, slab, &object, 1); 6716 6717 if (unlikely(!slab_free_hook(s, object, slab_want_init_on_free(s), false))) 6718 return; 6719 6720 if (s->cpu_sheaves && likely(!IS_ENABLED(CONFIG_NUMA) || 6721 slab_nid(slab) == numa_mem_id())) { 6722 if (likely(free_to_pcs(s, object))) 6723 return; 6724 } 6725 6726 do_slab_free(s, slab, object, object, 1, addr); 6727 } 6728 6729 #ifdef CONFIG_MEMCG 6730 /* Do not inline the rare memcg charging failed path into the allocation path */ 6731 static noinline 6732 void memcg_alloc_abort_single(struct kmem_cache *s, void *object) 6733 { 6734 struct slab *slab = virt_to_slab(object); 6735 6736 alloc_tagging_slab_free_hook(s, slab, &object, 1); 6737 6738 if (likely(slab_free_hook(s, object, slab_want_init_on_free(s), false))) 6739 do_slab_free(s, slab, object, object, 1, _RET_IP_); 6740 } 6741 #endif 6742 6743 static __fastpath_inline 6744 void slab_free_bulk(struct kmem_cache *s, struct slab *slab, void *head, 6745 void *tail, void **p, int cnt, unsigned long addr) 6746 { 6747 memcg_slab_free_hook(s, slab, p, cnt); 6748 alloc_tagging_slab_free_hook(s, slab, p, cnt); 6749 /* 6750 * With KASAN enabled slab_free_freelist_hook modifies the freelist 6751 * to remove objects, whose reuse must be delayed. 6752 */ 6753 if (likely(slab_free_freelist_hook(s, &head, &tail, &cnt))) 6754 do_slab_free(s, slab, head, tail, cnt, addr); 6755 } 6756 6757 #ifdef CONFIG_SLUB_RCU_DEBUG 6758 static void slab_free_after_rcu_debug(struct rcu_head *rcu_head) 6759 { 6760 struct rcu_delayed_free *delayed_free = 6761 container_of(rcu_head, struct rcu_delayed_free, head); 6762 void *object = delayed_free->object; 6763 struct slab *slab = virt_to_slab(object); 6764 struct kmem_cache *s; 6765 6766 kfree(delayed_free); 6767 6768 if (WARN_ON(is_kfence_address(object))) 6769 return; 6770 6771 /* find the object and the cache again */ 6772 if (WARN_ON(!slab)) 6773 return; 6774 s = slab->slab_cache; 6775 if (WARN_ON(!(s->flags & SLAB_TYPESAFE_BY_RCU))) 6776 return; 6777 6778 /* resume freeing */ 6779 if (slab_free_hook(s, object, slab_want_init_on_free(s), true)) 6780 do_slab_free(s, slab, object, object, 1, _THIS_IP_); 6781 } 6782 #endif /* CONFIG_SLUB_RCU_DEBUG */ 6783 6784 #ifdef CONFIG_KASAN_GENERIC 6785 void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr) 6786 { 6787 do_slab_free(cache, virt_to_slab(x), x, x, 1, addr); 6788 } 6789 #endif 6790 6791 static inline struct kmem_cache *virt_to_cache(const void *obj) 6792 { 6793 struct slab *slab; 6794 6795 slab = virt_to_slab(obj); 6796 if (WARN_ONCE(!slab, "%s: Object is not a Slab page!\n", __func__)) 6797 return NULL; 6798 return slab->slab_cache; 6799 } 6800 6801 static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x) 6802 { 6803 struct kmem_cache *cachep; 6804 6805 if (!IS_ENABLED(CONFIG_SLAB_FREELIST_HARDENED) && 6806 !kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS)) 6807 return s; 6808 6809 cachep = virt_to_cache(x); 6810 if (WARN(cachep && cachep != s, 6811 "%s: Wrong slab cache. %s but object is from %s\n", 6812 __func__, s->name, cachep->name)) 6813 print_tracking(cachep, x); 6814 return cachep; 6815 } 6816 6817 /** 6818 * kmem_cache_free - Deallocate an object 6819 * @s: The cache the allocation was from. 6820 * @x: The previously allocated object. 6821 * 6822 * Free an object which was previously allocated from this 6823 * cache. 6824 */ 6825 void kmem_cache_free(struct kmem_cache *s, void *x) 6826 { 6827 s = cache_from_obj(s, x); 6828 if (!s) 6829 return; 6830 trace_kmem_cache_free(_RET_IP_, x, s); 6831 slab_free(s, virt_to_slab(x), x, _RET_IP_); 6832 } 6833 EXPORT_SYMBOL(kmem_cache_free); 6834 6835 static void free_large_kmalloc(struct folio *folio, void *object) 6836 { 6837 unsigned int order = folio_order(folio); 6838 6839 if (WARN_ON_ONCE(!folio_test_large_kmalloc(folio))) { 6840 dump_page(&folio->page, "Not a kmalloc allocation"); 6841 return; 6842 } 6843 6844 if (WARN_ON_ONCE(order == 0)) 6845 pr_warn_once("object pointer: 0x%p\n", object); 6846 6847 kmemleak_free(object); 6848 kasan_kfree_large(object); 6849 kmsan_kfree_large(object); 6850 6851 lruvec_stat_mod_folio(folio, NR_SLAB_UNRECLAIMABLE_B, 6852 -(PAGE_SIZE << order)); 6853 __folio_clear_large_kmalloc(folio); 6854 free_frozen_pages(&folio->page, order); 6855 } 6856 6857 /* 6858 * Given an rcu_head embedded within an object obtained from kvmalloc at an 6859 * offset < 4k, free the object in question. 6860 */ 6861 void kvfree_rcu_cb(struct rcu_head *head) 6862 { 6863 void *obj = head; 6864 struct folio *folio; 6865 struct slab *slab; 6866 struct kmem_cache *s; 6867 void *slab_addr; 6868 6869 if (is_vmalloc_addr(obj)) { 6870 obj = (void *) PAGE_ALIGN_DOWN((unsigned long)obj); 6871 vfree(obj); 6872 return; 6873 } 6874 6875 folio = virt_to_folio(obj); 6876 if (!folio_test_slab(folio)) { 6877 /* 6878 * rcu_head offset can be only less than page size so no need to 6879 * consider folio order 6880 */ 6881 obj = (void *) PAGE_ALIGN_DOWN((unsigned long)obj); 6882 free_large_kmalloc(folio, obj); 6883 return; 6884 } 6885 6886 slab = folio_slab(folio); 6887 s = slab->slab_cache; 6888 slab_addr = folio_address(folio); 6889 6890 if (is_kfence_address(obj)) { 6891 obj = kfence_object_start(obj); 6892 } else { 6893 unsigned int idx = __obj_to_index(s, slab_addr, obj); 6894 6895 obj = slab_addr + s->size * idx; 6896 obj = fixup_red_left(s, obj); 6897 } 6898 6899 slab_free(s, slab, obj, _RET_IP_); 6900 } 6901 6902 /** 6903 * kfree - free previously allocated memory 6904 * @object: pointer returned by kmalloc() or kmem_cache_alloc() 6905 * 6906 * If @object is NULL, no operation is performed. 6907 */ 6908 void kfree(const void *object) 6909 { 6910 struct folio *folio; 6911 struct slab *slab; 6912 struct kmem_cache *s; 6913 void *x = (void *)object; 6914 6915 trace_kfree(_RET_IP_, object); 6916 6917 if (unlikely(ZERO_OR_NULL_PTR(object))) 6918 return; 6919 6920 folio = virt_to_folio(object); 6921 if (unlikely(!folio_test_slab(folio))) { 6922 free_large_kmalloc(folio, (void *)object); 6923 return; 6924 } 6925 6926 slab = folio_slab(folio); 6927 s = slab->slab_cache; 6928 slab_free(s, slab, x, _RET_IP_); 6929 } 6930 EXPORT_SYMBOL(kfree); 6931 6932 /* 6933 * Can be called while holding raw_spinlock_t or from IRQ and NMI, 6934 * but ONLY for objects allocated by kmalloc_nolock(). 6935 * Debug checks (like kmemleak and kfence) were skipped on allocation, 6936 * hence 6937 * obj = kmalloc(); kfree_nolock(obj); 6938 * will miss kmemleak/kfence book keeping and will cause false positives. 6939 * large_kmalloc is not supported either. 6940 */ 6941 void kfree_nolock(const void *object) 6942 { 6943 struct folio *folio; 6944 struct slab *slab; 6945 struct kmem_cache *s; 6946 void *x = (void *)object; 6947 6948 if (unlikely(ZERO_OR_NULL_PTR(object))) 6949 return; 6950 6951 folio = virt_to_folio(object); 6952 if (unlikely(!folio_test_slab(folio))) { 6953 WARN_ONCE(1, "large_kmalloc is not supported by kfree_nolock()"); 6954 return; 6955 } 6956 6957 slab = folio_slab(folio); 6958 s = slab->slab_cache; 6959 6960 memcg_slab_free_hook(s, slab, &x, 1); 6961 alloc_tagging_slab_free_hook(s, slab, &x, 1); 6962 /* 6963 * Unlike slab_free() do NOT call the following: 6964 * kmemleak_free_recursive(x, s->flags); 6965 * debug_check_no_locks_freed(x, s->object_size); 6966 * debug_check_no_obj_freed(x, s->object_size); 6967 * __kcsan_check_access(x, s->object_size, ..); 6968 * kfence_free(x); 6969 * since they take spinlocks or not safe from any context. 6970 */ 6971 kmsan_slab_free(s, x); 6972 /* 6973 * If KASAN finds a kernel bug it will do kasan_report_invalid_free() 6974 * which will call raw_spin_lock_irqsave() which is technically 6975 * unsafe from NMI, but take chance and report kernel bug. 6976 * The sequence of 6977 * kasan_report_invalid_free() -> raw_spin_lock_irqsave() -> NMI 6978 * -> kfree_nolock() -> kasan_report_invalid_free() on the same CPU 6979 * is double buggy and deserves to deadlock. 6980 */ 6981 if (kasan_slab_pre_free(s, x)) 6982 return; 6983 /* 6984 * memcg, kasan_slab_pre_free are done for 'x'. 6985 * The only thing left is kasan_poison without quarantine, 6986 * since kasan quarantine takes locks and not supported from NMI. 6987 */ 6988 kasan_slab_free(s, x, false, false, /* skip quarantine */true); 6989 do_slab_free(s, slab, x, x, 0, _RET_IP_); 6990 } 6991 EXPORT_SYMBOL_GPL(kfree_nolock); 6992 6993 static __always_inline __realloc_size(2) void * 6994 __do_krealloc(const void *p, size_t new_size, unsigned long align, gfp_t flags, int nid) 6995 { 6996 void *ret; 6997 size_t ks = 0; 6998 int orig_size = 0; 6999 struct kmem_cache *s = NULL; 7000 7001 if (unlikely(ZERO_OR_NULL_PTR(p))) 7002 goto alloc_new; 7003 7004 /* Check for double-free. */ 7005 if (!kasan_check_byte(p)) 7006 return NULL; 7007 7008 if (is_kfence_address(p)) { 7009 ks = orig_size = kfence_ksize(p); 7010 } else { 7011 struct folio *folio; 7012 7013 folio = virt_to_folio(p); 7014 if (unlikely(!folio_test_slab(folio))) { 7015 /* Big kmalloc object */ 7016 WARN_ON(folio_size(folio) <= KMALLOC_MAX_CACHE_SIZE); 7017 WARN_ON(p != folio_address(folio)); 7018 ks = folio_size(folio); 7019 } else { 7020 s = folio_slab(folio)->slab_cache; 7021 orig_size = get_orig_size(s, (void *)p); 7022 ks = s->object_size; 7023 } 7024 } 7025 7026 /* 7027 * If reallocation is not necessary (e. g. the new size is less 7028 * than the current allocated size), the current allocation will be 7029 * preserved unless __GFP_THISNODE is set. In the latter case a new 7030 * allocation on the requested node will be attempted. 7031 */ 7032 if (unlikely(flags & __GFP_THISNODE) && nid != NUMA_NO_NODE && 7033 nid != page_to_nid(virt_to_page(p))) 7034 goto alloc_new; 7035 7036 /* If the old object doesn't fit, allocate a bigger one */ 7037 if (new_size > ks) 7038 goto alloc_new; 7039 7040 /* If the old object doesn't satisfy the new alignment, allocate a new one */ 7041 if (!IS_ALIGNED((unsigned long)p, align)) 7042 goto alloc_new; 7043 7044 /* Zero out spare memory. */ 7045 if (want_init_on_alloc(flags)) { 7046 kasan_disable_current(); 7047 if (orig_size && orig_size < new_size) 7048 memset(kasan_reset_tag(p) + orig_size, 0, new_size - orig_size); 7049 else 7050 memset(kasan_reset_tag(p) + new_size, 0, ks - new_size); 7051 kasan_enable_current(); 7052 } 7053 7054 /* Setup kmalloc redzone when needed */ 7055 if (s && slub_debug_orig_size(s)) { 7056 set_orig_size(s, (void *)p, new_size); 7057 if (s->flags & SLAB_RED_ZONE && new_size < ks) 7058 memset_no_sanitize_memory(kasan_reset_tag(p) + new_size, 7059 SLUB_RED_ACTIVE, ks - new_size); 7060 } 7061 7062 p = kasan_krealloc(p, new_size, flags); 7063 return (void *)p; 7064 7065 alloc_new: 7066 ret = kmalloc_node_track_caller_noprof(new_size, flags, nid, _RET_IP_); 7067 if (ret && p) { 7068 /* Disable KASAN checks as the object's redzone is accessed. */ 7069 kasan_disable_current(); 7070 memcpy(ret, kasan_reset_tag(p), min(new_size, (size_t)(orig_size ?: ks))); 7071 kasan_enable_current(); 7072 } 7073 7074 return ret; 7075 } 7076 7077 /** 7078 * krealloc_node_align - reallocate memory. The contents will remain unchanged. 7079 * @p: object to reallocate memory for. 7080 * @new_size: how many bytes of memory are required. 7081 * @align: desired alignment. 7082 * @flags: the type of memory to allocate. 7083 * @nid: NUMA node or NUMA_NO_NODE 7084 * 7085 * If @p is %NULL, krealloc() behaves exactly like kmalloc(). If @new_size 7086 * is 0 and @p is not a %NULL pointer, the object pointed to is freed. 7087 * 7088 * Only alignments up to those guaranteed by kmalloc() will be honored. Please see 7089 * Documentation/core-api/memory-allocation.rst for more details. 7090 * 7091 * If __GFP_ZERO logic is requested, callers must ensure that, starting with the 7092 * initial memory allocation, every subsequent call to this API for the same 7093 * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that 7094 * __GFP_ZERO is not fully honored by this API. 7095 * 7096 * When slub_debug_orig_size() is off, krealloc() only knows about the bucket 7097 * size of an allocation (but not the exact size it was allocated with) and 7098 * hence implements the following semantics for shrinking and growing buffers 7099 * with __GFP_ZERO:: 7100 * 7101 * new bucket 7102 * 0 size size 7103 * |--------|----------------| 7104 * | keep | zero | 7105 * 7106 * Otherwise, the original allocation size 'orig_size' could be used to 7107 * precisely clear the requested size, and the new size will also be stored 7108 * as the new 'orig_size'. 7109 * 7110 * In any case, the contents of the object pointed to are preserved up to the 7111 * lesser of the new and old sizes. 7112 * 7113 * Return: pointer to the allocated memory or %NULL in case of error 7114 */ 7115 void *krealloc_node_align_noprof(const void *p, size_t new_size, unsigned long align, 7116 gfp_t flags, int nid) 7117 { 7118 void *ret; 7119 7120 if (unlikely(!new_size)) { 7121 kfree(p); 7122 return ZERO_SIZE_PTR; 7123 } 7124 7125 ret = __do_krealloc(p, new_size, align, flags, nid); 7126 if (ret && kasan_reset_tag(p) != kasan_reset_tag(ret)) 7127 kfree(p); 7128 7129 return ret; 7130 } 7131 EXPORT_SYMBOL(krealloc_node_align_noprof); 7132 7133 static gfp_t kmalloc_gfp_adjust(gfp_t flags, size_t size) 7134 { 7135 /* 7136 * We want to attempt a large physically contiguous block first because 7137 * it is less likely to fragment multiple larger blocks and therefore 7138 * contribute to a long term fragmentation less than vmalloc fallback. 7139 * However make sure that larger requests are not too disruptive - i.e. 7140 * do not direct reclaim unless physically continuous memory is preferred 7141 * (__GFP_RETRY_MAYFAIL mode). We still kick in kswapd/kcompactd to 7142 * start working in the background 7143 */ 7144 if (size > PAGE_SIZE) { 7145 flags |= __GFP_NOWARN; 7146 7147 if (!(flags & __GFP_RETRY_MAYFAIL)) 7148 flags &= ~__GFP_DIRECT_RECLAIM; 7149 7150 /* nofail semantic is implemented by the vmalloc fallback */ 7151 flags &= ~__GFP_NOFAIL; 7152 } 7153 7154 return flags; 7155 } 7156 7157 /** 7158 * __kvmalloc_node - attempt to allocate physically contiguous memory, but upon 7159 * failure, fall back to non-contiguous (vmalloc) allocation. 7160 * @size: size of the request. 7161 * @b: which set of kmalloc buckets to allocate from. 7162 * @align: desired alignment. 7163 * @flags: gfp mask for the allocation - must be compatible (superset) with GFP_KERNEL. 7164 * @node: numa node to allocate from 7165 * 7166 * Only alignments up to those guaranteed by kmalloc() will be honored. Please see 7167 * Documentation/core-api/memory-allocation.rst for more details. 7168 * 7169 * Uses kmalloc to get the memory but if the allocation fails then falls back 7170 * to the vmalloc allocator. Use kvfree for freeing the memory. 7171 * 7172 * GFP_NOWAIT and GFP_ATOMIC are not supported, neither is the __GFP_NORETRY modifier. 7173 * __GFP_RETRY_MAYFAIL is supported, and it should be used only if kmalloc is 7174 * preferable to the vmalloc fallback, due to visible performance drawbacks. 7175 * 7176 * Return: pointer to the allocated memory of %NULL in case of failure 7177 */ 7178 void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), unsigned long align, 7179 gfp_t flags, int node) 7180 { 7181 void *ret; 7182 7183 /* 7184 * It doesn't really make sense to fallback to vmalloc for sub page 7185 * requests 7186 */ 7187 ret = __do_kmalloc_node(size, PASS_BUCKET_PARAM(b), 7188 kmalloc_gfp_adjust(flags, size), 7189 node, _RET_IP_); 7190 if (ret || size <= PAGE_SIZE) 7191 return ret; 7192 7193 /* non-sleeping allocations are not supported by vmalloc */ 7194 if (!gfpflags_allow_blocking(flags)) 7195 return NULL; 7196 7197 /* Don't even allow crazy sizes */ 7198 if (unlikely(size > INT_MAX)) { 7199 WARN_ON_ONCE(!(flags & __GFP_NOWARN)); 7200 return NULL; 7201 } 7202 7203 /* 7204 * kvmalloc() can always use VM_ALLOW_HUGE_VMAP, 7205 * since the callers already cannot assume anything 7206 * about the resulting pointer, and cannot play 7207 * protection games. 7208 */ 7209 return __vmalloc_node_range_noprof(size, align, VMALLOC_START, VMALLOC_END, 7210 flags, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP, 7211 node, __builtin_return_address(0)); 7212 } 7213 EXPORT_SYMBOL(__kvmalloc_node_noprof); 7214 7215 /** 7216 * kvfree() - Free memory. 7217 * @addr: Pointer to allocated memory. 7218 * 7219 * kvfree frees memory allocated by any of vmalloc(), kmalloc() or kvmalloc(). 7220 * It is slightly more efficient to use kfree() or vfree() if you are certain 7221 * that you know which one to use. 7222 * 7223 * Context: Either preemptible task context or not-NMI interrupt. 7224 */ 7225 void kvfree(const void *addr) 7226 { 7227 if (is_vmalloc_addr(addr)) 7228 vfree(addr); 7229 else 7230 kfree(addr); 7231 } 7232 EXPORT_SYMBOL(kvfree); 7233 7234 /** 7235 * kvfree_sensitive - Free a data object containing sensitive information. 7236 * @addr: address of the data object to be freed. 7237 * @len: length of the data object. 7238 * 7239 * Use the special memzero_explicit() function to clear the content of a 7240 * kvmalloc'ed object containing sensitive data to make sure that the 7241 * compiler won't optimize out the data clearing. 7242 */ 7243 void kvfree_sensitive(const void *addr, size_t len) 7244 { 7245 if (likely(!ZERO_OR_NULL_PTR(addr))) { 7246 memzero_explicit((void *)addr, len); 7247 kvfree(addr); 7248 } 7249 } 7250 EXPORT_SYMBOL(kvfree_sensitive); 7251 7252 /** 7253 * kvrealloc_node_align - reallocate memory; contents remain unchanged 7254 * @p: object to reallocate memory for 7255 * @size: the size to reallocate 7256 * @align: desired alignment 7257 * @flags: the flags for the page level allocator 7258 * @nid: NUMA node id 7259 * 7260 * If @p is %NULL, kvrealloc() behaves exactly like kvmalloc(). If @size is 0 7261 * and @p is not a %NULL pointer, the object pointed to is freed. 7262 * 7263 * Only alignments up to those guaranteed by kmalloc() will be honored. Please see 7264 * Documentation/core-api/memory-allocation.rst for more details. 7265 * 7266 * If __GFP_ZERO logic is requested, callers must ensure that, starting with the 7267 * initial memory allocation, every subsequent call to this API for the same 7268 * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that 7269 * __GFP_ZERO is not fully honored by this API. 7270 * 7271 * In any case, the contents of the object pointed to are preserved up to the 7272 * lesser of the new and old sizes. 7273 * 7274 * This function must not be called concurrently with itself or kvfree() for the 7275 * same memory allocation. 7276 * 7277 * Return: pointer to the allocated memory or %NULL in case of error 7278 */ 7279 void *kvrealloc_node_align_noprof(const void *p, size_t size, unsigned long align, 7280 gfp_t flags, int nid) 7281 { 7282 void *n; 7283 7284 if (is_vmalloc_addr(p)) 7285 return vrealloc_node_align_noprof(p, size, align, flags, nid); 7286 7287 n = krealloc_node_align_noprof(p, size, align, kmalloc_gfp_adjust(flags, size), nid); 7288 if (!n) { 7289 /* We failed to krealloc(), fall back to kvmalloc(). */ 7290 n = kvmalloc_node_align_noprof(size, align, flags, nid); 7291 if (!n) 7292 return NULL; 7293 7294 if (p) { 7295 /* We already know that `p` is not a vmalloc address. */ 7296 kasan_disable_current(); 7297 memcpy(n, kasan_reset_tag(p), min(size, ksize(p))); 7298 kasan_enable_current(); 7299 7300 kfree(p); 7301 } 7302 } 7303 7304 return n; 7305 } 7306 EXPORT_SYMBOL(kvrealloc_node_align_noprof); 7307 7308 struct detached_freelist { 7309 struct slab *slab; 7310 void *tail; 7311 void *freelist; 7312 int cnt; 7313 struct kmem_cache *s; 7314 }; 7315 7316 /* 7317 * This function progressively scans the array with free objects (with 7318 * a limited look ahead) and extract objects belonging to the same 7319 * slab. It builds a detached freelist directly within the given 7320 * slab/objects. This can happen without any need for 7321 * synchronization, because the objects are owned by running process. 7322 * The freelist is build up as a single linked list in the objects. 7323 * The idea is, that this detached freelist can then be bulk 7324 * transferred to the real freelist(s), but only requiring a single 7325 * synchronization primitive. Look ahead in the array is limited due 7326 * to performance reasons. 7327 */ 7328 static inline 7329 int build_detached_freelist(struct kmem_cache *s, size_t size, 7330 void **p, struct detached_freelist *df) 7331 { 7332 int lookahead = 3; 7333 void *object; 7334 struct folio *folio; 7335 size_t same; 7336 7337 object = p[--size]; 7338 folio = virt_to_folio(object); 7339 if (!s) { 7340 /* Handle kalloc'ed objects */ 7341 if (unlikely(!folio_test_slab(folio))) { 7342 free_large_kmalloc(folio, object); 7343 df->slab = NULL; 7344 return size; 7345 } 7346 /* Derive kmem_cache from object */ 7347 df->slab = folio_slab(folio); 7348 df->s = df->slab->slab_cache; 7349 } else { 7350 df->slab = folio_slab(folio); 7351 df->s = cache_from_obj(s, object); /* Support for memcg */ 7352 } 7353 7354 /* Start new detached freelist */ 7355 df->tail = object; 7356 df->freelist = object; 7357 df->cnt = 1; 7358 7359 if (is_kfence_address(object)) 7360 return size; 7361 7362 set_freepointer(df->s, object, NULL); 7363 7364 same = size; 7365 while (size) { 7366 object = p[--size]; 7367 /* df->slab is always set at this point */ 7368 if (df->slab == virt_to_slab(object)) { 7369 /* Opportunity build freelist */ 7370 set_freepointer(df->s, object, df->freelist); 7371 df->freelist = object; 7372 df->cnt++; 7373 same--; 7374 if (size != same) 7375 swap(p[size], p[same]); 7376 continue; 7377 } 7378 7379 /* Limit look ahead search */ 7380 if (!--lookahead) 7381 break; 7382 } 7383 7384 return same; 7385 } 7386 7387 /* 7388 * Internal bulk free of objects that were not initialised by the post alloc 7389 * hooks and thus should not be processed by the free hooks 7390 */ 7391 static void __kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p) 7392 { 7393 if (!size) 7394 return; 7395 7396 do { 7397 struct detached_freelist df; 7398 7399 size = build_detached_freelist(s, size, p, &df); 7400 if (!df.slab) 7401 continue; 7402 7403 if (kfence_free(df.freelist)) 7404 continue; 7405 7406 do_slab_free(df.s, df.slab, df.freelist, df.tail, df.cnt, 7407 _RET_IP_); 7408 } while (likely(size)); 7409 } 7410 7411 /* Note that interrupts must be enabled when calling this function. */ 7412 void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p) 7413 { 7414 if (!size) 7415 return; 7416 7417 /* 7418 * freeing to sheaves is so incompatible with the detached freelist so 7419 * once we go that way, we have to do everything differently 7420 */ 7421 if (s && s->cpu_sheaves) { 7422 free_to_pcs_bulk(s, size, p); 7423 return; 7424 } 7425 7426 do { 7427 struct detached_freelist df; 7428 7429 size = build_detached_freelist(s, size, p, &df); 7430 if (!df.slab) 7431 continue; 7432 7433 slab_free_bulk(df.s, df.slab, df.freelist, df.tail, &p[size], 7434 df.cnt, _RET_IP_); 7435 } while (likely(size)); 7436 } 7437 EXPORT_SYMBOL(kmem_cache_free_bulk); 7438 7439 static inline 7440 int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, 7441 void **p) 7442 { 7443 struct kmem_cache_cpu *c; 7444 unsigned long irqflags; 7445 int i; 7446 7447 /* 7448 * Drain objects in the per cpu slab, while disabling local 7449 * IRQs, which protects against PREEMPT and interrupts 7450 * handlers invoking normal fastpath. 7451 */ 7452 c = slub_get_cpu_ptr(s->cpu_slab); 7453 local_lock_irqsave(&s->cpu_slab->lock, irqflags); 7454 7455 for (i = 0; i < size; i++) { 7456 void *object = kfence_alloc(s, s->object_size, flags); 7457 7458 if (unlikely(object)) { 7459 p[i] = object; 7460 continue; 7461 } 7462 7463 object = c->freelist; 7464 if (unlikely(!object)) { 7465 /* 7466 * We may have removed an object from c->freelist using 7467 * the fastpath in the previous iteration; in that case, 7468 * c->tid has not been bumped yet. 7469 * Since ___slab_alloc() may reenable interrupts while 7470 * allocating memory, we should bump c->tid now. 7471 */ 7472 c->tid = next_tid(c->tid); 7473 7474 local_unlock_irqrestore(&s->cpu_slab->lock, irqflags); 7475 7476 /* 7477 * Invoking slow path likely have side-effect 7478 * of re-populating per CPU c->freelist 7479 */ 7480 p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE, 7481 _RET_IP_, c, s->object_size); 7482 if (unlikely(!p[i])) 7483 goto error; 7484 7485 c = this_cpu_ptr(s->cpu_slab); 7486 maybe_wipe_obj_freeptr(s, p[i]); 7487 7488 local_lock_irqsave(&s->cpu_slab->lock, irqflags); 7489 7490 continue; /* goto for-loop */ 7491 } 7492 c->freelist = get_freepointer(s, object); 7493 p[i] = object; 7494 maybe_wipe_obj_freeptr(s, p[i]); 7495 stat(s, ALLOC_FASTPATH); 7496 } 7497 c->tid = next_tid(c->tid); 7498 local_unlock_irqrestore(&s->cpu_slab->lock, irqflags); 7499 slub_put_cpu_ptr(s->cpu_slab); 7500 7501 return i; 7502 7503 error: 7504 slub_put_cpu_ptr(s->cpu_slab); 7505 __kmem_cache_free_bulk(s, i, p); 7506 return 0; 7507 7508 } 7509 7510 /* Note that interrupts must be enabled when calling this function. */ 7511 int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size, 7512 void **p) 7513 { 7514 unsigned int i = 0; 7515 7516 if (!size) 7517 return 0; 7518 7519 s = slab_pre_alloc_hook(s, flags); 7520 if (unlikely(!s)) 7521 return 0; 7522 7523 if (s->cpu_sheaves) 7524 i = alloc_from_pcs_bulk(s, size, p); 7525 7526 if (i < size) { 7527 /* 7528 * If we ran out of memory, don't bother with freeing back to 7529 * the percpu sheaves, we have bigger problems. 7530 */ 7531 if (unlikely(__kmem_cache_alloc_bulk(s, flags, size - i, p + i) == 0)) { 7532 if (i > 0) 7533 __kmem_cache_free_bulk(s, i, p); 7534 return 0; 7535 } 7536 } 7537 7538 /* 7539 * memcg and kmem_cache debug support and memory initialization. 7540 * Done outside of the IRQ disabled fastpath loop. 7541 */ 7542 if (unlikely(!slab_post_alloc_hook(s, NULL, flags, size, p, 7543 slab_want_init_on_alloc(flags, s), s->object_size))) { 7544 return 0; 7545 } 7546 7547 return size; 7548 } 7549 EXPORT_SYMBOL(kmem_cache_alloc_bulk_noprof); 7550 7551 /* 7552 * Object placement in a slab is made very easy because we always start at 7553 * offset 0. If we tune the size of the object to the alignment then we can 7554 * get the required alignment by putting one properly sized object after 7555 * another. 7556 * 7557 * Notice that the allocation order determines the sizes of the per cpu 7558 * caches. Each processor has always one slab available for allocations. 7559 * Increasing the allocation order reduces the number of times that slabs 7560 * must be moved on and off the partial lists and is therefore a factor in 7561 * locking overhead. 7562 */ 7563 7564 /* 7565 * Minimum / Maximum order of slab pages. This influences locking overhead 7566 * and slab fragmentation. A higher order reduces the number of partial slabs 7567 * and increases the number of allocations possible without having to 7568 * take the list_lock. 7569 */ 7570 static unsigned int slub_min_order; 7571 static unsigned int slub_max_order = 7572 IS_ENABLED(CONFIG_SLUB_TINY) ? 1 : PAGE_ALLOC_COSTLY_ORDER; 7573 static unsigned int slub_min_objects; 7574 7575 /* 7576 * Calculate the order of allocation given an slab object size. 7577 * 7578 * The order of allocation has significant impact on performance and other 7579 * system components. Generally order 0 allocations should be preferred since 7580 * order 0 does not cause fragmentation in the page allocator. Larger objects 7581 * be problematic to put into order 0 slabs because there may be too much 7582 * unused space left. We go to a higher order if more than 1/16th of the slab 7583 * would be wasted. 7584 * 7585 * In order to reach satisfactory performance we must ensure that a minimum 7586 * number of objects is in one slab. Otherwise we may generate too much 7587 * activity on the partial lists which requires taking the list_lock. This is 7588 * less a concern for large slabs though which are rarely used. 7589 * 7590 * slab_max_order specifies the order where we begin to stop considering the 7591 * number of objects in a slab as critical. If we reach slab_max_order then 7592 * we try to keep the page order as low as possible. So we accept more waste 7593 * of space in favor of a small page order. 7594 * 7595 * Higher order allocations also allow the placement of more objects in a 7596 * slab and thereby reduce object handling overhead. If the user has 7597 * requested a higher minimum order then we start with that one instead of 7598 * the smallest order which will fit the object. 7599 */ 7600 static inline unsigned int calc_slab_order(unsigned int size, 7601 unsigned int min_order, unsigned int max_order, 7602 unsigned int fract_leftover) 7603 { 7604 unsigned int order; 7605 7606 for (order = min_order; order <= max_order; order++) { 7607 7608 unsigned int slab_size = (unsigned int)PAGE_SIZE << order; 7609 unsigned int rem; 7610 7611 rem = slab_size % size; 7612 7613 if (rem <= slab_size / fract_leftover) 7614 break; 7615 } 7616 7617 return order; 7618 } 7619 7620 static inline int calculate_order(unsigned int size) 7621 { 7622 unsigned int order; 7623 unsigned int min_objects; 7624 unsigned int max_objects; 7625 unsigned int min_order; 7626 7627 min_objects = slub_min_objects; 7628 if (!min_objects) { 7629 /* 7630 * Some architectures will only update present cpus when 7631 * onlining them, so don't trust the number if it's just 1. But 7632 * we also don't want to use nr_cpu_ids always, as on some other 7633 * architectures, there can be many possible cpus, but never 7634 * onlined. Here we compromise between trying to avoid too high 7635 * order on systems that appear larger than they are, and too 7636 * low order on systems that appear smaller than they are. 7637 */ 7638 unsigned int nr_cpus = num_present_cpus(); 7639 if (nr_cpus <= 1) 7640 nr_cpus = nr_cpu_ids; 7641 min_objects = 4 * (fls(nr_cpus) + 1); 7642 } 7643 /* min_objects can't be 0 because get_order(0) is undefined */ 7644 max_objects = max(order_objects(slub_max_order, size), 1U); 7645 min_objects = min(min_objects, max_objects); 7646 7647 min_order = max_t(unsigned int, slub_min_order, 7648 get_order(min_objects * size)); 7649 if (order_objects(min_order, size) > MAX_OBJS_PER_PAGE) 7650 return get_order(size * MAX_OBJS_PER_PAGE) - 1; 7651 7652 /* 7653 * Attempt to find best configuration for a slab. This works by first 7654 * attempting to generate a layout with the best possible configuration 7655 * and backing off gradually. 7656 * 7657 * We start with accepting at most 1/16 waste and try to find the 7658 * smallest order from min_objects-derived/slab_min_order up to 7659 * slab_max_order that will satisfy the constraint. Note that increasing 7660 * the order can only result in same or less fractional waste, not more. 7661 * 7662 * If that fails, we increase the acceptable fraction of waste and try 7663 * again. The last iteration with fraction of 1/2 would effectively 7664 * accept any waste and give us the order determined by min_objects, as 7665 * long as at least single object fits within slab_max_order. 7666 */ 7667 for (unsigned int fraction = 16; fraction > 1; fraction /= 2) { 7668 order = calc_slab_order(size, min_order, slub_max_order, 7669 fraction); 7670 if (order <= slub_max_order) 7671 return order; 7672 } 7673 7674 /* 7675 * Doh this slab cannot be placed using slab_max_order. 7676 */ 7677 order = get_order(size); 7678 if (order <= MAX_PAGE_ORDER) 7679 return order; 7680 return -ENOSYS; 7681 } 7682 7683 static void 7684 init_kmem_cache_node(struct kmem_cache_node *n, struct node_barn *barn) 7685 { 7686 n->nr_partial = 0; 7687 spin_lock_init(&n->list_lock); 7688 INIT_LIST_HEAD(&n->partial); 7689 #ifdef CONFIG_SLUB_DEBUG 7690 atomic_long_set(&n->nr_slabs, 0); 7691 atomic_long_set(&n->total_objects, 0); 7692 INIT_LIST_HEAD(&n->full); 7693 #endif 7694 n->barn = barn; 7695 if (barn) 7696 barn_init(barn); 7697 } 7698 7699 static inline int alloc_kmem_cache_cpus(struct kmem_cache *s) 7700 { 7701 BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE < 7702 NR_KMALLOC_TYPES * KMALLOC_SHIFT_HIGH * 7703 sizeof(struct kmem_cache_cpu)); 7704 7705 /* 7706 * Must align to double word boundary for the double cmpxchg 7707 * instructions to work; see __pcpu_double_call_return_bool(). 7708 */ 7709 s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu), 7710 2 * sizeof(void *)); 7711 7712 if (!s->cpu_slab) 7713 return 0; 7714 7715 init_kmem_cache_cpus(s); 7716 7717 return 1; 7718 } 7719 7720 static int init_percpu_sheaves(struct kmem_cache *s) 7721 { 7722 int cpu; 7723 7724 for_each_possible_cpu(cpu) { 7725 struct slub_percpu_sheaves *pcs; 7726 7727 pcs = per_cpu_ptr(s->cpu_sheaves, cpu); 7728 7729 local_trylock_init(&pcs->lock); 7730 7731 pcs->main = alloc_empty_sheaf(s, GFP_KERNEL); 7732 7733 if (!pcs->main) 7734 return -ENOMEM; 7735 } 7736 7737 return 0; 7738 } 7739 7740 static struct kmem_cache *kmem_cache_node; 7741 7742 /* 7743 * No kmalloc_node yet so do it by hand. We know that this is the first 7744 * slab on the node for this slabcache. There are no concurrent accesses 7745 * possible. 7746 * 7747 * Note that this function only works on the kmem_cache_node 7748 * when allocating for the kmem_cache_node. This is used for bootstrapping 7749 * memory on a fresh node that has no slab structures yet. 7750 */ 7751 static void early_kmem_cache_node_alloc(int node) 7752 { 7753 struct slab *slab; 7754 struct kmem_cache_node *n; 7755 7756 BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node)); 7757 7758 slab = new_slab(kmem_cache_node, GFP_NOWAIT, node); 7759 7760 BUG_ON(!slab); 7761 if (slab_nid(slab) != node) { 7762 pr_err("SLUB: Unable to allocate memory from node %d\n", node); 7763 pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n"); 7764 } 7765 7766 n = slab->freelist; 7767 BUG_ON(!n); 7768 #ifdef CONFIG_SLUB_DEBUG 7769 init_object(kmem_cache_node, n, SLUB_RED_ACTIVE); 7770 #endif 7771 n = kasan_slab_alloc(kmem_cache_node, n, GFP_KERNEL, false); 7772 slab->freelist = get_freepointer(kmem_cache_node, n); 7773 slab->inuse = 1; 7774 kmem_cache_node->node[node] = n; 7775 init_kmem_cache_node(n, NULL); 7776 inc_slabs_node(kmem_cache_node, node, slab->objects); 7777 7778 /* 7779 * No locks need to be taken here as it has just been 7780 * initialized and there is no concurrent access. 7781 */ 7782 __add_partial(n, slab, DEACTIVATE_TO_HEAD); 7783 } 7784 7785 static void free_kmem_cache_nodes(struct kmem_cache *s) 7786 { 7787 int node; 7788 struct kmem_cache_node *n; 7789 7790 for_each_kmem_cache_node(s, node, n) { 7791 if (n->barn) { 7792 WARN_ON(n->barn->nr_full); 7793 WARN_ON(n->barn->nr_empty); 7794 kfree(n->barn); 7795 n->barn = NULL; 7796 } 7797 7798 s->node[node] = NULL; 7799 kmem_cache_free(kmem_cache_node, n); 7800 } 7801 } 7802 7803 void __kmem_cache_release(struct kmem_cache *s) 7804 { 7805 cache_random_seq_destroy(s); 7806 if (s->cpu_sheaves) 7807 pcs_destroy(s); 7808 #ifdef CONFIG_PREEMPT_RT 7809 if (s->cpu_slab) 7810 lockdep_unregister_key(&s->lock_key); 7811 #endif 7812 free_percpu(s->cpu_slab); 7813 free_kmem_cache_nodes(s); 7814 } 7815 7816 static int init_kmem_cache_nodes(struct kmem_cache *s) 7817 { 7818 int node; 7819 7820 for_each_node_mask(node, slab_nodes) { 7821 struct kmem_cache_node *n; 7822 struct node_barn *barn = NULL; 7823 7824 if (slab_state == DOWN) { 7825 early_kmem_cache_node_alloc(node); 7826 continue; 7827 } 7828 7829 if (s->cpu_sheaves) { 7830 barn = kmalloc_node(sizeof(*barn), GFP_KERNEL, node); 7831 7832 if (!barn) 7833 return 0; 7834 } 7835 7836 n = kmem_cache_alloc_node(kmem_cache_node, 7837 GFP_KERNEL, node); 7838 if (!n) { 7839 kfree(barn); 7840 return 0; 7841 } 7842 7843 init_kmem_cache_node(n, barn); 7844 7845 s->node[node] = n; 7846 } 7847 return 1; 7848 } 7849 7850 static void set_cpu_partial(struct kmem_cache *s) 7851 { 7852 #ifdef CONFIG_SLUB_CPU_PARTIAL 7853 unsigned int nr_objects; 7854 7855 /* 7856 * cpu_partial determined the maximum number of objects kept in the 7857 * per cpu partial lists of a processor. 7858 * 7859 * Per cpu partial lists mainly contain slabs that just have one 7860 * object freed. If they are used for allocation then they can be 7861 * filled up again with minimal effort. The slab will never hit the 7862 * per node partial lists and therefore no locking will be required. 7863 * 7864 * For backwards compatibility reasons, this is determined as number 7865 * of objects, even though we now limit maximum number of pages, see 7866 * slub_set_cpu_partial() 7867 */ 7868 if (!kmem_cache_has_cpu_partial(s)) 7869 nr_objects = 0; 7870 else if (s->size >= PAGE_SIZE) 7871 nr_objects = 6; 7872 else if (s->size >= 1024) 7873 nr_objects = 24; 7874 else if (s->size >= 256) 7875 nr_objects = 52; 7876 else 7877 nr_objects = 120; 7878 7879 slub_set_cpu_partial(s, nr_objects); 7880 #endif 7881 } 7882 7883 /* 7884 * calculate_sizes() determines the order and the distribution of data within 7885 * a slab object. 7886 */ 7887 static int calculate_sizes(struct kmem_cache_args *args, struct kmem_cache *s) 7888 { 7889 slab_flags_t flags = s->flags; 7890 unsigned int size = s->object_size; 7891 unsigned int order; 7892 7893 /* 7894 * Round up object size to the next word boundary. We can only 7895 * place the free pointer at word boundaries and this determines 7896 * the possible location of the free pointer. 7897 */ 7898 size = ALIGN(size, sizeof(void *)); 7899 7900 #ifdef CONFIG_SLUB_DEBUG 7901 /* 7902 * Determine if we can poison the object itself. If the user of 7903 * the slab may touch the object after free or before allocation 7904 * then we should never poison the object itself. 7905 */ 7906 if ((flags & SLAB_POISON) && !(flags & SLAB_TYPESAFE_BY_RCU) && 7907 !s->ctor) 7908 s->flags |= __OBJECT_POISON; 7909 else 7910 s->flags &= ~__OBJECT_POISON; 7911 7912 7913 /* 7914 * If we are Redzoning then check if there is some space between the 7915 * end of the object and the free pointer. If not then add an 7916 * additional word to have some bytes to store Redzone information. 7917 */ 7918 if ((flags & SLAB_RED_ZONE) && size == s->object_size) 7919 size += sizeof(void *); 7920 #endif 7921 7922 /* 7923 * With that we have determined the number of bytes in actual use 7924 * by the object and redzoning. 7925 */ 7926 s->inuse = size; 7927 7928 if (((flags & SLAB_TYPESAFE_BY_RCU) && !args->use_freeptr_offset) || 7929 (flags & SLAB_POISON) || s->ctor || 7930 ((flags & SLAB_RED_ZONE) && 7931 (s->object_size < sizeof(void *) || slub_debug_orig_size(s)))) { 7932 /* 7933 * Relocate free pointer after the object if it is not 7934 * permitted to overwrite the first word of the object on 7935 * kmem_cache_free. 7936 * 7937 * This is the case if we do RCU, have a constructor or 7938 * destructor, are poisoning the objects, or are 7939 * redzoning an object smaller than sizeof(void *) or are 7940 * redzoning an object with slub_debug_orig_size() enabled, 7941 * in which case the right redzone may be extended. 7942 * 7943 * The assumption that s->offset >= s->inuse means free 7944 * pointer is outside of the object is used in the 7945 * freeptr_outside_object() function. If that is no 7946 * longer true, the function needs to be modified. 7947 */ 7948 s->offset = size; 7949 size += sizeof(void *); 7950 } else if ((flags & SLAB_TYPESAFE_BY_RCU) && args->use_freeptr_offset) { 7951 s->offset = args->freeptr_offset; 7952 } else { 7953 /* 7954 * Store freelist pointer near middle of object to keep 7955 * it away from the edges of the object to avoid small 7956 * sized over/underflows from neighboring allocations. 7957 */ 7958 s->offset = ALIGN_DOWN(s->object_size / 2, sizeof(void *)); 7959 } 7960 7961 #ifdef CONFIG_SLUB_DEBUG 7962 if (flags & SLAB_STORE_USER) { 7963 /* 7964 * Need to store information about allocs and frees after 7965 * the object. 7966 */ 7967 size += 2 * sizeof(struct track); 7968 7969 /* Save the original kmalloc request size */ 7970 if (flags & SLAB_KMALLOC) 7971 size += sizeof(unsigned long); 7972 } 7973 #endif 7974 7975 kasan_cache_create(s, &size, &s->flags); 7976 #ifdef CONFIG_SLUB_DEBUG 7977 if (flags & SLAB_RED_ZONE) { 7978 /* 7979 * Add some empty padding so that we can catch 7980 * overwrites from earlier objects rather than let 7981 * tracking information or the free pointer be 7982 * corrupted if a user writes before the start 7983 * of the object. 7984 */ 7985 size += sizeof(void *); 7986 7987 s->red_left_pad = sizeof(void *); 7988 s->red_left_pad = ALIGN(s->red_left_pad, s->align); 7989 size += s->red_left_pad; 7990 } 7991 #endif 7992 7993 /* 7994 * SLUB stores one object immediately after another beginning from 7995 * offset 0. In order to align the objects we have to simply size 7996 * each object to conform to the alignment. 7997 */ 7998 size = ALIGN(size, s->align); 7999 s->size = size; 8000 s->reciprocal_size = reciprocal_value(size); 8001 order = calculate_order(size); 8002 8003 if ((int)order < 0) 8004 return 0; 8005 8006 s->allocflags = __GFP_COMP; 8007 8008 if (s->flags & SLAB_CACHE_DMA) 8009 s->allocflags |= GFP_DMA; 8010 8011 if (s->flags & SLAB_CACHE_DMA32) 8012 s->allocflags |= GFP_DMA32; 8013 8014 if (s->flags & SLAB_RECLAIM_ACCOUNT) 8015 s->allocflags |= __GFP_RECLAIMABLE; 8016 8017 /* 8018 * Determine the number of objects per slab 8019 */ 8020 s->oo = oo_make(order, size); 8021 s->min = oo_make(get_order(size), size); 8022 8023 return !!oo_objects(s->oo); 8024 } 8025 8026 static void list_slab_objects(struct kmem_cache *s, struct slab *slab) 8027 { 8028 #ifdef CONFIG_SLUB_DEBUG 8029 void *addr = slab_address(slab); 8030 void *p; 8031 8032 if (!slab_add_kunit_errors()) 8033 slab_bug(s, "Objects remaining on __kmem_cache_shutdown()"); 8034 8035 spin_lock(&object_map_lock); 8036 __fill_map(object_map, s, slab); 8037 8038 for_each_object(p, s, addr, slab->objects) { 8039 8040 if (!test_bit(__obj_to_index(s, addr, p), object_map)) { 8041 if (slab_add_kunit_errors()) 8042 continue; 8043 pr_err("Object 0x%p @offset=%tu\n", p, p - addr); 8044 print_tracking(s, p); 8045 } 8046 } 8047 spin_unlock(&object_map_lock); 8048 8049 __slab_err(slab); 8050 #endif 8051 } 8052 8053 /* 8054 * Attempt to free all partial slabs on a node. 8055 * This is called from __kmem_cache_shutdown(). We must take list_lock 8056 * because sysfs file might still access partial list after the shutdowning. 8057 */ 8058 static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n) 8059 { 8060 LIST_HEAD(discard); 8061 struct slab *slab, *h; 8062 8063 BUG_ON(irqs_disabled()); 8064 spin_lock_irq(&n->list_lock); 8065 list_for_each_entry_safe(slab, h, &n->partial, slab_list) { 8066 if (!slab->inuse) { 8067 remove_partial(n, slab); 8068 list_add(&slab->slab_list, &discard); 8069 } else { 8070 list_slab_objects(s, slab); 8071 } 8072 } 8073 spin_unlock_irq(&n->list_lock); 8074 8075 list_for_each_entry_safe(slab, h, &discard, slab_list) 8076 discard_slab(s, slab); 8077 } 8078 8079 bool __kmem_cache_empty(struct kmem_cache *s) 8080 { 8081 int node; 8082 struct kmem_cache_node *n; 8083 8084 for_each_kmem_cache_node(s, node, n) 8085 if (n->nr_partial || node_nr_slabs(n)) 8086 return false; 8087 return true; 8088 } 8089 8090 /* 8091 * Release all resources used by a slab cache. 8092 */ 8093 int __kmem_cache_shutdown(struct kmem_cache *s) 8094 { 8095 int node; 8096 struct kmem_cache_node *n; 8097 8098 flush_all_cpus_locked(s); 8099 8100 /* we might have rcu sheaves in flight */ 8101 if (s->cpu_sheaves) 8102 rcu_barrier(); 8103 8104 /* Attempt to free all objects */ 8105 for_each_kmem_cache_node(s, node, n) { 8106 if (n->barn) 8107 barn_shrink(s, n->barn); 8108 free_partial(s, n); 8109 if (n->nr_partial || node_nr_slabs(n)) 8110 return 1; 8111 } 8112 return 0; 8113 } 8114 8115 #ifdef CONFIG_PRINTK 8116 void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab) 8117 { 8118 void *base; 8119 int __maybe_unused i; 8120 unsigned int objnr; 8121 void *objp; 8122 void *objp0; 8123 struct kmem_cache *s = slab->slab_cache; 8124 struct track __maybe_unused *trackp; 8125 8126 kpp->kp_ptr = object; 8127 kpp->kp_slab = slab; 8128 kpp->kp_slab_cache = s; 8129 base = slab_address(slab); 8130 objp0 = kasan_reset_tag(object); 8131 #ifdef CONFIG_SLUB_DEBUG 8132 objp = restore_red_left(s, objp0); 8133 #else 8134 objp = objp0; 8135 #endif 8136 objnr = obj_to_index(s, slab, objp); 8137 kpp->kp_data_offset = (unsigned long)((char *)objp0 - (char *)objp); 8138 objp = base + s->size * objnr; 8139 kpp->kp_objp = objp; 8140 if (WARN_ON_ONCE(objp < base || objp >= base + slab->objects * s->size 8141 || (objp - base) % s->size) || 8142 !(s->flags & SLAB_STORE_USER)) 8143 return; 8144 #ifdef CONFIG_SLUB_DEBUG 8145 objp = fixup_red_left(s, objp); 8146 trackp = get_track(s, objp, TRACK_ALLOC); 8147 kpp->kp_ret = (void *)trackp->addr; 8148 #ifdef CONFIG_STACKDEPOT 8149 { 8150 depot_stack_handle_t handle; 8151 unsigned long *entries; 8152 unsigned int nr_entries; 8153 8154 handle = READ_ONCE(trackp->handle); 8155 if (handle) { 8156 nr_entries = stack_depot_fetch(handle, &entries); 8157 for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++) 8158 kpp->kp_stack[i] = (void *)entries[i]; 8159 } 8160 8161 trackp = get_track(s, objp, TRACK_FREE); 8162 handle = READ_ONCE(trackp->handle); 8163 if (handle) { 8164 nr_entries = stack_depot_fetch(handle, &entries); 8165 for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++) 8166 kpp->kp_free_stack[i] = (void *)entries[i]; 8167 } 8168 } 8169 #endif 8170 #endif 8171 } 8172 #endif 8173 8174 /******************************************************************** 8175 * Kmalloc subsystem 8176 *******************************************************************/ 8177 8178 static int __init setup_slub_min_order(char *str) 8179 { 8180 get_option(&str, (int *)&slub_min_order); 8181 8182 if (slub_min_order > slub_max_order) 8183 slub_max_order = slub_min_order; 8184 8185 return 1; 8186 } 8187 8188 __setup("slab_min_order=", setup_slub_min_order); 8189 __setup_param("slub_min_order=", slub_min_order, setup_slub_min_order, 0); 8190 8191 8192 static int __init setup_slub_max_order(char *str) 8193 { 8194 get_option(&str, (int *)&slub_max_order); 8195 slub_max_order = min_t(unsigned int, slub_max_order, MAX_PAGE_ORDER); 8196 8197 if (slub_min_order > slub_max_order) 8198 slub_min_order = slub_max_order; 8199 8200 return 1; 8201 } 8202 8203 __setup("slab_max_order=", setup_slub_max_order); 8204 __setup_param("slub_max_order=", slub_max_order, setup_slub_max_order, 0); 8205 8206 static int __init setup_slub_min_objects(char *str) 8207 { 8208 get_option(&str, (int *)&slub_min_objects); 8209 8210 return 1; 8211 } 8212 8213 __setup("slab_min_objects=", setup_slub_min_objects); 8214 __setup_param("slub_min_objects=", slub_min_objects, setup_slub_min_objects, 0); 8215 8216 #ifdef CONFIG_NUMA 8217 static int __init setup_slab_strict_numa(char *str) 8218 { 8219 if (nr_node_ids > 1) { 8220 static_branch_enable(&strict_numa); 8221 pr_info("SLUB: Strict NUMA enabled.\n"); 8222 } else { 8223 pr_warn("slab_strict_numa parameter set on non NUMA system.\n"); 8224 } 8225 8226 return 1; 8227 } 8228 8229 __setup("slab_strict_numa", setup_slab_strict_numa); 8230 #endif 8231 8232 8233 #ifdef CONFIG_HARDENED_USERCOPY 8234 /* 8235 * Rejects incorrectly sized objects and objects that are to be copied 8236 * to/from userspace but do not fall entirely within the containing slab 8237 * cache's usercopy region. 8238 * 8239 * Returns NULL if check passes, otherwise const char * to name of cache 8240 * to indicate an error. 8241 */ 8242 void __check_heap_object(const void *ptr, unsigned long n, 8243 const struct slab *slab, bool to_user) 8244 { 8245 struct kmem_cache *s; 8246 unsigned int offset; 8247 bool is_kfence = is_kfence_address(ptr); 8248 8249 ptr = kasan_reset_tag(ptr); 8250 8251 /* Find object and usable object size. */ 8252 s = slab->slab_cache; 8253 8254 /* Reject impossible pointers. */ 8255 if (ptr < slab_address(slab)) 8256 usercopy_abort("SLUB object not in SLUB page?!", NULL, 8257 to_user, 0, n); 8258 8259 /* Find offset within object. */ 8260 if (is_kfence) 8261 offset = ptr - kfence_object_start(ptr); 8262 else 8263 offset = (ptr - slab_address(slab)) % s->size; 8264 8265 /* Adjust for redzone and reject if within the redzone. */ 8266 if (!is_kfence && kmem_cache_debug_flags(s, SLAB_RED_ZONE)) { 8267 if (offset < s->red_left_pad) 8268 usercopy_abort("SLUB object in left red zone", 8269 s->name, to_user, offset, n); 8270 offset -= s->red_left_pad; 8271 } 8272 8273 /* Allow address range falling entirely within usercopy region. */ 8274 if (offset >= s->useroffset && 8275 offset - s->useroffset <= s->usersize && 8276 n <= s->useroffset - offset + s->usersize) 8277 return; 8278 8279 usercopy_abort("SLUB object", s->name, to_user, offset, n); 8280 } 8281 #endif /* CONFIG_HARDENED_USERCOPY */ 8282 8283 #define SHRINK_PROMOTE_MAX 32 8284 8285 /* 8286 * kmem_cache_shrink discards empty slabs and promotes the slabs filled 8287 * up most to the head of the partial lists. New allocations will then 8288 * fill those up and thus they can be removed from the partial lists. 8289 * 8290 * The slabs with the least items are placed last. This results in them 8291 * being allocated from last increasing the chance that the last objects 8292 * are freed in them. 8293 */ 8294 static int __kmem_cache_do_shrink(struct kmem_cache *s) 8295 { 8296 int node; 8297 int i; 8298 struct kmem_cache_node *n; 8299 struct slab *slab; 8300 struct slab *t; 8301 struct list_head discard; 8302 struct list_head promote[SHRINK_PROMOTE_MAX]; 8303 unsigned long flags; 8304 int ret = 0; 8305 8306 for_each_kmem_cache_node(s, node, n) { 8307 INIT_LIST_HEAD(&discard); 8308 for (i = 0; i < SHRINK_PROMOTE_MAX; i++) 8309 INIT_LIST_HEAD(promote + i); 8310 8311 if (n->barn) 8312 barn_shrink(s, n->barn); 8313 8314 spin_lock_irqsave(&n->list_lock, flags); 8315 8316 /* 8317 * Build lists of slabs to discard or promote. 8318 * 8319 * Note that concurrent frees may occur while we hold the 8320 * list_lock. slab->inuse here is the upper limit. 8321 */ 8322 list_for_each_entry_safe(slab, t, &n->partial, slab_list) { 8323 int free = slab->objects - slab->inuse; 8324 8325 /* Do not reread slab->inuse */ 8326 barrier(); 8327 8328 /* We do not keep full slabs on the list */ 8329 BUG_ON(free <= 0); 8330 8331 if (free == slab->objects) { 8332 list_move(&slab->slab_list, &discard); 8333 slab_clear_node_partial(slab); 8334 n->nr_partial--; 8335 dec_slabs_node(s, node, slab->objects); 8336 } else if (free <= SHRINK_PROMOTE_MAX) 8337 list_move(&slab->slab_list, promote + free - 1); 8338 } 8339 8340 /* 8341 * Promote the slabs filled up most to the head of the 8342 * partial list. 8343 */ 8344 for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--) 8345 list_splice(promote + i, &n->partial); 8346 8347 spin_unlock_irqrestore(&n->list_lock, flags); 8348 8349 /* Release empty slabs */ 8350 list_for_each_entry_safe(slab, t, &discard, slab_list) 8351 free_slab(s, slab); 8352 8353 if (node_nr_slabs(n)) 8354 ret = 1; 8355 } 8356 8357 return ret; 8358 } 8359 8360 int __kmem_cache_shrink(struct kmem_cache *s) 8361 { 8362 flush_all(s); 8363 return __kmem_cache_do_shrink(s); 8364 } 8365 8366 static int slab_mem_going_offline_callback(void) 8367 { 8368 struct kmem_cache *s; 8369 8370 mutex_lock(&slab_mutex); 8371 list_for_each_entry(s, &slab_caches, list) { 8372 flush_all_cpus_locked(s); 8373 __kmem_cache_do_shrink(s); 8374 } 8375 mutex_unlock(&slab_mutex); 8376 8377 return 0; 8378 } 8379 8380 static int slab_mem_going_online_callback(int nid) 8381 { 8382 struct kmem_cache_node *n; 8383 struct kmem_cache *s; 8384 int ret = 0; 8385 8386 /* 8387 * We are bringing a node online. No memory is available yet. We must 8388 * allocate a kmem_cache_node structure in order to bring the node 8389 * online. 8390 */ 8391 mutex_lock(&slab_mutex); 8392 list_for_each_entry(s, &slab_caches, list) { 8393 struct node_barn *barn = NULL; 8394 8395 /* 8396 * The structure may already exist if the node was previously 8397 * onlined and offlined. 8398 */ 8399 if (get_node(s, nid)) 8400 continue; 8401 8402 if (s->cpu_sheaves) { 8403 barn = kmalloc_node(sizeof(*barn), GFP_KERNEL, nid); 8404 8405 if (!barn) { 8406 ret = -ENOMEM; 8407 goto out; 8408 } 8409 } 8410 8411 /* 8412 * XXX: kmem_cache_alloc_node will fallback to other nodes 8413 * since memory is not yet available from the node that 8414 * is brought up. 8415 */ 8416 n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL); 8417 if (!n) { 8418 kfree(barn); 8419 ret = -ENOMEM; 8420 goto out; 8421 } 8422 8423 init_kmem_cache_node(n, barn); 8424 8425 s->node[nid] = n; 8426 } 8427 /* 8428 * Any cache created after this point will also have kmem_cache_node 8429 * initialized for the new node. 8430 */ 8431 node_set(nid, slab_nodes); 8432 out: 8433 mutex_unlock(&slab_mutex); 8434 return ret; 8435 } 8436 8437 static int slab_memory_callback(struct notifier_block *self, 8438 unsigned long action, void *arg) 8439 { 8440 struct node_notify *nn = arg; 8441 int nid = nn->nid; 8442 int ret = 0; 8443 8444 switch (action) { 8445 case NODE_ADDING_FIRST_MEMORY: 8446 ret = slab_mem_going_online_callback(nid); 8447 break; 8448 case NODE_REMOVING_LAST_MEMORY: 8449 ret = slab_mem_going_offline_callback(); 8450 break; 8451 } 8452 if (ret) 8453 ret = notifier_from_errno(ret); 8454 else 8455 ret = NOTIFY_OK; 8456 return ret; 8457 } 8458 8459 /******************************************************************** 8460 * Basic setup of slabs 8461 *******************************************************************/ 8462 8463 /* 8464 * Used for early kmem_cache structures that were allocated using 8465 * the page allocator. Allocate them properly then fix up the pointers 8466 * that may be pointing to the wrong kmem_cache structure. 8467 */ 8468 8469 static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache) 8470 { 8471 int node; 8472 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT); 8473 struct kmem_cache_node *n; 8474 8475 memcpy(s, static_cache, kmem_cache->object_size); 8476 8477 /* 8478 * This runs very early, and only the boot processor is supposed to be 8479 * up. Even if it weren't true, IRQs are not up so we couldn't fire 8480 * IPIs around. 8481 */ 8482 __flush_cpu_slab(s, smp_processor_id()); 8483 for_each_kmem_cache_node(s, node, n) { 8484 struct slab *p; 8485 8486 list_for_each_entry(p, &n->partial, slab_list) 8487 p->slab_cache = s; 8488 8489 #ifdef CONFIG_SLUB_DEBUG 8490 list_for_each_entry(p, &n->full, slab_list) 8491 p->slab_cache = s; 8492 #endif 8493 } 8494 list_add(&s->list, &slab_caches); 8495 return s; 8496 } 8497 8498 void __init kmem_cache_init(void) 8499 { 8500 static __initdata struct kmem_cache boot_kmem_cache, 8501 boot_kmem_cache_node; 8502 int node; 8503 8504 if (debug_guardpage_minorder()) 8505 slub_max_order = 0; 8506 8507 /* Inform pointer hashing choice about slub debugging state. */ 8508 hash_pointers_finalize(__slub_debug_enabled()); 8509 8510 kmem_cache_node = &boot_kmem_cache_node; 8511 kmem_cache = &boot_kmem_cache; 8512 8513 /* 8514 * Initialize the nodemask for which we will allocate per node 8515 * structures. Here we don't need taking slab_mutex yet. 8516 */ 8517 for_each_node_state(node, N_MEMORY) 8518 node_set(node, slab_nodes); 8519 8520 create_boot_cache(kmem_cache_node, "kmem_cache_node", 8521 sizeof(struct kmem_cache_node), 8522 SLAB_HWCACHE_ALIGN | SLAB_NO_OBJ_EXT, 0, 0); 8523 8524 hotplug_node_notifier(slab_memory_callback, SLAB_CALLBACK_PRI); 8525 8526 /* Able to allocate the per node structures */ 8527 slab_state = PARTIAL; 8528 8529 create_boot_cache(kmem_cache, "kmem_cache", 8530 offsetof(struct kmem_cache, node) + 8531 nr_node_ids * sizeof(struct kmem_cache_node *), 8532 SLAB_HWCACHE_ALIGN | SLAB_NO_OBJ_EXT, 0, 0); 8533 8534 kmem_cache = bootstrap(&boot_kmem_cache); 8535 kmem_cache_node = bootstrap(&boot_kmem_cache_node); 8536 8537 /* Now we can use the kmem_cache to allocate kmalloc slabs */ 8538 setup_kmalloc_cache_index_table(); 8539 create_kmalloc_caches(); 8540 8541 /* Setup random freelists for each cache */ 8542 init_freelist_randomization(); 8543 8544 cpuhp_setup_state_nocalls(CPUHP_SLUB_DEAD, "slub:dead", NULL, 8545 slub_cpu_dead); 8546 8547 pr_info("SLUB: HWalign=%d, Order=%u-%u, MinObjects=%u, CPUs=%u, Nodes=%u\n", 8548 cache_line_size(), 8549 slub_min_order, slub_max_order, slub_min_objects, 8550 nr_cpu_ids, nr_node_ids); 8551 } 8552 8553 void __init kmem_cache_init_late(void) 8554 { 8555 flushwq = alloc_workqueue("slub_flushwq", WQ_MEM_RECLAIM, 0); 8556 WARN_ON(!flushwq); 8557 #ifdef CONFIG_SLAB_FREELIST_RANDOM 8558 prandom_init_once(&slab_rnd_state); 8559 #endif 8560 } 8561 8562 struct kmem_cache * 8563 __kmem_cache_alias(const char *name, unsigned int size, unsigned int align, 8564 slab_flags_t flags, void (*ctor)(void *)) 8565 { 8566 struct kmem_cache *s; 8567 8568 s = find_mergeable(size, align, flags, name, ctor); 8569 if (s) { 8570 if (sysfs_slab_alias(s, name)) 8571 pr_err("SLUB: Unable to add cache alias %s to sysfs\n", 8572 name); 8573 8574 s->refcount++; 8575 8576 /* 8577 * Adjust the object sizes so that we clear 8578 * the complete object on kzalloc. 8579 */ 8580 s->object_size = max(s->object_size, size); 8581 s->inuse = max(s->inuse, ALIGN(size, sizeof(void *))); 8582 } 8583 8584 return s; 8585 } 8586 8587 int do_kmem_cache_create(struct kmem_cache *s, const char *name, 8588 unsigned int size, struct kmem_cache_args *args, 8589 slab_flags_t flags) 8590 { 8591 int err = -EINVAL; 8592 8593 s->name = name; 8594 s->size = s->object_size = size; 8595 8596 s->flags = kmem_cache_flags(flags, s->name); 8597 #ifdef CONFIG_SLAB_FREELIST_HARDENED 8598 s->random = get_random_long(); 8599 #endif 8600 s->align = args->align; 8601 s->ctor = args->ctor; 8602 #ifdef CONFIG_HARDENED_USERCOPY 8603 s->useroffset = args->useroffset; 8604 s->usersize = args->usersize; 8605 #endif 8606 8607 if (!calculate_sizes(args, s)) 8608 goto out; 8609 if (disable_higher_order_debug) { 8610 /* 8611 * Disable debugging flags that store metadata if the min slab 8612 * order increased. 8613 */ 8614 if (get_order(s->size) > get_order(s->object_size)) { 8615 s->flags &= ~DEBUG_METADATA_FLAGS; 8616 s->offset = 0; 8617 if (!calculate_sizes(args, s)) 8618 goto out; 8619 } 8620 } 8621 8622 #ifdef system_has_freelist_aba 8623 if (system_has_freelist_aba() && !(s->flags & SLAB_NO_CMPXCHG)) { 8624 /* Enable fast mode */ 8625 s->flags |= __CMPXCHG_DOUBLE; 8626 } 8627 #endif 8628 8629 /* 8630 * The larger the object size is, the more slabs we want on the partial 8631 * list to avoid pounding the page allocator excessively. 8632 */ 8633 s->min_partial = min_t(unsigned long, MAX_PARTIAL, ilog2(s->size) / 2); 8634 s->min_partial = max_t(unsigned long, MIN_PARTIAL, s->min_partial); 8635 8636 set_cpu_partial(s); 8637 8638 if (args->sheaf_capacity && !IS_ENABLED(CONFIG_SLUB_TINY) 8639 && !(s->flags & SLAB_DEBUG_FLAGS)) { 8640 s->cpu_sheaves = alloc_percpu(struct slub_percpu_sheaves); 8641 if (!s->cpu_sheaves) { 8642 err = -ENOMEM; 8643 goto out; 8644 } 8645 // TODO: increase capacity to grow slab_sheaf up to next kmalloc size? 8646 s->sheaf_capacity = args->sheaf_capacity; 8647 } 8648 8649 #ifdef CONFIG_NUMA 8650 s->remote_node_defrag_ratio = 1000; 8651 #endif 8652 8653 /* Initialize the pre-computed randomized freelist if slab is up */ 8654 if (slab_state >= UP) { 8655 if (init_cache_random_seq(s)) 8656 goto out; 8657 } 8658 8659 if (!init_kmem_cache_nodes(s)) 8660 goto out; 8661 8662 if (!alloc_kmem_cache_cpus(s)) 8663 goto out; 8664 8665 if (s->cpu_sheaves) { 8666 err = init_percpu_sheaves(s); 8667 if (err) 8668 goto out; 8669 } 8670 8671 err = 0; 8672 8673 /* Mutex is not taken during early boot */ 8674 if (slab_state <= UP) 8675 goto out; 8676 8677 /* 8678 * Failing to create sysfs files is not critical to SLUB functionality. 8679 * If it fails, proceed with cache creation without these files. 8680 */ 8681 if (sysfs_slab_add(s)) 8682 pr_err("SLUB: Unable to add cache %s to sysfs\n", s->name); 8683 8684 if (s->flags & SLAB_STORE_USER) 8685 debugfs_slab_add(s); 8686 8687 out: 8688 if (err) 8689 __kmem_cache_release(s); 8690 return err; 8691 } 8692 8693 #ifdef SLAB_SUPPORTS_SYSFS 8694 static int count_inuse(struct slab *slab) 8695 { 8696 return slab->inuse; 8697 } 8698 8699 static int count_total(struct slab *slab) 8700 { 8701 return slab->objects; 8702 } 8703 #endif 8704 8705 #ifdef CONFIG_SLUB_DEBUG 8706 static void validate_slab(struct kmem_cache *s, struct slab *slab, 8707 unsigned long *obj_map) 8708 { 8709 void *p; 8710 void *addr = slab_address(slab); 8711 8712 if (!validate_slab_ptr(slab)) { 8713 slab_err(s, slab, "Not a valid slab page"); 8714 return; 8715 } 8716 8717 if (!check_slab(s, slab) || !on_freelist(s, slab, NULL)) 8718 return; 8719 8720 /* Now we know that a valid freelist exists */ 8721 __fill_map(obj_map, s, slab); 8722 for_each_object(p, s, addr, slab->objects) { 8723 u8 val = test_bit(__obj_to_index(s, addr, p), obj_map) ? 8724 SLUB_RED_INACTIVE : SLUB_RED_ACTIVE; 8725 8726 if (!check_object(s, slab, p, val)) 8727 break; 8728 } 8729 } 8730 8731 static int validate_slab_node(struct kmem_cache *s, 8732 struct kmem_cache_node *n, unsigned long *obj_map) 8733 { 8734 unsigned long count = 0; 8735 struct slab *slab; 8736 unsigned long flags; 8737 8738 spin_lock_irqsave(&n->list_lock, flags); 8739 8740 list_for_each_entry(slab, &n->partial, slab_list) { 8741 validate_slab(s, slab, obj_map); 8742 count++; 8743 } 8744 if (count != n->nr_partial) { 8745 pr_err("SLUB %s: %ld partial slabs counted but counter=%ld\n", 8746 s->name, count, n->nr_partial); 8747 slab_add_kunit_errors(); 8748 } 8749 8750 if (!(s->flags & SLAB_STORE_USER)) 8751 goto out; 8752 8753 list_for_each_entry(slab, &n->full, slab_list) { 8754 validate_slab(s, slab, obj_map); 8755 count++; 8756 } 8757 if (count != node_nr_slabs(n)) { 8758 pr_err("SLUB: %s %ld slabs counted but counter=%ld\n", 8759 s->name, count, node_nr_slabs(n)); 8760 slab_add_kunit_errors(); 8761 } 8762 8763 out: 8764 spin_unlock_irqrestore(&n->list_lock, flags); 8765 return count; 8766 } 8767 8768 long validate_slab_cache(struct kmem_cache *s) 8769 { 8770 int node; 8771 unsigned long count = 0; 8772 struct kmem_cache_node *n; 8773 unsigned long *obj_map; 8774 8775 obj_map = bitmap_alloc(oo_objects(s->oo), GFP_KERNEL); 8776 if (!obj_map) 8777 return -ENOMEM; 8778 8779 flush_all(s); 8780 for_each_kmem_cache_node(s, node, n) 8781 count += validate_slab_node(s, n, obj_map); 8782 8783 bitmap_free(obj_map); 8784 8785 return count; 8786 } 8787 EXPORT_SYMBOL(validate_slab_cache); 8788 8789 #ifdef CONFIG_DEBUG_FS 8790 /* 8791 * Generate lists of code addresses where slabcache objects are allocated 8792 * and freed. 8793 */ 8794 8795 struct location { 8796 depot_stack_handle_t handle; 8797 unsigned long count; 8798 unsigned long addr; 8799 unsigned long waste; 8800 long long sum_time; 8801 long min_time; 8802 long max_time; 8803 long min_pid; 8804 long max_pid; 8805 DECLARE_BITMAP(cpus, NR_CPUS); 8806 nodemask_t nodes; 8807 }; 8808 8809 struct loc_track { 8810 unsigned long max; 8811 unsigned long count; 8812 struct location *loc; 8813 loff_t idx; 8814 }; 8815 8816 static struct dentry *slab_debugfs_root; 8817 8818 static void free_loc_track(struct loc_track *t) 8819 { 8820 if (t->max) 8821 free_pages((unsigned long)t->loc, 8822 get_order(sizeof(struct location) * t->max)); 8823 } 8824 8825 static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags) 8826 { 8827 struct location *l; 8828 int order; 8829 8830 order = get_order(sizeof(struct location) * max); 8831 8832 l = (void *)__get_free_pages(flags, order); 8833 if (!l) 8834 return 0; 8835 8836 if (t->count) { 8837 memcpy(l, t->loc, sizeof(struct location) * t->count); 8838 free_loc_track(t); 8839 } 8840 t->max = max; 8841 t->loc = l; 8842 return 1; 8843 } 8844 8845 static int add_location(struct loc_track *t, struct kmem_cache *s, 8846 const struct track *track, 8847 unsigned int orig_size) 8848 { 8849 long start, end, pos; 8850 struct location *l; 8851 unsigned long caddr, chandle, cwaste; 8852 unsigned long age = jiffies - track->when; 8853 depot_stack_handle_t handle = 0; 8854 unsigned int waste = s->object_size - orig_size; 8855 8856 #ifdef CONFIG_STACKDEPOT 8857 handle = READ_ONCE(track->handle); 8858 #endif 8859 start = -1; 8860 end = t->count; 8861 8862 for ( ; ; ) { 8863 pos = start + (end - start + 1) / 2; 8864 8865 /* 8866 * There is nothing at "end". If we end up there 8867 * we need to add something to before end. 8868 */ 8869 if (pos == end) 8870 break; 8871 8872 l = &t->loc[pos]; 8873 caddr = l->addr; 8874 chandle = l->handle; 8875 cwaste = l->waste; 8876 if ((track->addr == caddr) && (handle == chandle) && 8877 (waste == cwaste)) { 8878 8879 l->count++; 8880 if (track->when) { 8881 l->sum_time += age; 8882 if (age < l->min_time) 8883 l->min_time = age; 8884 if (age > l->max_time) 8885 l->max_time = age; 8886 8887 if (track->pid < l->min_pid) 8888 l->min_pid = track->pid; 8889 if (track->pid > l->max_pid) 8890 l->max_pid = track->pid; 8891 8892 cpumask_set_cpu(track->cpu, 8893 to_cpumask(l->cpus)); 8894 } 8895 node_set(page_to_nid(virt_to_page(track)), l->nodes); 8896 return 1; 8897 } 8898 8899 if (track->addr < caddr) 8900 end = pos; 8901 else if (track->addr == caddr && handle < chandle) 8902 end = pos; 8903 else if (track->addr == caddr && handle == chandle && 8904 waste < cwaste) 8905 end = pos; 8906 else 8907 start = pos; 8908 } 8909 8910 /* 8911 * Not found. Insert new tracking element. 8912 */ 8913 if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC)) 8914 return 0; 8915 8916 l = t->loc + pos; 8917 if (pos < t->count) 8918 memmove(l + 1, l, 8919 (t->count - pos) * sizeof(struct location)); 8920 t->count++; 8921 l->count = 1; 8922 l->addr = track->addr; 8923 l->sum_time = age; 8924 l->min_time = age; 8925 l->max_time = age; 8926 l->min_pid = track->pid; 8927 l->max_pid = track->pid; 8928 l->handle = handle; 8929 l->waste = waste; 8930 cpumask_clear(to_cpumask(l->cpus)); 8931 cpumask_set_cpu(track->cpu, to_cpumask(l->cpus)); 8932 nodes_clear(l->nodes); 8933 node_set(page_to_nid(virt_to_page(track)), l->nodes); 8934 return 1; 8935 } 8936 8937 static void process_slab(struct loc_track *t, struct kmem_cache *s, 8938 struct slab *slab, enum track_item alloc, 8939 unsigned long *obj_map) 8940 { 8941 void *addr = slab_address(slab); 8942 bool is_alloc = (alloc == TRACK_ALLOC); 8943 void *p; 8944 8945 __fill_map(obj_map, s, slab); 8946 8947 for_each_object(p, s, addr, slab->objects) 8948 if (!test_bit(__obj_to_index(s, addr, p), obj_map)) 8949 add_location(t, s, get_track(s, p, alloc), 8950 is_alloc ? get_orig_size(s, p) : 8951 s->object_size); 8952 } 8953 #endif /* CONFIG_DEBUG_FS */ 8954 #endif /* CONFIG_SLUB_DEBUG */ 8955 8956 #ifdef SLAB_SUPPORTS_SYSFS 8957 enum slab_stat_type { 8958 SL_ALL, /* All slabs */ 8959 SL_PARTIAL, /* Only partially allocated slabs */ 8960 SL_CPU, /* Only slabs used for cpu caches */ 8961 SL_OBJECTS, /* Determine allocated objects not slabs */ 8962 SL_TOTAL /* Determine object capacity not slabs */ 8963 }; 8964 8965 #define SO_ALL (1 << SL_ALL) 8966 #define SO_PARTIAL (1 << SL_PARTIAL) 8967 #define SO_CPU (1 << SL_CPU) 8968 #define SO_OBJECTS (1 << SL_OBJECTS) 8969 #define SO_TOTAL (1 << SL_TOTAL) 8970 8971 static ssize_t show_slab_objects(struct kmem_cache *s, 8972 char *buf, unsigned long flags) 8973 { 8974 unsigned long total = 0; 8975 int node; 8976 int x; 8977 unsigned long *nodes; 8978 int len = 0; 8979 8980 nodes = kcalloc(nr_node_ids, sizeof(unsigned long), GFP_KERNEL); 8981 if (!nodes) 8982 return -ENOMEM; 8983 8984 if (flags & SO_CPU) { 8985 int cpu; 8986 8987 for_each_possible_cpu(cpu) { 8988 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, 8989 cpu); 8990 int node; 8991 struct slab *slab; 8992 8993 slab = READ_ONCE(c->slab); 8994 if (!slab) 8995 continue; 8996 8997 node = slab_nid(slab); 8998 if (flags & SO_TOTAL) 8999 x = slab->objects; 9000 else if (flags & SO_OBJECTS) 9001 x = slab->inuse; 9002 else 9003 x = 1; 9004 9005 total += x; 9006 nodes[node] += x; 9007 9008 #ifdef CONFIG_SLUB_CPU_PARTIAL 9009 slab = slub_percpu_partial_read_once(c); 9010 if (slab) { 9011 node = slab_nid(slab); 9012 if (flags & SO_TOTAL) 9013 WARN_ON_ONCE(1); 9014 else if (flags & SO_OBJECTS) 9015 WARN_ON_ONCE(1); 9016 else 9017 x = data_race(slab->slabs); 9018 total += x; 9019 nodes[node] += x; 9020 } 9021 #endif 9022 } 9023 } 9024 9025 /* 9026 * It is impossible to take "mem_hotplug_lock" here with "kernfs_mutex" 9027 * already held which will conflict with an existing lock order: 9028 * 9029 * mem_hotplug_lock->slab_mutex->kernfs_mutex 9030 * 9031 * We don't really need mem_hotplug_lock (to hold off 9032 * slab_mem_going_offline_callback) here because slab's memory hot 9033 * unplug code doesn't destroy the kmem_cache->node[] data. 9034 */ 9035 9036 #ifdef CONFIG_SLUB_DEBUG 9037 if (flags & SO_ALL) { 9038 struct kmem_cache_node *n; 9039 9040 for_each_kmem_cache_node(s, node, n) { 9041 9042 if (flags & SO_TOTAL) 9043 x = node_nr_objs(n); 9044 else if (flags & SO_OBJECTS) 9045 x = node_nr_objs(n) - count_partial(n, count_free); 9046 else 9047 x = node_nr_slabs(n); 9048 total += x; 9049 nodes[node] += x; 9050 } 9051 9052 } else 9053 #endif 9054 if (flags & SO_PARTIAL) { 9055 struct kmem_cache_node *n; 9056 9057 for_each_kmem_cache_node(s, node, n) { 9058 if (flags & SO_TOTAL) 9059 x = count_partial(n, count_total); 9060 else if (flags & SO_OBJECTS) 9061 x = count_partial(n, count_inuse); 9062 else 9063 x = n->nr_partial; 9064 total += x; 9065 nodes[node] += x; 9066 } 9067 } 9068 9069 len += sysfs_emit_at(buf, len, "%lu", total); 9070 #ifdef CONFIG_NUMA 9071 for (node = 0; node < nr_node_ids; node++) { 9072 if (nodes[node]) 9073 len += sysfs_emit_at(buf, len, " N%d=%lu", 9074 node, nodes[node]); 9075 } 9076 #endif 9077 len += sysfs_emit_at(buf, len, "\n"); 9078 kfree(nodes); 9079 9080 return len; 9081 } 9082 9083 #define to_slab_attr(n) container_of(n, struct slab_attribute, attr) 9084 #define to_slab(n) container_of(n, struct kmem_cache, kobj) 9085 9086 struct slab_attribute { 9087 struct attribute attr; 9088 ssize_t (*show)(struct kmem_cache *s, char *buf); 9089 ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count); 9090 }; 9091 9092 #define SLAB_ATTR_RO(_name) \ 9093 static struct slab_attribute _name##_attr = __ATTR_RO_MODE(_name, 0400) 9094 9095 #define SLAB_ATTR(_name) \ 9096 static struct slab_attribute _name##_attr = __ATTR_RW_MODE(_name, 0600) 9097 9098 static ssize_t slab_size_show(struct kmem_cache *s, char *buf) 9099 { 9100 return sysfs_emit(buf, "%u\n", s->size); 9101 } 9102 SLAB_ATTR_RO(slab_size); 9103 9104 static ssize_t align_show(struct kmem_cache *s, char *buf) 9105 { 9106 return sysfs_emit(buf, "%u\n", s->align); 9107 } 9108 SLAB_ATTR_RO(align); 9109 9110 static ssize_t object_size_show(struct kmem_cache *s, char *buf) 9111 { 9112 return sysfs_emit(buf, "%u\n", s->object_size); 9113 } 9114 SLAB_ATTR_RO(object_size); 9115 9116 static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf) 9117 { 9118 return sysfs_emit(buf, "%u\n", oo_objects(s->oo)); 9119 } 9120 SLAB_ATTR_RO(objs_per_slab); 9121 9122 static ssize_t order_show(struct kmem_cache *s, char *buf) 9123 { 9124 return sysfs_emit(buf, "%u\n", oo_order(s->oo)); 9125 } 9126 SLAB_ATTR_RO(order); 9127 9128 static ssize_t sheaf_capacity_show(struct kmem_cache *s, char *buf) 9129 { 9130 return sysfs_emit(buf, "%u\n", s->sheaf_capacity); 9131 } 9132 SLAB_ATTR_RO(sheaf_capacity); 9133 9134 static ssize_t min_partial_show(struct kmem_cache *s, char *buf) 9135 { 9136 return sysfs_emit(buf, "%lu\n", s->min_partial); 9137 } 9138 9139 static ssize_t min_partial_store(struct kmem_cache *s, const char *buf, 9140 size_t length) 9141 { 9142 unsigned long min; 9143 int err; 9144 9145 err = kstrtoul(buf, 10, &min); 9146 if (err) 9147 return err; 9148 9149 s->min_partial = min; 9150 return length; 9151 } 9152 SLAB_ATTR(min_partial); 9153 9154 static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf) 9155 { 9156 unsigned int nr_partial = 0; 9157 #ifdef CONFIG_SLUB_CPU_PARTIAL 9158 nr_partial = s->cpu_partial; 9159 #endif 9160 9161 return sysfs_emit(buf, "%u\n", nr_partial); 9162 } 9163 9164 static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf, 9165 size_t length) 9166 { 9167 unsigned int objects; 9168 int err; 9169 9170 err = kstrtouint(buf, 10, &objects); 9171 if (err) 9172 return err; 9173 if (objects && !kmem_cache_has_cpu_partial(s)) 9174 return -EINVAL; 9175 9176 slub_set_cpu_partial(s, objects); 9177 flush_all(s); 9178 return length; 9179 } 9180 SLAB_ATTR(cpu_partial); 9181 9182 static ssize_t ctor_show(struct kmem_cache *s, char *buf) 9183 { 9184 if (!s->ctor) 9185 return 0; 9186 return sysfs_emit(buf, "%pS\n", s->ctor); 9187 } 9188 SLAB_ATTR_RO(ctor); 9189 9190 static ssize_t aliases_show(struct kmem_cache *s, char *buf) 9191 { 9192 return sysfs_emit(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1); 9193 } 9194 SLAB_ATTR_RO(aliases); 9195 9196 static ssize_t partial_show(struct kmem_cache *s, char *buf) 9197 { 9198 return show_slab_objects(s, buf, SO_PARTIAL); 9199 } 9200 SLAB_ATTR_RO(partial); 9201 9202 static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf) 9203 { 9204 return show_slab_objects(s, buf, SO_CPU); 9205 } 9206 SLAB_ATTR_RO(cpu_slabs); 9207 9208 static ssize_t objects_partial_show(struct kmem_cache *s, char *buf) 9209 { 9210 return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS); 9211 } 9212 SLAB_ATTR_RO(objects_partial); 9213 9214 static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf) 9215 { 9216 int objects = 0; 9217 int slabs = 0; 9218 int cpu __maybe_unused; 9219 int len = 0; 9220 9221 #ifdef CONFIG_SLUB_CPU_PARTIAL 9222 for_each_online_cpu(cpu) { 9223 struct slab *slab; 9224 9225 slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu)); 9226 9227 if (slab) 9228 slabs += data_race(slab->slabs); 9229 } 9230 #endif 9231 9232 /* Approximate half-full slabs, see slub_set_cpu_partial() */ 9233 objects = (slabs * oo_objects(s->oo)) / 2; 9234 len += sysfs_emit_at(buf, len, "%d(%d)", objects, slabs); 9235 9236 #ifdef CONFIG_SLUB_CPU_PARTIAL 9237 for_each_online_cpu(cpu) { 9238 struct slab *slab; 9239 9240 slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu)); 9241 if (slab) { 9242 slabs = data_race(slab->slabs); 9243 objects = (slabs * oo_objects(s->oo)) / 2; 9244 len += sysfs_emit_at(buf, len, " C%d=%d(%d)", 9245 cpu, objects, slabs); 9246 } 9247 } 9248 #endif 9249 len += sysfs_emit_at(buf, len, "\n"); 9250 9251 return len; 9252 } 9253 SLAB_ATTR_RO(slabs_cpu_partial); 9254 9255 static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf) 9256 { 9257 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT)); 9258 } 9259 SLAB_ATTR_RO(reclaim_account); 9260 9261 static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf) 9262 { 9263 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN)); 9264 } 9265 SLAB_ATTR_RO(hwcache_align); 9266 9267 #ifdef CONFIG_ZONE_DMA 9268 static ssize_t cache_dma_show(struct kmem_cache *s, char *buf) 9269 { 9270 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA)); 9271 } 9272 SLAB_ATTR_RO(cache_dma); 9273 #endif 9274 9275 #ifdef CONFIG_HARDENED_USERCOPY 9276 static ssize_t usersize_show(struct kmem_cache *s, char *buf) 9277 { 9278 return sysfs_emit(buf, "%u\n", s->usersize); 9279 } 9280 SLAB_ATTR_RO(usersize); 9281 #endif 9282 9283 static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf) 9284 { 9285 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_TYPESAFE_BY_RCU)); 9286 } 9287 SLAB_ATTR_RO(destroy_by_rcu); 9288 9289 #ifdef CONFIG_SLUB_DEBUG 9290 static ssize_t slabs_show(struct kmem_cache *s, char *buf) 9291 { 9292 return show_slab_objects(s, buf, SO_ALL); 9293 } 9294 SLAB_ATTR_RO(slabs); 9295 9296 static ssize_t total_objects_show(struct kmem_cache *s, char *buf) 9297 { 9298 return show_slab_objects(s, buf, SO_ALL|SO_TOTAL); 9299 } 9300 SLAB_ATTR_RO(total_objects); 9301 9302 static ssize_t objects_show(struct kmem_cache *s, char *buf) 9303 { 9304 return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS); 9305 } 9306 SLAB_ATTR_RO(objects); 9307 9308 static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf) 9309 { 9310 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_CONSISTENCY_CHECKS)); 9311 } 9312 SLAB_ATTR_RO(sanity_checks); 9313 9314 static ssize_t trace_show(struct kmem_cache *s, char *buf) 9315 { 9316 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_TRACE)); 9317 } 9318 SLAB_ATTR_RO(trace); 9319 9320 static ssize_t red_zone_show(struct kmem_cache *s, char *buf) 9321 { 9322 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE)); 9323 } 9324 9325 SLAB_ATTR_RO(red_zone); 9326 9327 static ssize_t poison_show(struct kmem_cache *s, char *buf) 9328 { 9329 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_POISON)); 9330 } 9331 9332 SLAB_ATTR_RO(poison); 9333 9334 static ssize_t store_user_show(struct kmem_cache *s, char *buf) 9335 { 9336 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_STORE_USER)); 9337 } 9338 9339 SLAB_ATTR_RO(store_user); 9340 9341 static ssize_t validate_show(struct kmem_cache *s, char *buf) 9342 { 9343 return 0; 9344 } 9345 9346 static ssize_t validate_store(struct kmem_cache *s, 9347 const char *buf, size_t length) 9348 { 9349 int ret = -EINVAL; 9350 9351 if (buf[0] == '1' && kmem_cache_debug(s)) { 9352 ret = validate_slab_cache(s); 9353 if (ret >= 0) 9354 ret = length; 9355 } 9356 return ret; 9357 } 9358 SLAB_ATTR(validate); 9359 9360 #endif /* CONFIG_SLUB_DEBUG */ 9361 9362 #ifdef CONFIG_FAILSLAB 9363 static ssize_t failslab_show(struct kmem_cache *s, char *buf) 9364 { 9365 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB)); 9366 } 9367 9368 static ssize_t failslab_store(struct kmem_cache *s, const char *buf, 9369 size_t length) 9370 { 9371 if (s->refcount > 1) 9372 return -EINVAL; 9373 9374 if (buf[0] == '1') 9375 WRITE_ONCE(s->flags, s->flags | SLAB_FAILSLAB); 9376 else 9377 WRITE_ONCE(s->flags, s->flags & ~SLAB_FAILSLAB); 9378 9379 return length; 9380 } 9381 SLAB_ATTR(failslab); 9382 #endif 9383 9384 static ssize_t shrink_show(struct kmem_cache *s, char *buf) 9385 { 9386 return 0; 9387 } 9388 9389 static ssize_t shrink_store(struct kmem_cache *s, 9390 const char *buf, size_t length) 9391 { 9392 if (buf[0] == '1') 9393 kmem_cache_shrink(s); 9394 else 9395 return -EINVAL; 9396 return length; 9397 } 9398 SLAB_ATTR(shrink); 9399 9400 #ifdef CONFIG_NUMA 9401 static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf) 9402 { 9403 return sysfs_emit(buf, "%u\n", s->remote_node_defrag_ratio / 10); 9404 } 9405 9406 static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s, 9407 const char *buf, size_t length) 9408 { 9409 unsigned int ratio; 9410 int err; 9411 9412 err = kstrtouint(buf, 10, &ratio); 9413 if (err) 9414 return err; 9415 if (ratio > 100) 9416 return -ERANGE; 9417 9418 s->remote_node_defrag_ratio = ratio * 10; 9419 9420 return length; 9421 } 9422 SLAB_ATTR(remote_node_defrag_ratio); 9423 #endif 9424 9425 #ifdef CONFIG_SLUB_STATS 9426 static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si) 9427 { 9428 unsigned long sum = 0; 9429 int cpu; 9430 int len = 0; 9431 int *data = kmalloc_array(nr_cpu_ids, sizeof(int), GFP_KERNEL); 9432 9433 if (!data) 9434 return -ENOMEM; 9435 9436 for_each_online_cpu(cpu) { 9437 unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si]; 9438 9439 data[cpu] = x; 9440 sum += x; 9441 } 9442 9443 len += sysfs_emit_at(buf, len, "%lu", sum); 9444 9445 #ifdef CONFIG_SMP 9446 for_each_online_cpu(cpu) { 9447 if (data[cpu]) 9448 len += sysfs_emit_at(buf, len, " C%d=%u", 9449 cpu, data[cpu]); 9450 } 9451 #endif 9452 kfree(data); 9453 len += sysfs_emit_at(buf, len, "\n"); 9454 9455 return len; 9456 } 9457 9458 static void clear_stat(struct kmem_cache *s, enum stat_item si) 9459 { 9460 int cpu; 9461 9462 for_each_online_cpu(cpu) 9463 per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0; 9464 } 9465 9466 #define STAT_ATTR(si, text) \ 9467 static ssize_t text##_show(struct kmem_cache *s, char *buf) \ 9468 { \ 9469 return show_stat(s, buf, si); \ 9470 } \ 9471 static ssize_t text##_store(struct kmem_cache *s, \ 9472 const char *buf, size_t length) \ 9473 { \ 9474 if (buf[0] != '0') \ 9475 return -EINVAL; \ 9476 clear_stat(s, si); \ 9477 return length; \ 9478 } \ 9479 SLAB_ATTR(text); \ 9480 9481 STAT_ATTR(ALLOC_PCS, alloc_cpu_sheaf); 9482 STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath); 9483 STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath); 9484 STAT_ATTR(FREE_PCS, free_cpu_sheaf); 9485 STAT_ATTR(FREE_RCU_SHEAF, free_rcu_sheaf); 9486 STAT_ATTR(FREE_RCU_SHEAF_FAIL, free_rcu_sheaf_fail); 9487 STAT_ATTR(FREE_FASTPATH, free_fastpath); 9488 STAT_ATTR(FREE_SLOWPATH, free_slowpath); 9489 STAT_ATTR(FREE_FROZEN, free_frozen); 9490 STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial); 9491 STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial); 9492 STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial); 9493 STAT_ATTR(ALLOC_SLAB, alloc_slab); 9494 STAT_ATTR(ALLOC_REFILL, alloc_refill); 9495 STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch); 9496 STAT_ATTR(FREE_SLAB, free_slab); 9497 STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush); 9498 STAT_ATTR(DEACTIVATE_FULL, deactivate_full); 9499 STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty); 9500 STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head); 9501 STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail); 9502 STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees); 9503 STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass); 9504 STAT_ATTR(ORDER_FALLBACK, order_fallback); 9505 STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail); 9506 STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail); 9507 STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc); 9508 STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free); 9509 STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node); 9510 STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain); 9511 STAT_ATTR(SHEAF_FLUSH, sheaf_flush); 9512 STAT_ATTR(SHEAF_REFILL, sheaf_refill); 9513 STAT_ATTR(SHEAF_ALLOC, sheaf_alloc); 9514 STAT_ATTR(SHEAF_FREE, sheaf_free); 9515 STAT_ATTR(BARN_GET, barn_get); 9516 STAT_ATTR(BARN_GET_FAIL, barn_get_fail); 9517 STAT_ATTR(BARN_PUT, barn_put); 9518 STAT_ATTR(BARN_PUT_FAIL, barn_put_fail); 9519 STAT_ATTR(SHEAF_PREFILL_FAST, sheaf_prefill_fast); 9520 STAT_ATTR(SHEAF_PREFILL_SLOW, sheaf_prefill_slow); 9521 STAT_ATTR(SHEAF_PREFILL_OVERSIZE, sheaf_prefill_oversize); 9522 STAT_ATTR(SHEAF_RETURN_FAST, sheaf_return_fast); 9523 STAT_ATTR(SHEAF_RETURN_SLOW, sheaf_return_slow); 9524 #endif /* CONFIG_SLUB_STATS */ 9525 9526 #ifdef CONFIG_KFENCE 9527 static ssize_t skip_kfence_show(struct kmem_cache *s, char *buf) 9528 { 9529 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_SKIP_KFENCE)); 9530 } 9531 9532 static ssize_t skip_kfence_store(struct kmem_cache *s, 9533 const char *buf, size_t length) 9534 { 9535 int ret = length; 9536 9537 if (buf[0] == '0') 9538 s->flags &= ~SLAB_SKIP_KFENCE; 9539 else if (buf[0] == '1') 9540 s->flags |= SLAB_SKIP_KFENCE; 9541 else 9542 ret = -EINVAL; 9543 9544 return ret; 9545 } 9546 SLAB_ATTR(skip_kfence); 9547 #endif 9548 9549 static struct attribute *slab_attrs[] = { 9550 &slab_size_attr.attr, 9551 &object_size_attr.attr, 9552 &objs_per_slab_attr.attr, 9553 &order_attr.attr, 9554 &sheaf_capacity_attr.attr, 9555 &min_partial_attr.attr, 9556 &cpu_partial_attr.attr, 9557 &objects_partial_attr.attr, 9558 &partial_attr.attr, 9559 &cpu_slabs_attr.attr, 9560 &ctor_attr.attr, 9561 &aliases_attr.attr, 9562 &align_attr.attr, 9563 &hwcache_align_attr.attr, 9564 &reclaim_account_attr.attr, 9565 &destroy_by_rcu_attr.attr, 9566 &shrink_attr.attr, 9567 &slabs_cpu_partial_attr.attr, 9568 #ifdef CONFIG_SLUB_DEBUG 9569 &total_objects_attr.attr, 9570 &objects_attr.attr, 9571 &slabs_attr.attr, 9572 &sanity_checks_attr.attr, 9573 &trace_attr.attr, 9574 &red_zone_attr.attr, 9575 &poison_attr.attr, 9576 &store_user_attr.attr, 9577 &validate_attr.attr, 9578 #endif 9579 #ifdef CONFIG_ZONE_DMA 9580 &cache_dma_attr.attr, 9581 #endif 9582 #ifdef CONFIG_NUMA 9583 &remote_node_defrag_ratio_attr.attr, 9584 #endif 9585 #ifdef CONFIG_SLUB_STATS 9586 &alloc_cpu_sheaf_attr.attr, 9587 &alloc_fastpath_attr.attr, 9588 &alloc_slowpath_attr.attr, 9589 &free_cpu_sheaf_attr.attr, 9590 &free_rcu_sheaf_attr.attr, 9591 &free_rcu_sheaf_fail_attr.attr, 9592 &free_fastpath_attr.attr, 9593 &free_slowpath_attr.attr, 9594 &free_frozen_attr.attr, 9595 &free_add_partial_attr.attr, 9596 &free_remove_partial_attr.attr, 9597 &alloc_from_partial_attr.attr, 9598 &alloc_slab_attr.attr, 9599 &alloc_refill_attr.attr, 9600 &alloc_node_mismatch_attr.attr, 9601 &free_slab_attr.attr, 9602 &cpuslab_flush_attr.attr, 9603 &deactivate_full_attr.attr, 9604 &deactivate_empty_attr.attr, 9605 &deactivate_to_head_attr.attr, 9606 &deactivate_to_tail_attr.attr, 9607 &deactivate_remote_frees_attr.attr, 9608 &deactivate_bypass_attr.attr, 9609 &order_fallback_attr.attr, 9610 &cmpxchg_double_fail_attr.attr, 9611 &cmpxchg_double_cpu_fail_attr.attr, 9612 &cpu_partial_alloc_attr.attr, 9613 &cpu_partial_free_attr.attr, 9614 &cpu_partial_node_attr.attr, 9615 &cpu_partial_drain_attr.attr, 9616 &sheaf_flush_attr.attr, 9617 &sheaf_refill_attr.attr, 9618 &sheaf_alloc_attr.attr, 9619 &sheaf_free_attr.attr, 9620 &barn_get_attr.attr, 9621 &barn_get_fail_attr.attr, 9622 &barn_put_attr.attr, 9623 &barn_put_fail_attr.attr, 9624 &sheaf_prefill_fast_attr.attr, 9625 &sheaf_prefill_slow_attr.attr, 9626 &sheaf_prefill_oversize_attr.attr, 9627 &sheaf_return_fast_attr.attr, 9628 &sheaf_return_slow_attr.attr, 9629 #endif 9630 #ifdef CONFIG_FAILSLAB 9631 &failslab_attr.attr, 9632 #endif 9633 #ifdef CONFIG_HARDENED_USERCOPY 9634 &usersize_attr.attr, 9635 #endif 9636 #ifdef CONFIG_KFENCE 9637 &skip_kfence_attr.attr, 9638 #endif 9639 9640 NULL 9641 }; 9642 9643 static const struct attribute_group slab_attr_group = { 9644 .attrs = slab_attrs, 9645 }; 9646 9647 static ssize_t slab_attr_show(struct kobject *kobj, 9648 struct attribute *attr, 9649 char *buf) 9650 { 9651 struct slab_attribute *attribute; 9652 struct kmem_cache *s; 9653 9654 attribute = to_slab_attr(attr); 9655 s = to_slab(kobj); 9656 9657 if (!attribute->show) 9658 return -EIO; 9659 9660 return attribute->show(s, buf); 9661 } 9662 9663 static ssize_t slab_attr_store(struct kobject *kobj, 9664 struct attribute *attr, 9665 const char *buf, size_t len) 9666 { 9667 struct slab_attribute *attribute; 9668 struct kmem_cache *s; 9669 9670 attribute = to_slab_attr(attr); 9671 s = to_slab(kobj); 9672 9673 if (!attribute->store) 9674 return -EIO; 9675 9676 return attribute->store(s, buf, len); 9677 } 9678 9679 static void kmem_cache_release(struct kobject *k) 9680 { 9681 slab_kmem_cache_release(to_slab(k)); 9682 } 9683 9684 static const struct sysfs_ops slab_sysfs_ops = { 9685 .show = slab_attr_show, 9686 .store = slab_attr_store, 9687 }; 9688 9689 static const struct kobj_type slab_ktype = { 9690 .sysfs_ops = &slab_sysfs_ops, 9691 .release = kmem_cache_release, 9692 }; 9693 9694 static struct kset *slab_kset; 9695 9696 static inline struct kset *cache_kset(struct kmem_cache *s) 9697 { 9698 return slab_kset; 9699 } 9700 9701 #define ID_STR_LENGTH 32 9702 9703 /* Create a unique string id for a slab cache: 9704 * 9705 * Format :[flags-]size 9706 */ 9707 static char *create_unique_id(struct kmem_cache *s) 9708 { 9709 char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL); 9710 char *p = name; 9711 9712 if (!name) 9713 return ERR_PTR(-ENOMEM); 9714 9715 *p++ = ':'; 9716 /* 9717 * First flags affecting slabcache operations. We will only 9718 * get here for aliasable slabs so we do not need to support 9719 * too many flags. The flags here must cover all flags that 9720 * are matched during merging to guarantee that the id is 9721 * unique. 9722 */ 9723 if (s->flags & SLAB_CACHE_DMA) 9724 *p++ = 'd'; 9725 if (s->flags & SLAB_CACHE_DMA32) 9726 *p++ = 'D'; 9727 if (s->flags & SLAB_RECLAIM_ACCOUNT) 9728 *p++ = 'a'; 9729 if (s->flags & SLAB_CONSISTENCY_CHECKS) 9730 *p++ = 'F'; 9731 if (s->flags & SLAB_ACCOUNT) 9732 *p++ = 'A'; 9733 if (p != name + 1) 9734 *p++ = '-'; 9735 p += snprintf(p, ID_STR_LENGTH - (p - name), "%07u", s->size); 9736 9737 if (WARN_ON(p > name + ID_STR_LENGTH - 1)) { 9738 kfree(name); 9739 return ERR_PTR(-EINVAL); 9740 } 9741 kmsan_unpoison_memory(name, p - name); 9742 return name; 9743 } 9744 9745 static int sysfs_slab_add(struct kmem_cache *s) 9746 { 9747 int err; 9748 const char *name; 9749 struct kset *kset = cache_kset(s); 9750 int unmergeable = slab_unmergeable(s); 9751 9752 if (!unmergeable && disable_higher_order_debug && 9753 (slub_debug & DEBUG_METADATA_FLAGS)) 9754 unmergeable = 1; 9755 9756 if (unmergeable) { 9757 /* 9758 * Slabcache can never be merged so we can use the name proper. 9759 * This is typically the case for debug situations. In that 9760 * case we can catch duplicate names easily. 9761 */ 9762 sysfs_remove_link(&slab_kset->kobj, s->name); 9763 name = s->name; 9764 } else { 9765 /* 9766 * Create a unique name for the slab as a target 9767 * for the symlinks. 9768 */ 9769 name = create_unique_id(s); 9770 if (IS_ERR(name)) 9771 return PTR_ERR(name); 9772 } 9773 9774 s->kobj.kset = kset; 9775 err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name); 9776 if (err) 9777 goto out; 9778 9779 err = sysfs_create_group(&s->kobj, &slab_attr_group); 9780 if (err) 9781 goto out_del_kobj; 9782 9783 if (!unmergeable) { 9784 /* Setup first alias */ 9785 sysfs_slab_alias(s, s->name); 9786 } 9787 out: 9788 if (!unmergeable) 9789 kfree(name); 9790 return err; 9791 out_del_kobj: 9792 kobject_del(&s->kobj); 9793 goto out; 9794 } 9795 9796 void sysfs_slab_unlink(struct kmem_cache *s) 9797 { 9798 if (s->kobj.state_in_sysfs) 9799 kobject_del(&s->kobj); 9800 } 9801 9802 void sysfs_slab_release(struct kmem_cache *s) 9803 { 9804 kobject_put(&s->kobj); 9805 } 9806 9807 /* 9808 * Need to buffer aliases during bootup until sysfs becomes 9809 * available lest we lose that information. 9810 */ 9811 struct saved_alias { 9812 struct kmem_cache *s; 9813 const char *name; 9814 struct saved_alias *next; 9815 }; 9816 9817 static struct saved_alias *alias_list; 9818 9819 static int sysfs_slab_alias(struct kmem_cache *s, const char *name) 9820 { 9821 struct saved_alias *al; 9822 9823 if (slab_state == FULL) { 9824 /* 9825 * If we have a leftover link then remove it. 9826 */ 9827 sysfs_remove_link(&slab_kset->kobj, name); 9828 /* 9829 * The original cache may have failed to generate sysfs file. 9830 * In that case, sysfs_create_link() returns -ENOENT and 9831 * symbolic link creation is skipped. 9832 */ 9833 return sysfs_create_link(&slab_kset->kobj, &s->kobj, name); 9834 } 9835 9836 al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL); 9837 if (!al) 9838 return -ENOMEM; 9839 9840 al->s = s; 9841 al->name = name; 9842 al->next = alias_list; 9843 alias_list = al; 9844 kmsan_unpoison_memory(al, sizeof(*al)); 9845 return 0; 9846 } 9847 9848 static int __init slab_sysfs_init(void) 9849 { 9850 struct kmem_cache *s; 9851 int err; 9852 9853 mutex_lock(&slab_mutex); 9854 9855 slab_kset = kset_create_and_add("slab", NULL, kernel_kobj); 9856 if (!slab_kset) { 9857 mutex_unlock(&slab_mutex); 9858 pr_err("Cannot register slab subsystem.\n"); 9859 return -ENOMEM; 9860 } 9861 9862 slab_state = FULL; 9863 9864 list_for_each_entry(s, &slab_caches, list) { 9865 err = sysfs_slab_add(s); 9866 if (err) 9867 pr_err("SLUB: Unable to add boot slab %s to sysfs\n", 9868 s->name); 9869 } 9870 9871 while (alias_list) { 9872 struct saved_alias *al = alias_list; 9873 9874 alias_list = alias_list->next; 9875 err = sysfs_slab_alias(al->s, al->name); 9876 if (err) 9877 pr_err("SLUB: Unable to add boot slab alias %s to sysfs\n", 9878 al->name); 9879 kfree(al); 9880 } 9881 9882 mutex_unlock(&slab_mutex); 9883 return 0; 9884 } 9885 late_initcall(slab_sysfs_init); 9886 #endif /* SLAB_SUPPORTS_SYSFS */ 9887 9888 #if defined(CONFIG_SLUB_DEBUG) && defined(CONFIG_DEBUG_FS) 9889 static int slab_debugfs_show(struct seq_file *seq, void *v) 9890 { 9891 struct loc_track *t = seq->private; 9892 struct location *l; 9893 unsigned long idx; 9894 9895 idx = (unsigned long) t->idx; 9896 if (idx < t->count) { 9897 l = &t->loc[idx]; 9898 9899 seq_printf(seq, "%7ld ", l->count); 9900 9901 if (l->addr) 9902 seq_printf(seq, "%pS", (void *)l->addr); 9903 else 9904 seq_puts(seq, "<not-available>"); 9905 9906 if (l->waste) 9907 seq_printf(seq, " waste=%lu/%lu", 9908 l->count * l->waste, l->waste); 9909 9910 if (l->sum_time != l->min_time) { 9911 seq_printf(seq, " age=%ld/%llu/%ld", 9912 l->min_time, div_u64(l->sum_time, l->count), 9913 l->max_time); 9914 } else 9915 seq_printf(seq, " age=%ld", l->min_time); 9916 9917 if (l->min_pid != l->max_pid) 9918 seq_printf(seq, " pid=%ld-%ld", l->min_pid, l->max_pid); 9919 else 9920 seq_printf(seq, " pid=%ld", 9921 l->min_pid); 9922 9923 if (num_online_cpus() > 1 && !cpumask_empty(to_cpumask(l->cpus))) 9924 seq_printf(seq, " cpus=%*pbl", 9925 cpumask_pr_args(to_cpumask(l->cpus))); 9926 9927 if (nr_online_nodes > 1 && !nodes_empty(l->nodes)) 9928 seq_printf(seq, " nodes=%*pbl", 9929 nodemask_pr_args(&l->nodes)); 9930 9931 #ifdef CONFIG_STACKDEPOT 9932 { 9933 depot_stack_handle_t handle; 9934 unsigned long *entries; 9935 unsigned int nr_entries, j; 9936 9937 handle = READ_ONCE(l->handle); 9938 if (handle) { 9939 nr_entries = stack_depot_fetch(handle, &entries); 9940 seq_puts(seq, "\n"); 9941 for (j = 0; j < nr_entries; j++) 9942 seq_printf(seq, " %pS\n", (void *)entries[j]); 9943 } 9944 } 9945 #endif 9946 seq_puts(seq, "\n"); 9947 } 9948 9949 if (!idx && !t->count) 9950 seq_puts(seq, "No data\n"); 9951 9952 return 0; 9953 } 9954 9955 static void slab_debugfs_stop(struct seq_file *seq, void *v) 9956 { 9957 } 9958 9959 static void *slab_debugfs_next(struct seq_file *seq, void *v, loff_t *ppos) 9960 { 9961 struct loc_track *t = seq->private; 9962 9963 t->idx = ++(*ppos); 9964 if (*ppos <= t->count) 9965 return ppos; 9966 9967 return NULL; 9968 } 9969 9970 static int cmp_loc_by_count(const void *a, const void *b) 9971 { 9972 struct location *loc1 = (struct location *)a; 9973 struct location *loc2 = (struct location *)b; 9974 9975 return cmp_int(loc2->count, loc1->count); 9976 } 9977 9978 static void *slab_debugfs_start(struct seq_file *seq, loff_t *ppos) 9979 { 9980 struct loc_track *t = seq->private; 9981 9982 t->idx = *ppos; 9983 return ppos; 9984 } 9985 9986 static const struct seq_operations slab_debugfs_sops = { 9987 .start = slab_debugfs_start, 9988 .next = slab_debugfs_next, 9989 .stop = slab_debugfs_stop, 9990 .show = slab_debugfs_show, 9991 }; 9992 9993 static int slab_debug_trace_open(struct inode *inode, struct file *filep) 9994 { 9995 9996 struct kmem_cache_node *n; 9997 enum track_item alloc; 9998 int node; 9999 struct loc_track *t = __seq_open_private(filep, &slab_debugfs_sops, 10000 sizeof(struct loc_track)); 10001 struct kmem_cache *s = file_inode(filep)->i_private; 10002 unsigned long *obj_map; 10003 10004 if (!t) 10005 return -ENOMEM; 10006 10007 obj_map = bitmap_alloc(oo_objects(s->oo), GFP_KERNEL); 10008 if (!obj_map) { 10009 seq_release_private(inode, filep); 10010 return -ENOMEM; 10011 } 10012 10013 alloc = debugfs_get_aux_num(filep); 10014 10015 if (!alloc_loc_track(t, PAGE_SIZE / sizeof(struct location), GFP_KERNEL)) { 10016 bitmap_free(obj_map); 10017 seq_release_private(inode, filep); 10018 return -ENOMEM; 10019 } 10020 10021 for_each_kmem_cache_node(s, node, n) { 10022 unsigned long flags; 10023 struct slab *slab; 10024 10025 if (!node_nr_slabs(n)) 10026 continue; 10027 10028 spin_lock_irqsave(&n->list_lock, flags); 10029 list_for_each_entry(slab, &n->partial, slab_list) 10030 process_slab(t, s, slab, alloc, obj_map); 10031 list_for_each_entry(slab, &n->full, slab_list) 10032 process_slab(t, s, slab, alloc, obj_map); 10033 spin_unlock_irqrestore(&n->list_lock, flags); 10034 } 10035 10036 /* Sort locations by count */ 10037 sort(t->loc, t->count, sizeof(struct location), 10038 cmp_loc_by_count, NULL); 10039 10040 bitmap_free(obj_map); 10041 return 0; 10042 } 10043 10044 static int slab_debug_trace_release(struct inode *inode, struct file *file) 10045 { 10046 struct seq_file *seq = file->private_data; 10047 struct loc_track *t = seq->private; 10048 10049 free_loc_track(t); 10050 return seq_release_private(inode, file); 10051 } 10052 10053 static const struct file_operations slab_debugfs_fops = { 10054 .open = slab_debug_trace_open, 10055 .read = seq_read, 10056 .llseek = seq_lseek, 10057 .release = slab_debug_trace_release, 10058 }; 10059 10060 static void debugfs_slab_add(struct kmem_cache *s) 10061 { 10062 struct dentry *slab_cache_dir; 10063 10064 if (unlikely(!slab_debugfs_root)) 10065 return; 10066 10067 slab_cache_dir = debugfs_create_dir(s->name, slab_debugfs_root); 10068 10069 debugfs_create_file_aux_num("alloc_traces", 0400, slab_cache_dir, s, 10070 TRACK_ALLOC, &slab_debugfs_fops); 10071 10072 debugfs_create_file_aux_num("free_traces", 0400, slab_cache_dir, s, 10073 TRACK_FREE, &slab_debugfs_fops); 10074 } 10075 10076 void debugfs_slab_release(struct kmem_cache *s) 10077 { 10078 debugfs_lookup_and_remove(s->name, slab_debugfs_root); 10079 } 10080 10081 static int __init slab_debugfs_init(void) 10082 { 10083 struct kmem_cache *s; 10084 10085 slab_debugfs_root = debugfs_create_dir("slab", NULL); 10086 10087 list_for_each_entry(s, &slab_caches, list) 10088 if (s->flags & SLAB_STORE_USER) 10089 debugfs_slab_add(s); 10090 10091 return 0; 10092 10093 } 10094 __initcall(slab_debugfs_init); 10095 #endif 10096 /* 10097 * The /proc/slabinfo ABI 10098 */ 10099 #ifdef CONFIG_SLUB_DEBUG 10100 void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo) 10101 { 10102 unsigned long nr_slabs = 0; 10103 unsigned long nr_objs = 0; 10104 unsigned long nr_free = 0; 10105 int node; 10106 struct kmem_cache_node *n; 10107 10108 for_each_kmem_cache_node(s, node, n) { 10109 nr_slabs += node_nr_slabs(n); 10110 nr_objs += node_nr_objs(n); 10111 nr_free += count_partial_free_approx(n); 10112 } 10113 10114 sinfo->active_objs = nr_objs - nr_free; 10115 sinfo->num_objs = nr_objs; 10116 sinfo->active_slabs = nr_slabs; 10117 sinfo->num_slabs = nr_slabs; 10118 sinfo->objects_per_slab = oo_objects(s->oo); 10119 sinfo->cache_order = oo_order(s->oo); 10120 } 10121 #endif /* CONFIG_SLUB_DEBUG */