개념 설명 전체 · v6.18.37 / kernel/locking/mutex.c
1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * kernel/locking/mutex.c 4 * 5 * Mutexes: blocking mutual exclusion locks 6 * 7 * Started by Ingo Molnar: 8 * 9 * Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> 10 * 11 * Many thanks to Arjan van de Ven, Thomas Gleixner, Steven Rostedt and 12 * David Howells for suggestions and improvements. 13 * 14 * - Adaptive spinning for mutexes by Peter Zijlstra. (Ported to mainline 15 * from the -rt tree, where it was originally implemented for rtmutexes 16 * by Steven Rostedt, based on work by Gregory Haskins, Peter Morreale 17 * and Sven Dietrich. 18 * 19 * Also see Documentation/locking/mutex-design.rst. 20 */ 21 #include <linux/mutex.h> 22 #include <linux/ww_mutex.h> 23 #include <linux/sched/signal.h> 24 #include <linux/sched/rt.h> 25 #include <linux/sched/wake_q.h> 26 #include <linux/sched/debug.h> 27 #include <linux/export.h> 28 #include <linux/spinlock.h> 29 #include <linux/interrupt.h> 30 #include <linux/debug_locks.h> 31 #include <linux/osq_lock.h> 32 #include <linux/hung_task.h> 33 34 #define CREATE_TRACE_POINTS 35 #include <trace/events/lock.h> 36 37 #ifndef CONFIG_PREEMPT_RT 38 #include "mutex.h" 39 40 #ifdef CONFIG_DEBUG_MUTEXES 41 # define MUTEX_WARN_ON(cond) DEBUG_LOCKS_WARN_ON(cond) 42 #else 43 # define MUTEX_WARN_ON(cond) 44 #endif 45 46 void 47 __mutex_init(struct mutex *lock, const char *name, struct lock_class_key *key) 48 { 49 atomic_long_set(&lock->owner, 0); 50 raw_spin_lock_init(&lock->wait_lock); 51 INIT_LIST_HEAD(&lock->wait_list); 52 #ifdef CONFIG_MUTEX_SPIN_ON_OWNER 53 osq_lock_init(&lock->osq); 54 #endif 55 56 debug_mutex_init(lock, name, key); 57 } 58 EXPORT_SYMBOL(__mutex_init); 59 60 static inline struct task_struct *__owner_task(unsigned long owner) 61 { 62 return (struct task_struct *)(owner & ~MUTEX_FLAGS); 63 } 64 65 bool mutex_is_locked(struct mutex *lock) 66 { 67 return __mutex_owner(lock) != NULL; 68 } 69 EXPORT_SYMBOL(mutex_is_locked); 70 71 static inline unsigned long __owner_flags(unsigned long owner) 72 { 73 return owner & MUTEX_FLAGS; 74 } 75 76 /* Do not use the return value as a pointer directly. */ 77 unsigned long mutex_get_owner(struct mutex *lock) 78 { 79 unsigned long owner = atomic_long_read(&lock->owner); 80 81 return (unsigned long)__owner_task(owner); 82 } 83 84 /* 85 * Returns: __mutex_owner(lock) on failure or NULL on success. 86 */ 87 static inline struct task_struct *__mutex_trylock_common(struct mutex *lock, bool handoff) 88 { 89 unsigned long owner, curr = (unsigned long)current; 90 91 owner = atomic_long_read(&lock->owner); 92 for (;;) { /* must loop, can race against a flag */ 93 unsigned long flags = __owner_flags(owner); 94 unsigned long task = owner & ~MUTEX_FLAGS; 95 96 if (task) { 97 if (flags & MUTEX_FLAG_PICKUP) { 98 if (task != curr) 99 break; 100 flags &= ~MUTEX_FLAG_PICKUP; 101 } else if (handoff) { 102 if (flags & MUTEX_FLAG_HANDOFF) 103 break; 104 flags |= MUTEX_FLAG_HANDOFF; 105 } else { 106 break; 107 } 108 } else { 109 MUTEX_WARN_ON(flags & (MUTEX_FLAG_HANDOFF | MUTEX_FLAG_PICKUP)); 110 task = curr; 111 } 112 113 if (atomic_long_try_cmpxchg_acquire(&lock->owner, &owner, task | flags)) { 114 if (task == curr) 115 return NULL; 116 break; 117 } 118 } 119 120 return __owner_task(owner); 121 } 122 123 /* 124 * Trylock or set HANDOFF 125 */ 126 static inline bool __mutex_trylock_or_handoff(struct mutex *lock, bool handoff) 127 { 128 return !__mutex_trylock_common(lock, handoff); 129 } 130 131 /* 132 * Actual trylock that will work on any unlocked state. 133 */ 134 static inline bool __mutex_trylock(struct mutex *lock) 135 { 136 return !__mutex_trylock_common(lock, false); 137 } 138 139 #ifndef CONFIG_DEBUG_LOCK_ALLOC 140 /* 141 * Lockdep annotations are contained to the slow paths for simplicity. 142 * There is nothing that would stop spreading the lockdep annotations outwards 143 * except more code. 144 */ 145 146 /* 147 * Optimistic trylock that only works in the uncontended case. Make sure to 148 * follow with a __mutex_trylock() before failing. 149 */ 150 static __always_inline bool __mutex_trylock_fast(struct mutex *lock) 151 { 152 unsigned long curr = (unsigned long)current; 153 unsigned long zero = 0UL; 154 155 MUTEX_WARN_ON(lock->magic != lock); 156 157 if (atomic_long_try_cmpxchg_acquire(&lock->owner, &zero, curr)) 158 return true; 159 160 return false; 161 } 162 163 static __always_inline bool __mutex_unlock_fast(struct mutex *lock) 164 { 165 unsigned long curr = (unsigned long)current; 166 167 return atomic_long_try_cmpxchg_release(&lock->owner, &curr, 0UL); 168 } 169 #endif 170 171 static inline void __mutex_set_flag(struct mutex *lock, unsigned long flag) 172 { 173 atomic_long_or(flag, &lock->owner); 174 } 175 176 static inline void __mutex_clear_flag(struct mutex *lock, unsigned long flag) 177 { 178 atomic_long_andnot(flag, &lock->owner); 179 } 180 181 static inline bool __mutex_waiter_is_first(struct mutex *lock, struct mutex_waiter *waiter) 182 { 183 return list_first_entry(&lock->wait_list, struct mutex_waiter, list) == waiter; 184 } 185 186 /* 187 * Add @waiter to a given location in the lock wait_list and set the 188 * FLAG_WAITERS flag if it's the first waiter. 189 */ 190 static void 191 __mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter, 192 struct list_head *list) 193 { 194 hung_task_set_blocker(lock, BLOCKER_TYPE_MUTEX); 195 debug_mutex_add_waiter(lock, waiter, current); 196 197 list_add_tail(&waiter->list, list); 198 if (__mutex_waiter_is_first(lock, waiter)) 199 __mutex_set_flag(lock, MUTEX_FLAG_WAITERS); 200 } 201 202 static void 203 __mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter) 204 { 205 list_del(&waiter->list); 206 if (likely(list_empty(&lock->wait_list))) 207 __mutex_clear_flag(lock, MUTEX_FLAGS); 208 209 debug_mutex_remove_waiter(lock, waiter, current); 210 hung_task_clear_blocker(); 211 } 212 213 /* 214 * Give up ownership to a specific task, when @task = NULL, this is equivalent 215 * to a regular unlock. Sets PICKUP on a handoff, clears HANDOFF, preserves 216 * WAITERS. Provides RELEASE semantics like a regular unlock, the 217 * __mutex_trylock() provides a matching ACQUIRE semantics for the handoff. 218 */ 219 static void __mutex_handoff(struct mutex *lock, struct task_struct *task) 220 { 221 unsigned long owner = atomic_long_read(&lock->owner); 222 223 for (;;) { 224 unsigned long new; 225 226 MUTEX_WARN_ON(__owner_task(owner) != current); 227 MUTEX_WARN_ON(owner & MUTEX_FLAG_PICKUP); 228 229 new = (owner & MUTEX_FLAG_WAITERS); 230 new |= (unsigned long)task; 231 if (task) 232 new |= MUTEX_FLAG_PICKUP; 233 234 if (atomic_long_try_cmpxchg_release(&lock->owner, &owner, new)) 235 break; 236 } 237 } 238 239 #ifndef CONFIG_DEBUG_LOCK_ALLOC 240 /* 241 * We split the mutex lock/unlock logic into separate fastpath and 242 * slowpath functions, to reduce the register pressure on the fastpath. 243 * We also put the fastpath first in the kernel image, to make sure the 244 * branch is predicted by the CPU as default-untaken. 245 */ 246 static void __sched __mutex_lock_slowpath(struct mutex *lock); 247 248 /** 249 * mutex_lock - acquire the mutex 250 * @lock: the mutex to be acquired 251 * 252 * Lock the mutex exclusively for this task. If the mutex is not 253 * available right now, it will sleep until it can get it. 254 * 255 * The mutex must later on be released by the same task that 256 * acquired it. Recursive locking is not allowed. The task 257 * may not exit without first unlocking the mutex. Also, kernel 258 * memory where the mutex resides must not be freed with 259 * the mutex still locked. The mutex must first be initialized 260 * (or statically defined) before it can be locked. memset()-ing 261 * the mutex to 0 is not allowed. 262 * 263 * (The CONFIG_DEBUG_MUTEXES .config option turns on debugging 264 * checks that will enforce the restrictions and will also do 265 * deadlock debugging) 266 * 267 * This function is similar to (but not equivalent to) down(). 268 */ 269 void __sched mutex_lock(struct mutex *lock) 270 { 271 might_sleep(); 272 273 if (!__mutex_trylock_fast(lock)) 274 __mutex_lock_slowpath(lock); 275 } 276 EXPORT_SYMBOL(mutex_lock); 277 #endif 278 279 #include "ww_mutex.h" 280 281 #ifdef CONFIG_MUTEX_SPIN_ON_OWNER 282 283 /* 284 * Trylock variant that returns the owning task on failure. 285 */ 286 static inline struct task_struct *__mutex_trylock_or_owner(struct mutex *lock) 287 { 288 return __mutex_trylock_common(lock, false); 289 } 290 291 static inline 292 bool ww_mutex_spin_on_owner(struct mutex *lock, struct ww_acquire_ctx *ww_ctx, 293 struct mutex_waiter *waiter) 294 { 295 struct ww_mutex *ww; 296 297 ww = container_of(lock, struct ww_mutex, base); 298 299 /* 300 * If ww->ctx is set the contents are undefined, only 301 * by acquiring wait_lock there is a guarantee that 302 * they are not invalid when reading. 303 * 304 * As such, when deadlock detection needs to be 305 * performed the optimistic spinning cannot be done. 306 * 307 * Check this in every inner iteration because we may 308 * be racing against another thread's ww_mutex_lock. 309 */ 310 if (ww_ctx->acquired > 0 && READ_ONCE(ww->ctx)) 311 return false; 312 313 /* 314 * If we aren't on the wait list yet, cancel the spin 315 * if there are waiters. We want to avoid stealing the 316 * lock from a waiter with an earlier stamp, since the 317 * other thread may already own a lock that we also 318 * need. 319 */ 320 if (!waiter && (atomic_long_read(&lock->owner) & MUTEX_FLAG_WAITERS)) 321 return false; 322 323 /* 324 * Similarly, stop spinning if we are no longer the 325 * first waiter. 326 */ 327 if (waiter && !__mutex_waiter_is_first(lock, waiter)) 328 return false; 329 330 return true; 331 } 332 333 /* 334 * Look out! "owner" is an entirely speculative pointer access and not 335 * reliable. 336 * 337 * "noinline" so that this function shows up on perf profiles. 338 */ 339 static noinline 340 bool mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner, 341 struct ww_acquire_ctx *ww_ctx, struct mutex_waiter *waiter) 342 { 343 bool ret = true; 344 345 lockdep_assert_preemption_disabled(); 346 347 while (__mutex_owner(lock) == owner) { 348 /* 349 * Ensure we emit the owner->on_cpu, dereference _after_ 350 * checking lock->owner still matches owner. And we already 351 * disabled preemption which is equal to the RCU read-side 352 * crital section in optimistic spinning code. Thus the 353 * task_strcut structure won't go away during the spinning 354 * period 355 */ 356 barrier(); 357 358 /* 359 * Use vcpu_is_preempted to detect lock holder preemption issue. 360 */ 361 if (!owner_on_cpu(owner) || need_resched()) { 362 ret = false; 363 break; 364 } 365 366 if (ww_ctx && !ww_mutex_spin_on_owner(lock, ww_ctx, waiter)) { 367 ret = false; 368 break; 369 } 370 371 cpu_relax(); 372 } 373 374 return ret; 375 } 376 377 /* 378 * Initial check for entering the mutex spinning loop 379 */ 380 static inline int mutex_can_spin_on_owner(struct mutex *lock) 381 { 382 struct task_struct *owner; 383 int retval = 1; 384 385 lockdep_assert_preemption_disabled(); 386 387 if (need_resched()) 388 return 0; 389 390 /* 391 * We already disabled preemption which is equal to the RCU read-side 392 * crital section in optimistic spinning code. Thus the task_strcut 393 * structure won't go away during the spinning period. 394 */ 395 owner = __mutex_owner(lock); 396 if (owner) 397 retval = owner_on_cpu(owner); 398 399 /* 400 * If lock->owner is not set, the mutex has been released. Return true 401 * such that we'll trylock in the spin path, which is a faster option 402 * than the blocking slow path. 403 */ 404 return retval; 405 } 406 407 /* 408 * Optimistic spinning. 409 * 410 * We try to spin for acquisition when we find that the lock owner 411 * is currently running on a (different) CPU and while we don't 412 * need to reschedule. The rationale is that if the lock owner is 413 * running, it is likely to release the lock soon. 414 * 415 * The mutex spinners are queued up using MCS lock so that only one 416 * spinner can compete for the mutex. However, if mutex spinning isn't 417 * going to happen, there is no point in going through the lock/unlock 418 * overhead. 419 * 420 * Returns true when the lock was taken, otherwise false, indicating 421 * that we need to jump to the slowpath and sleep. 422 * 423 * The waiter flag is set to true if the spinner is a waiter in the wait 424 * queue. The waiter-spinner will spin on the lock directly and concurrently 425 * with the spinner at the head of the OSQ, if present, until the owner is 426 * changed to itself. 427 */ 428 static __always_inline bool 429 mutex_optimistic_spin(struct mutex *lock, struct ww_acquire_ctx *ww_ctx, 430 struct mutex_waiter *waiter) 431 { 432 if (!waiter) { 433 /* 434 * The purpose of the mutex_can_spin_on_owner() function is 435 * to eliminate the overhead of osq_lock() and osq_unlock() 436 * in case spinning isn't possible. As a waiter-spinner 437 * is not going to take OSQ lock anyway, there is no need 438 * to call mutex_can_spin_on_owner(). 439 */ 440 if (!mutex_can_spin_on_owner(lock)) 441 goto fail; 442 443 /* 444 * In order to avoid a stampede of mutex spinners trying to 445 * acquire the mutex all at once, the spinners need to take a 446 * MCS (queued) lock first before spinning on the owner field. 447 */ 448 if (!osq_lock(&lock->osq)) 449 goto fail; 450 } 451 452 for (;;) { 453 struct task_struct *owner; 454 455 /* Try to acquire the mutex... */ 456 owner = __mutex_trylock_or_owner(lock); 457 if (!owner) 458 break; 459 460 /* 461 * There's an owner, wait for it to either 462 * release the lock or go to sleep. 463 */ 464 if (!mutex_spin_on_owner(lock, owner, ww_ctx, waiter)) 465 goto fail_unlock; 466 467 /* 468 * The cpu_relax() call is a compiler barrier which forces 469 * everything in this loop to be re-loaded. We don't need 470 * memory barriers as we'll eventually observe the right 471 * values at the cost of a few extra spins. 472 */ 473 cpu_relax(); 474 } 475 476 if (!waiter) 477 osq_unlock(&lock->osq); 478 479 return true; 480 481 482 fail_unlock: 483 if (!waiter) 484 osq_unlock(&lock->osq); 485 486 fail: 487 /* 488 * If we fell out of the spin path because of need_resched(), 489 * reschedule now, before we try-lock the mutex. This avoids getting 490 * scheduled out right after we obtained the mutex. 491 */ 492 if (need_resched()) { 493 /* 494 * We _should_ have TASK_RUNNING here, but just in case 495 * we do not, make it so, otherwise we might get stuck. 496 */ 497 __set_current_state(TASK_RUNNING); 498 schedule_preempt_disabled(); 499 } 500 501 return false; 502 } 503 #else 504 static __always_inline bool 505 mutex_optimistic_spin(struct mutex *lock, struct ww_acquire_ctx *ww_ctx, 506 struct mutex_waiter *waiter) 507 { 508 return false; 509 } 510 #endif 511 512 static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip); 513 514 /** 515 * mutex_unlock - release the mutex 516 * @lock: the mutex to be released 517 * 518 * Unlock a mutex that has been locked by this task previously. 519 * 520 * This function must not be used in interrupt context. Unlocking 521 * of a not locked mutex is not allowed. 522 * 523 * The caller must ensure that the mutex stays alive until this function has 524 * returned - mutex_unlock() can NOT directly be used to release an object such 525 * that another concurrent task can free it. 526 * Mutexes are different from spinlocks & refcounts in this aspect. 527 * 528 * This function is similar to (but not equivalent to) up(). 529 */ 530 void __sched mutex_unlock(struct mutex *lock) 531 { 532 #ifndef CONFIG_DEBUG_LOCK_ALLOC 533 if (__mutex_unlock_fast(lock)) 534 return; 535 #endif 536 __mutex_unlock_slowpath(lock, _RET_IP_); 537 } 538 EXPORT_SYMBOL(mutex_unlock); 539 540 /** 541 * ww_mutex_unlock - release the w/w mutex 542 * @lock: the mutex to be released 543 * 544 * Unlock a mutex that has been locked by this task previously with any of the 545 * ww_mutex_lock* functions (with or without an acquire context). It is 546 * forbidden to release the locks after releasing the acquire context. 547 * 548 * This function must not be used in interrupt context. Unlocking 549 * of a unlocked mutex is not allowed. 550 */ 551 void __sched ww_mutex_unlock(struct ww_mutex *lock) 552 { 553 __ww_mutex_unlock(lock); 554 mutex_unlock(&lock->base); 555 } 556 EXPORT_SYMBOL(ww_mutex_unlock); 557 558 /* 559 * Lock a mutex (possibly interruptible), slowpath: 560 */ 561 static __always_inline int __sched 562 __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclass, 563 struct lockdep_map *nest_lock, unsigned long ip, 564 struct ww_acquire_ctx *ww_ctx, const bool use_ww_ctx) 565 { 566 DEFINE_WAKE_Q(wake_q); 567 struct mutex_waiter waiter; 568 struct ww_mutex *ww; 569 unsigned long flags; 570 int ret; 571 572 if (!use_ww_ctx) 573 ww_ctx = NULL; 574 575 might_sleep(); 576 577 MUTEX_WARN_ON(lock->magic != lock); 578 579 ww = container_of(lock, struct ww_mutex, base); 580 if (ww_ctx) { 581 if (unlikely(ww_ctx == READ_ONCE(ww->ctx))) 582 return -EALREADY; 583 584 /* 585 * Reset the wounded flag after a kill. No other process can 586 * race and wound us here since they can't have a valid owner 587 * pointer if we don't have any locks held. 588 */ 589 if (ww_ctx->acquired == 0) 590 ww_ctx->wounded = 0; 591 592 #ifdef CONFIG_DEBUG_LOCK_ALLOC 593 nest_lock = &ww_ctx->dep_map; 594 #endif 595 } 596 597 preempt_disable(); 598 mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip); 599 600 trace_contention_begin(lock, LCB_F_MUTEX | LCB_F_SPIN); 601 if (__mutex_trylock(lock) || 602 mutex_optimistic_spin(lock, ww_ctx, NULL)) { 603 /* got the lock, yay! */ 604 lock_acquired(&lock->dep_map, ip); 605 if (ww_ctx) 606 ww_mutex_set_context_fastpath(ww, ww_ctx); 607 trace_contention_end(lock, 0); 608 preempt_enable(); 609 return 0; 610 } 611 612 raw_spin_lock_irqsave(&lock->wait_lock, flags); 613 /* 614 * After waiting to acquire the wait_lock, try again. 615 */ 616 if (__mutex_trylock(lock)) { 617 if (ww_ctx) 618 __ww_mutex_check_waiters(lock, ww_ctx, &wake_q); 619 620 goto skip_wait; 621 } 622 623 debug_mutex_lock_common(lock, &waiter); 624 waiter.task = current; 625 if (use_ww_ctx) 626 waiter.ww_ctx = ww_ctx; 627 628 lock_contended(&lock->dep_map, ip); 629 630 if (!use_ww_ctx) { 631 /* add waiting tasks to the end of the waitqueue (FIFO): */ 632 __mutex_add_waiter(lock, &waiter, &lock->wait_list); 633 } else { 634 /* 635 * Add in stamp order, waking up waiters that must kill 636 * themselves. 637 */ 638 ret = __ww_mutex_add_waiter(&waiter, lock, ww_ctx, &wake_q); 639 if (ret) 640 goto err_early_kill; 641 } 642 643 __set_task_blocked_on(current, lock); 644 set_current_state(state); 645 trace_contention_begin(lock, LCB_F_MUTEX); 646 for (;;) { 647 bool first; 648 649 /* 650 * Once we hold wait_lock, we're serialized against 651 * mutex_unlock() handing the lock off to us, do a trylock 652 * before testing the error conditions to make sure we pick up 653 * the handoff. 654 */ 655 if (__mutex_trylock(lock)) 656 goto acquired; 657 658 /* 659 * Check for signals and kill conditions while holding 660 * wait_lock. This ensures the lock cancellation is ordered 661 * against mutex_unlock() and wake-ups do not go missing. 662 */ 663 if (signal_pending_state(state, current)) { 664 ret = -EINTR; 665 goto err; 666 } 667 668 if (ww_ctx) { 669 ret = __ww_mutex_check_kill(lock, &waiter, ww_ctx); 670 if (ret) 671 goto err; 672 } 673 674 raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q); 675 676 schedule_preempt_disabled(); 677 678 first = __mutex_waiter_is_first(lock, &waiter); 679 680 /* 681 * As we likely have been woken up by task 682 * that has cleared our blocked_on state, re-set 683 * it to the lock we are trying to acquire. 684 */ 685 set_task_blocked_on(current, lock); 686 set_current_state(state); 687 /* 688 * Here we order against unlock; we must either see it change 689 * state back to RUNNING and fall through the next schedule(), 690 * or we must see its unlock and acquire. 691 */ 692 if (__mutex_trylock_or_handoff(lock, first)) 693 break; 694 695 if (first) { 696 trace_contention_begin(lock, LCB_F_MUTEX | LCB_F_SPIN); 697 /* 698 * mutex_optimistic_spin() can call schedule(), so 699 * clear blocked on so we don't become unselectable 700 * to run. 701 */ 702 clear_task_blocked_on(current, lock); 703 if (mutex_optimistic_spin(lock, ww_ctx, &waiter)) 704 break; 705 set_task_blocked_on(current, lock); 706 trace_contention_begin(lock, LCB_F_MUTEX); 707 } 708 709 raw_spin_lock_irqsave(&lock->wait_lock, flags); 710 } 711 raw_spin_lock_irqsave(&lock->wait_lock, flags); 712 acquired: 713 __clear_task_blocked_on(current, lock); 714 __set_current_state(TASK_RUNNING); 715 716 if (ww_ctx) { 717 /* 718 * Wound-Wait; we stole the lock (!first_waiter), check the 719 * waiters as anyone might want to wound us. 720 */ 721 if (!ww_ctx->is_wait_die && 722 !__mutex_waiter_is_first(lock, &waiter)) 723 __ww_mutex_check_waiters(lock, ww_ctx, &wake_q); 724 } 725 726 __mutex_remove_waiter(lock, &waiter); 727 728 debug_mutex_free_waiter(&waiter); 729 730 skip_wait: 731 /* got the lock - cleanup and rejoice! */ 732 lock_acquired(&lock->dep_map, ip); 733 trace_contention_end(lock, 0); 734 735 if (ww_ctx) 736 ww_mutex_lock_acquired(ww, ww_ctx); 737 738 raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q); 739 preempt_enable(); 740 return 0; 741 742 err: 743 __clear_task_blocked_on(current, lock); 744 __set_current_state(TASK_RUNNING); 745 __mutex_remove_waiter(lock, &waiter); 746 err_early_kill: 747 WARN_ON(__get_task_blocked_on(current)); 748 trace_contention_end(lock, ret); 749 raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q); 750 debug_mutex_free_waiter(&waiter); 751 mutex_release(&lock->dep_map, ip); 752 preempt_enable(); 753 return ret; 754 } 755 756 static int __sched 757 __mutex_lock(struct mutex *lock, unsigned int state, unsigned int subclass, 758 struct lockdep_map *nest_lock, unsigned long ip) 759 { 760 return __mutex_lock_common(lock, state, subclass, nest_lock, ip, NULL, false); 761 } 762 763 static int __sched 764 __ww_mutex_lock(struct mutex *lock, unsigned int state, unsigned int subclass, 765 unsigned long ip, struct ww_acquire_ctx *ww_ctx) 766 { 767 return __mutex_lock_common(lock, state, subclass, NULL, ip, ww_ctx, true); 768 } 769 770 /** 771 * ww_mutex_trylock - tries to acquire the w/w mutex with optional acquire context 772 * @ww: mutex to lock 773 * @ww_ctx: optional w/w acquire context 774 * 775 * Trylocks a mutex with the optional acquire context; no deadlock detection is 776 * possible. Returns 1 if the mutex has been acquired successfully, 0 otherwise. 777 * 778 * Unlike ww_mutex_lock, no deadlock handling is performed. However, if a @ctx is 779 * specified, -EALREADY handling may happen in calls to ww_mutex_trylock. 780 * 781 * A mutex acquired with this function must be released with ww_mutex_unlock. 782 */ 783 int ww_mutex_trylock(struct ww_mutex *ww, struct ww_acquire_ctx *ww_ctx) 784 { 785 if (!ww_ctx) 786 return mutex_trylock(&ww->base); 787 788 MUTEX_WARN_ON(ww->base.magic != &ww->base); 789 790 /* 791 * Reset the wounded flag after a kill. No other process can 792 * race and wound us here, since they can't have a valid owner 793 * pointer if we don't have any locks held. 794 */ 795 if (ww_ctx->acquired == 0) 796 ww_ctx->wounded = 0; 797 798 if (__mutex_trylock(&ww->base)) { 799 ww_mutex_set_context_fastpath(ww, ww_ctx); 800 mutex_acquire_nest(&ww->base.dep_map, 0, 1, &ww_ctx->dep_map, _RET_IP_); 801 return 1; 802 } 803 804 return 0; 805 } 806 EXPORT_SYMBOL(ww_mutex_trylock); 807 808 #ifdef CONFIG_DEBUG_LOCK_ALLOC 809 void __sched 810 mutex_lock_nested(struct mutex *lock, unsigned int subclass) 811 { 812 __mutex_lock(lock, TASK_UNINTERRUPTIBLE, subclass, NULL, _RET_IP_); 813 } 814 815 EXPORT_SYMBOL_GPL(mutex_lock_nested); 816 817 void __sched 818 _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest) 819 { 820 __mutex_lock(lock, TASK_UNINTERRUPTIBLE, 0, nest, _RET_IP_); 821 } 822 EXPORT_SYMBOL_GPL(_mutex_lock_nest_lock); 823 824 int __sched 825 _mutex_lock_killable(struct mutex *lock, unsigned int subclass, 826 struct lockdep_map *nest) 827 { 828 return __mutex_lock(lock, TASK_KILLABLE, subclass, nest, _RET_IP_); 829 } 830 EXPORT_SYMBOL_GPL(_mutex_lock_killable); 831 832 int __sched 833 mutex_lock_interruptible_nested(struct mutex *lock, unsigned int subclass) 834 { 835 return __mutex_lock(lock, TASK_INTERRUPTIBLE, subclass, NULL, _RET_IP_); 836 } 837 EXPORT_SYMBOL_GPL(mutex_lock_interruptible_nested); 838 839 void __sched 840 mutex_lock_io_nested(struct mutex *lock, unsigned int subclass) 841 { 842 int token; 843 844 might_sleep(); 845 846 token = io_schedule_prepare(); 847 __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 848 subclass, NULL, _RET_IP_, NULL, 0); 849 io_schedule_finish(token); 850 } 851 EXPORT_SYMBOL_GPL(mutex_lock_io_nested); 852 853 static inline int 854 ww_mutex_deadlock_injection(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) 855 { 856 #ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH 857 unsigned tmp; 858 859 if (ctx->deadlock_inject_countdown-- == 0) { 860 tmp = ctx->deadlock_inject_interval; 861 if (tmp > UINT_MAX/4) 862 tmp = UINT_MAX; 863 else 864 tmp = tmp*2 + tmp + tmp/2; 865 866 ctx->deadlock_inject_interval = tmp; 867 ctx->deadlock_inject_countdown = tmp; 868 ctx->contending_lock = lock; 869 870 ww_mutex_unlock(lock); 871 872 return -EDEADLK; 873 } 874 #endif 875 876 return 0; 877 } 878 879 int __sched 880 ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) 881 { 882 int ret; 883 884 might_sleep(); 885 ret = __ww_mutex_lock(&lock->base, TASK_UNINTERRUPTIBLE, 886 0, _RET_IP_, ctx); 887 if (!ret && ctx && ctx->acquired > 1) 888 return ww_mutex_deadlock_injection(lock, ctx); 889 890 return ret; 891 } 892 EXPORT_SYMBOL_GPL(ww_mutex_lock); 893 894 int __sched 895 ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) 896 { 897 int ret; 898 899 might_sleep(); 900 ret = __ww_mutex_lock(&lock->base, TASK_INTERRUPTIBLE, 901 0, _RET_IP_, ctx); 902 903 if (!ret && ctx && ctx->acquired > 1) 904 return ww_mutex_deadlock_injection(lock, ctx); 905 906 return ret; 907 } 908 EXPORT_SYMBOL_GPL(ww_mutex_lock_interruptible); 909 910 #endif 911 912 /* 913 * Release the lock, slowpath: 914 */ 915 static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip) 916 { 917 struct task_struct *next = NULL; 918 DEFINE_WAKE_Q(wake_q); 919 unsigned long owner; 920 unsigned long flags; 921 922 mutex_release(&lock->dep_map, ip); 923 924 /* 925 * Release the lock before (potentially) taking the spinlock such that 926 * other contenders can get on with things ASAP. 927 * 928 * Except when HANDOFF, in that case we must not clear the owner field, 929 * but instead set it to the top waiter. 930 */ 931 owner = atomic_long_read(&lock->owner); 932 for (;;) { 933 MUTEX_WARN_ON(__owner_task(owner) != current); 934 MUTEX_WARN_ON(owner & MUTEX_FLAG_PICKUP); 935 936 if (owner & MUTEX_FLAG_HANDOFF) 937 break; 938 939 if (atomic_long_try_cmpxchg_release(&lock->owner, &owner, __owner_flags(owner))) { 940 if (owner & MUTEX_FLAG_WAITERS) 941 break; 942 943 return; 944 } 945 } 946 947 raw_spin_lock_irqsave(&lock->wait_lock, flags); 948 debug_mutex_unlock(lock); 949 if (!list_empty(&lock->wait_list)) { 950 /* get the first entry from the wait-list: */ 951 struct mutex_waiter *waiter = 952 list_first_entry(&lock->wait_list, 953 struct mutex_waiter, list); 954 955 next = waiter->task; 956 957 debug_mutex_wake_waiter(lock, waiter); 958 __clear_task_blocked_on(next, lock); 959 wake_q_add(&wake_q, next); 960 } 961 962 if (owner & MUTEX_FLAG_HANDOFF) 963 __mutex_handoff(lock, next); 964 965 raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q); 966 } 967 968 #ifndef CONFIG_DEBUG_LOCK_ALLOC 969 /* 970 * Here come the less common (and hence less performance-critical) APIs: 971 * mutex_lock_interruptible() and mutex_trylock(). 972 */ 973 static noinline int __sched 974 __mutex_lock_killable_slowpath(struct mutex *lock); 975 976 static noinline int __sched 977 __mutex_lock_interruptible_slowpath(struct mutex *lock); 978 979 /** 980 * mutex_lock_interruptible() - Acquire the mutex, interruptible by signals. 981 * @lock: The mutex to be acquired. 982 * 983 * Lock the mutex like mutex_lock(). If a signal is delivered while the 984 * process is sleeping, this function will return without acquiring the 985 * mutex. 986 * 987 * Context: Process context. 988 * Return: 0 if the lock was successfully acquired or %-EINTR if a 989 * signal arrived. 990 */ 991 int __sched mutex_lock_interruptible(struct mutex *lock) 992 { 993 might_sleep(); 994 995 if (__mutex_trylock_fast(lock)) 996 return 0; 997 998 return __mutex_lock_interruptible_slowpath(lock); 999 } 1000 1001 EXPORT_SYMBOL(mutex_lock_interruptible); 1002 1003 /** 1004 * mutex_lock_killable() - Acquire the mutex, interruptible by fatal signals. 1005 * @lock: The mutex to be acquired. 1006 * 1007 * Lock the mutex like mutex_lock(). If a signal which will be fatal to 1008 * the current process is delivered while the process is sleeping, this 1009 * function will return without acquiring the mutex. 1010 * 1011 * Context: Process context. 1012 * Return: 0 if the lock was successfully acquired or %-EINTR if a 1013 * fatal signal arrived. 1014 */ 1015 int __sched mutex_lock_killable(struct mutex *lock) 1016 { 1017 might_sleep(); 1018 1019 if (__mutex_trylock_fast(lock)) 1020 return 0; 1021 1022 return __mutex_lock_killable_slowpath(lock); 1023 } 1024 EXPORT_SYMBOL(mutex_lock_killable); 1025 1026 /** 1027 * mutex_lock_io() - Acquire the mutex and mark the process as waiting for I/O 1028 * @lock: The mutex to be acquired. 1029 * 1030 * Lock the mutex like mutex_lock(). While the task is waiting for this 1031 * mutex, it will be accounted as being in the IO wait state by the 1032 * scheduler. 1033 * 1034 * Context: Process context. 1035 */ 1036 void __sched mutex_lock_io(struct mutex *lock) 1037 { 1038 int token; 1039 1040 token = io_schedule_prepare(); 1041 mutex_lock(lock); 1042 io_schedule_finish(token); 1043 } 1044 EXPORT_SYMBOL_GPL(mutex_lock_io); 1045 1046 static noinline void __sched 1047 __mutex_lock_slowpath(struct mutex *lock) 1048 { 1049 __mutex_lock(lock, TASK_UNINTERRUPTIBLE, 0, NULL, _RET_IP_); 1050 } 1051 1052 static noinline int __sched 1053 __mutex_lock_killable_slowpath(struct mutex *lock) 1054 { 1055 return __mutex_lock(lock, TASK_KILLABLE, 0, NULL, _RET_IP_); 1056 } 1057 1058 static noinline int __sched 1059 __mutex_lock_interruptible_slowpath(struct mutex *lock) 1060 { 1061 return __mutex_lock(lock, TASK_INTERRUPTIBLE, 0, NULL, _RET_IP_); 1062 } 1063 1064 static noinline int __sched 1065 __ww_mutex_lock_slowpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) 1066 { 1067 return __ww_mutex_lock(&lock->base, TASK_UNINTERRUPTIBLE, 0, 1068 _RET_IP_, ctx); 1069 } 1070 1071 static noinline int __sched 1072 __ww_mutex_lock_interruptible_slowpath(struct ww_mutex *lock, 1073 struct ww_acquire_ctx *ctx) 1074 { 1075 return __ww_mutex_lock(&lock->base, TASK_INTERRUPTIBLE, 0, 1076 _RET_IP_, ctx); 1077 } 1078 1079 #endif 1080 1081 #ifndef CONFIG_DEBUG_LOCK_ALLOC 1082 /** 1083 * mutex_trylock - try to acquire the mutex, without waiting 1084 * @lock: the mutex to be acquired 1085 * 1086 * Try to acquire the mutex atomically. Returns 1 if the mutex 1087 * has been acquired successfully, and 0 on contention. 1088 * 1089 * NOTE: this function follows the spin_trylock() convention, so 1090 * it is negated from the down_trylock() return values! Be careful 1091 * about this when converting semaphore users to mutexes. 1092 * 1093 * This function must not be used in interrupt context. The 1094 * mutex must be released by the same task that acquired it. 1095 */ 1096 int __sched mutex_trylock(struct mutex *lock) 1097 { 1098 MUTEX_WARN_ON(lock->magic != lock); 1099 return __mutex_trylock(lock); 1100 } 1101 EXPORT_SYMBOL(mutex_trylock); 1102 #else 1103 int __sched _mutex_trylock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock) 1104 { 1105 bool locked; 1106 1107 MUTEX_WARN_ON(lock->magic != lock); 1108 locked = __mutex_trylock(lock); 1109 if (locked) 1110 mutex_acquire_nest(&lock->dep_map, 0, 1, nest_lock, _RET_IP_); 1111 1112 return locked; 1113 } 1114 EXPORT_SYMBOL(_mutex_trylock_nest_lock); 1115 #endif 1116 1117 #ifndef CONFIG_DEBUG_LOCK_ALLOC 1118 int __sched 1119 ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) 1120 { 1121 might_sleep(); 1122 1123 if (__mutex_trylock_fast(&lock->base)) { 1124 if (ctx) 1125 ww_mutex_set_context_fastpath(lock, ctx); 1126 return 0; 1127 } 1128 1129 return __ww_mutex_lock_slowpath(lock, ctx); 1130 } 1131 EXPORT_SYMBOL(ww_mutex_lock); 1132 1133 int __sched 1134 ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) 1135 { 1136 might_sleep(); 1137 1138 if (__mutex_trylock_fast(&lock->base)) { 1139 if (ctx) 1140 ww_mutex_set_context_fastpath(lock, ctx); 1141 return 0; 1142 } 1143 1144 return __ww_mutex_lock_interruptible_slowpath(lock, ctx); 1145 } 1146 EXPORT_SYMBOL(ww_mutex_lock_interruptible); 1147 1148 #endif /* !CONFIG_DEBUG_LOCK_ALLOC */ 1149 #endif /* !CONFIG_PREEMPT_RT */ 1150 1151 EXPORT_TRACEPOINT_SYMBOL_GPL(contention_begin); 1152 EXPORT_TRACEPOINT_SYMBOL_GPL(contention_end); 1153 1154 /** 1155 * atomic_dec_and_mutex_lock - return holding mutex if we dec to 0 1156 * @cnt: the atomic which we are to dec 1157 * @lock: the mutex to return holding if we dec to 0 1158 * 1159 * return true and hold lock if we dec to 0, return false otherwise 1160 */ 1161 int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock) 1162 { 1163 /* dec if we can't possibly hit 0 */ 1164 if (atomic_add_unless(cnt, -1, 1)) 1165 return 0; 1166 /* we might hit 0, so take the lock */ 1167 mutex_lock(lock); 1168 if (!atomic_dec_and_test(cnt)) { 1169 /* when we actually did the dec, we didn't hit 0 */ 1170 mutex_unlock(lock); 1171 return 0; 1172 } 1173 /* we hit 0, and we hold the lock */ 1174 return 1; 1175 } 1176 EXPORT_SYMBOL(atomic_dec_and_mutex_lock);