개념 설명 전체 · v6.6 / mm/filemap.c
1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/mm/filemap.c 4 * 5 * Copyright (C) 1994-1999 Linus Torvalds 6 */ 7 8 /* 9 * This file handles the generic file mmap semantics used by 10 * most "normal" filesystems (but you don't /have/ to use this: 11 * the NFS filesystem used to do this differently, for example) 12 */ 13 #include <linux/export.h> 14 #include <linux/compiler.h> 15 #include <linux/dax.h> 16 #include <linux/fs.h> 17 #include <linux/sched/signal.h> 18 #include <linux/uaccess.h> 19 #include <linux/capability.h> 20 #include <linux/kernel_stat.h> 21 #include <linux/gfp.h> 22 #include <linux/mm.h> 23 #include <linux/swap.h> 24 #include <linux/swapops.h> 25 #include <linux/syscalls.h> 26 #include <linux/mman.h> 27 #include <linux/pagemap.h> 28 #include <linux/file.h> 29 #include <linux/uio.h> 30 #include <linux/error-injection.h> 31 #include <linux/hash.h> 32 #include <linux/writeback.h> 33 #include <linux/backing-dev.h> 34 #include <linux/pagevec.h> 35 #include <linux/security.h> 36 #include <linux/cpuset.h> 37 #include <linux/hugetlb.h> 38 #include <linux/memcontrol.h> 39 #include <linux/shmem_fs.h> 40 #include <linux/rmap.h> 41 #include <linux/delayacct.h> 42 #include <linux/psi.h> 43 #include <linux/ramfs.h> 44 #include <linux/page_idle.h> 45 #include <linux/migrate.h> 46 #include <linux/pipe_fs_i.h> 47 #include <linux/splice.h> 48 #include <asm/pgalloc.h> 49 #include <asm/tlbflush.h> 50 #include "internal.h" 51 52 #define CREATE_TRACE_POINTS 53 #include <trace/events/filemap.h> 54 55 /* 56 * FIXME: remove all knowledge of the buffer layer from the core VM 57 */ 58 #include <linux/buffer_head.h> /* for try_to_free_buffers */ 59 60 #include <asm/mman.h> 61 62 #include "swap.h" 63 64 /* 65 * Shared mappings implemented 30.11.1994. It's not fully working yet, 66 * though. 67 * 68 * Shared mappings now work. 15.8.1995 Bruno. 69 * 70 * finished 'unifying' the page and buffer cache and SMP-threaded the 71 * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com> 72 * 73 * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de> 74 */ 75 76 /* 77 * Lock ordering: 78 * 79 * ->i_mmap_rwsem (truncate_pagecache) 80 * ->private_lock (__free_pte->block_dirty_folio) 81 * ->swap_lock (exclusive_swap_page, others) 82 * ->i_pages lock 83 * 84 * ->i_rwsem 85 * ->invalidate_lock (acquired by fs in truncate path) 86 * ->i_mmap_rwsem (truncate->unmap_mapping_range) 87 * 88 * ->mmap_lock 89 * ->i_mmap_rwsem 90 * ->page_table_lock or pte_lock (various, mainly in memory.c) 91 * ->i_pages lock (arch-dependent flush_dcache_mmap_lock) 92 * 93 * ->mmap_lock 94 * ->invalidate_lock (filemap_fault) 95 * ->lock_page (filemap_fault, access_process_vm) 96 * 97 * ->i_rwsem (generic_perform_write) 98 * ->mmap_lock (fault_in_readable->do_page_fault) 99 * 100 * bdi->wb.list_lock 101 * sb_lock (fs/fs-writeback.c) 102 * ->i_pages lock (__sync_single_inode) 103 * 104 * ->i_mmap_rwsem 105 * ->anon_vma.lock (vma_merge) 106 * 107 * ->anon_vma.lock 108 * ->page_table_lock or pte_lock (anon_vma_prepare and various) 109 * 110 * ->page_table_lock or pte_lock 111 * ->swap_lock (try_to_unmap_one) 112 * ->private_lock (try_to_unmap_one) 113 * ->i_pages lock (try_to_unmap_one) 114 * ->lruvec->lru_lock (follow_page->mark_page_accessed) 115 * ->lruvec->lru_lock (check_pte_range->isolate_lru_page) 116 * ->private_lock (page_remove_rmap->set_page_dirty) 117 * ->i_pages lock (page_remove_rmap->set_page_dirty) 118 * bdi.wb->list_lock (page_remove_rmap->set_page_dirty) 119 * ->inode->i_lock (page_remove_rmap->set_page_dirty) 120 * ->memcg->move_lock (page_remove_rmap->folio_memcg_lock) 121 * bdi.wb->list_lock (zap_pte_range->set_page_dirty) 122 * ->inode->i_lock (zap_pte_range->set_page_dirty) 123 * ->private_lock (zap_pte_range->block_dirty_folio) 124 */ 125 126 static void page_cache_delete(struct address_space *mapping, 127 struct folio *folio, void *shadow) 128 { 129 XA_STATE(xas, &mapping->i_pages, folio->index); 130 long nr = 1; 131 132 mapping_set_update(&xas, mapping); 133 134 /* hugetlb pages are represented by a single entry in the xarray */ 135 if (!folio_test_hugetlb(folio)) { 136 xas_set_order(&xas, folio->index, folio_order(folio)); 137 nr = folio_nr_pages(folio); 138 } 139 140 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 141 142 xas_store(&xas, shadow); 143 xas_init_marks(&xas); 144 145 folio->mapping = NULL; 146 /* Leave page->index set: truncation lookup relies upon it */ 147 mapping->nrpages -= nr; 148 } 149 150 static void filemap_unaccount_folio(struct address_space *mapping, 151 struct folio *folio) 152 { 153 long nr; 154 155 VM_BUG_ON_FOLIO(folio_mapped(folio), folio); 156 if (!IS_ENABLED(CONFIG_DEBUG_VM) && unlikely(folio_mapped(folio))) { 157 pr_alert("BUG: Bad page cache in process %s pfn:%05lx\n", 158 current->comm, folio_pfn(folio)); 159 dump_page(&folio->page, "still mapped when deleted"); 160 dump_stack(); 161 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 162 163 if (mapping_exiting(mapping) && !folio_test_large(folio)) { 164 int mapcount = page_mapcount(&folio->page); 165 166 if (folio_ref_count(folio) >= mapcount + 2) { 167 /* 168 * All vmas have already been torn down, so it's 169 * a good bet that actually the page is unmapped 170 * and we'd rather not leak it: if we're wrong, 171 * another bad page check should catch it later. 172 */ 173 page_mapcount_reset(&folio->page); 174 folio_ref_sub(folio, mapcount); 175 } 176 } 177 } 178 179 /* hugetlb folios do not participate in page cache accounting. */ 180 if (folio_test_hugetlb(folio)) 181 return; 182 183 nr = folio_nr_pages(folio); 184 185 __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, -nr); 186 if (folio_test_swapbacked(folio)) { 187 __lruvec_stat_mod_folio(folio, NR_SHMEM, -nr); 188 if (folio_test_pmd_mappable(folio)) 189 __lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, -nr); 190 } else if (folio_test_pmd_mappable(folio)) { 191 __lruvec_stat_mod_folio(folio, NR_FILE_THPS, -nr); 192 filemap_nr_thps_dec(mapping); 193 } 194 195 /* 196 * At this point folio must be either written or cleaned by 197 * truncate. Dirty folio here signals a bug and loss of 198 * unwritten data - on ordinary filesystems. 199 * 200 * But it's harmless on in-memory filesystems like tmpfs; and can 201 * occur when a driver which did get_user_pages() sets page dirty 202 * before putting it, while the inode is being finally evicted. 203 * 204 * Below fixes dirty accounting after removing the folio entirely 205 * but leaves the dirty flag set: it has no effect for truncated 206 * folio and anyway will be cleared before returning folio to 207 * buddy allocator. 208 */ 209 if (WARN_ON_ONCE(folio_test_dirty(folio) && 210 mapping_can_writeback(mapping))) 211 folio_account_cleaned(folio, inode_to_wb(mapping->host)); 212 } 213 214 /* 215 * Delete a page from the page cache and free it. Caller has to make 216 * sure the page is locked and that nobody else uses it - or that usage 217 * is safe. The caller must hold the i_pages lock. 218 */ 219 void __filemap_remove_folio(struct folio *folio, void *shadow) 220 { 221 struct address_space *mapping = folio->mapping; 222 223 trace_mm_filemap_delete_from_page_cache(folio); 224 filemap_unaccount_folio(mapping, folio); 225 page_cache_delete(mapping, folio, shadow); 226 } 227 228 void filemap_free_folio(struct address_space *mapping, struct folio *folio) 229 { 230 void (*free_folio)(struct folio *); 231 int refs = 1; 232 233 free_folio = mapping->a_ops->free_folio; 234 if (free_folio) 235 free_folio(folio); 236 237 if (folio_test_large(folio) && !folio_test_hugetlb(folio)) 238 refs = folio_nr_pages(folio); 239 folio_put_refs(folio, refs); 240 } 241 242 /** 243 * filemap_remove_folio - Remove folio from page cache. 244 * @folio: The folio. 245 * 246 * This must be called only on folios that are locked and have been 247 * verified to be in the page cache. It will never put the folio into 248 * the free list because the caller has a reference on the page. 249 */ 250 void filemap_remove_folio(struct folio *folio) 251 { 252 struct address_space *mapping = folio->mapping; 253 254 BUG_ON(!folio_test_locked(folio)); 255 spin_lock(&mapping->host->i_lock); 256 xa_lock_irq(&mapping->i_pages); 257 __filemap_remove_folio(folio, NULL); 258 xa_unlock_irq(&mapping->i_pages); 259 if (mapping_shrinkable(mapping)) 260 inode_add_lru(mapping->host); 261 spin_unlock(&mapping->host->i_lock); 262 263 filemap_free_folio(mapping, folio); 264 } 265 266 /* 267 * page_cache_delete_batch - delete several folios from page cache 268 * @mapping: the mapping to which folios belong 269 * @fbatch: batch of folios to delete 270 * 271 * The function walks over mapping->i_pages and removes folios passed in 272 * @fbatch from the mapping. The function expects @fbatch to be sorted 273 * by page index and is optimised for it to be dense. 274 * It tolerates holes in @fbatch (mapping entries at those indices are not 275 * modified). 276 * 277 * The function expects the i_pages lock to be held. 278 */ 279 static void page_cache_delete_batch(struct address_space *mapping, 280 struct folio_batch *fbatch) 281 { 282 XA_STATE(xas, &mapping->i_pages, fbatch->folios[0]->index); 283 long total_pages = 0; 284 int i = 0; 285 struct folio *folio; 286 287 mapping_set_update(&xas, mapping); 288 xas_for_each(&xas, folio, ULONG_MAX) { 289 if (i >= folio_batch_count(fbatch)) 290 break; 291 292 /* A swap/dax/shadow entry got inserted? Skip it. */ 293 if (xa_is_value(folio)) 294 continue; 295 /* 296 * A page got inserted in our range? Skip it. We have our 297 * pages locked so they are protected from being removed. 298 * If we see a page whose index is higher than ours, it 299 * means our page has been removed, which shouldn't be 300 * possible because we're holding the PageLock. 301 */ 302 if (folio != fbatch->folios[i]) { 303 VM_BUG_ON_FOLIO(folio->index > 304 fbatch->folios[i]->index, folio); 305 continue; 306 } 307 308 WARN_ON_ONCE(!folio_test_locked(folio)); 309 310 folio->mapping = NULL; 311 /* Leave folio->index set: truncation lookup relies on it */ 312 313 i++; 314 xas_store(&xas, NULL); 315 total_pages += folio_nr_pages(folio); 316 } 317 mapping->nrpages -= total_pages; 318 } 319 320 void delete_from_page_cache_batch(struct address_space *mapping, 321 struct folio_batch *fbatch) 322 { 323 int i; 324 325 if (!folio_batch_count(fbatch)) 326 return; 327 328 spin_lock(&mapping->host->i_lock); 329 xa_lock_irq(&mapping->i_pages); 330 for (i = 0; i < folio_batch_count(fbatch); i++) { 331 struct folio *folio = fbatch->folios[i]; 332 333 trace_mm_filemap_delete_from_page_cache(folio); 334 filemap_unaccount_folio(mapping, folio); 335 } 336 page_cache_delete_batch(mapping, fbatch); 337 xa_unlock_irq(&mapping->i_pages); 338 if (mapping_shrinkable(mapping)) 339 inode_add_lru(mapping->host); 340 spin_unlock(&mapping->host->i_lock); 341 342 for (i = 0; i < folio_batch_count(fbatch); i++) 343 filemap_free_folio(mapping, fbatch->folios[i]); 344 } 345 346 int filemap_check_errors(struct address_space *mapping) 347 { 348 int ret = 0; 349 /* Check for outstanding write errors */ 350 if (test_bit(AS_ENOSPC, &mapping->flags) && 351 test_and_clear_bit(AS_ENOSPC, &mapping->flags)) 352 ret = -ENOSPC; 353 if (test_bit(AS_EIO, &mapping->flags) && 354 test_and_clear_bit(AS_EIO, &mapping->flags)) 355 ret = -EIO; 356 return ret; 357 } 358 EXPORT_SYMBOL(filemap_check_errors); 359 360 static int filemap_check_and_keep_errors(struct address_space *mapping) 361 { 362 /* Check for outstanding write errors */ 363 if (test_bit(AS_EIO, &mapping->flags)) 364 return -EIO; 365 if (test_bit(AS_ENOSPC, &mapping->flags)) 366 return -ENOSPC; 367 return 0; 368 } 369 370 /** 371 * filemap_fdatawrite_wbc - start writeback on mapping dirty pages in range 372 * @mapping: address space structure to write 373 * @wbc: the writeback_control controlling the writeout 374 * 375 * Call writepages on the mapping using the provided wbc to control the 376 * writeout. 377 * 378 * Return: %0 on success, negative error code otherwise. 379 */ 380 int filemap_fdatawrite_wbc(struct address_space *mapping, 381 struct writeback_control *wbc) 382 { 383 int ret; 384 385 if (!mapping_can_writeback(mapping) || 386 !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) 387 return 0; 388 389 wbc_attach_fdatawrite_inode(wbc, mapping->host); 390 ret = do_writepages(mapping, wbc); 391 wbc_detach_inode(wbc); 392 return ret; 393 } 394 EXPORT_SYMBOL(filemap_fdatawrite_wbc); 395 396 /** 397 * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range 398 * @mapping: address space structure to write 399 * @start: offset in bytes where the range starts 400 * @end: offset in bytes where the range ends (inclusive) 401 * @sync_mode: enable synchronous operation 402 * 403 * Start writeback against all of a mapping's dirty pages that lie 404 * within the byte offsets <start, end> inclusive. 405 * 406 * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as 407 * opposed to a regular memory cleansing writeback. The difference between 408 * these two operations is that if a dirty page/buffer is encountered, it must 409 * be waited upon, and not just skipped over. 410 * 411 * Return: %0 on success, negative error code otherwise. 412 */ 413 int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start, 414 loff_t end, int sync_mode) 415 { 416 struct writeback_control wbc = { 417 .sync_mode = sync_mode, 418 .nr_to_write = LONG_MAX, 419 .range_start = start, 420 .range_end = end, 421 }; 422 423 return filemap_fdatawrite_wbc(mapping, &wbc); 424 } 425 426 static inline int __filemap_fdatawrite(struct address_space *mapping, 427 int sync_mode) 428 { 429 return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode); 430 } 431 432 int filemap_fdatawrite(struct address_space *mapping) 433 { 434 return __filemap_fdatawrite(mapping, WB_SYNC_ALL); 435 } 436 EXPORT_SYMBOL(filemap_fdatawrite); 437 438 int filemap_fdatawrite_range(struct address_space *mapping, loff_t start, 439 loff_t end) 440 { 441 return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL); 442 } 443 EXPORT_SYMBOL(filemap_fdatawrite_range); 444 445 /** 446 * filemap_flush - mostly a non-blocking flush 447 * @mapping: target address_space 448 * 449 * This is a mostly non-blocking flush. Not suitable for data-integrity 450 * purposes - I/O may not be started against all dirty pages. 451 * 452 * Return: %0 on success, negative error code otherwise. 453 */ 454 int filemap_flush(struct address_space *mapping) 455 { 456 return __filemap_fdatawrite(mapping, WB_SYNC_NONE); 457 } 458 EXPORT_SYMBOL(filemap_flush); 459 460 /** 461 * filemap_range_has_page - check if a page exists in range. 462 * @mapping: address space within which to check 463 * @start_byte: offset in bytes where the range starts 464 * @end_byte: offset in bytes where the range ends (inclusive) 465 * 466 * Find at least one page in the range supplied, usually used to check if 467 * direct writing in this range will trigger a writeback. 468 * 469 * Return: %true if at least one page exists in the specified range, 470 * %false otherwise. 471 */ 472 bool filemap_range_has_page(struct address_space *mapping, 473 loff_t start_byte, loff_t end_byte) 474 { 475 struct folio *folio; 476 XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT); 477 pgoff_t max = end_byte >> PAGE_SHIFT; 478 479 if (end_byte < start_byte) 480 return false; 481 482 rcu_read_lock(); 483 for (;;) { 484 folio = xas_find(&xas, max); 485 if (xas_retry(&xas, folio)) 486 continue; 487 /* Shadow entries don't count */ 488 if (xa_is_value(folio)) 489 continue; 490 /* 491 * We don't need to try to pin this page; we're about to 492 * release the RCU lock anyway. It is enough to know that 493 * there was a page here recently. 494 */ 495 break; 496 } 497 rcu_read_unlock(); 498 499 return folio != NULL; 500 } 501 EXPORT_SYMBOL(filemap_range_has_page); 502 503 static void __filemap_fdatawait_range(struct address_space *mapping, 504 loff_t start_byte, loff_t end_byte) 505 { 506 pgoff_t index = start_byte >> PAGE_SHIFT; 507 pgoff_t end = end_byte >> PAGE_SHIFT; 508 struct folio_batch fbatch; 509 unsigned nr_folios; 510 511 folio_batch_init(&fbatch); 512 513 while (index <= end) { 514 unsigned i; 515 516 nr_folios = filemap_get_folios_tag(mapping, &index, end, 517 PAGECACHE_TAG_WRITEBACK, &fbatch); 518 519 if (!nr_folios) 520 break; 521 522 for (i = 0; i < nr_folios; i++) { 523 struct folio *folio = fbatch.folios[i]; 524 525 folio_wait_writeback(folio); 526 folio_clear_error(folio); 527 } 528 folio_batch_release(&fbatch); 529 cond_resched(); 530 } 531 } 532 533 /** 534 * filemap_fdatawait_range - wait for writeback to complete 535 * @mapping: address space structure to wait for 536 * @start_byte: offset in bytes where the range starts 537 * @end_byte: offset in bytes where the range ends (inclusive) 538 * 539 * Walk the list of under-writeback pages of the given address space 540 * in the given range and wait for all of them. Check error status of 541 * the address space and return it. 542 * 543 * Since the error status of the address space is cleared by this function, 544 * callers are responsible for checking the return value and handling and/or 545 * reporting the error. 546 * 547 * Return: error status of the address space. 548 */ 549 int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte, 550 loff_t end_byte) 551 { 552 __filemap_fdatawait_range(mapping, start_byte, end_byte); 553 return filemap_check_errors(mapping); 554 } 555 EXPORT_SYMBOL(filemap_fdatawait_range); 556 557 /** 558 * filemap_fdatawait_range_keep_errors - wait for writeback to complete 559 * @mapping: address space structure to wait for 560 * @start_byte: offset in bytes where the range starts 561 * @end_byte: offset in bytes where the range ends (inclusive) 562 * 563 * Walk the list of under-writeback pages of the given address space in the 564 * given range and wait for all of them. Unlike filemap_fdatawait_range(), 565 * this function does not clear error status of the address space. 566 * 567 * Use this function if callers don't handle errors themselves. Expected 568 * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2), 569 * fsfreeze(8) 570 */ 571 int filemap_fdatawait_range_keep_errors(struct address_space *mapping, 572 loff_t start_byte, loff_t end_byte) 573 { 574 __filemap_fdatawait_range(mapping, start_byte, end_byte); 575 return filemap_check_and_keep_errors(mapping); 576 } 577 EXPORT_SYMBOL(filemap_fdatawait_range_keep_errors); 578 579 /** 580 * file_fdatawait_range - wait for writeback to complete 581 * @file: file pointing to address space structure to wait for 582 * @start_byte: offset in bytes where the range starts 583 * @end_byte: offset in bytes where the range ends (inclusive) 584 * 585 * Walk the list of under-writeback pages of the address space that file 586 * refers to, in the given range and wait for all of them. Check error 587 * status of the address space vs. the file->f_wb_err cursor and return it. 588 * 589 * Since the error status of the file is advanced by this function, 590 * callers are responsible for checking the return value and handling and/or 591 * reporting the error. 592 * 593 * Return: error status of the address space vs. the file->f_wb_err cursor. 594 */ 595 int file_fdatawait_range(struct file *file, loff_t start_byte, loff_t end_byte) 596 { 597 struct address_space *mapping = file->f_mapping; 598 599 __filemap_fdatawait_range(mapping, start_byte, end_byte); 600 return file_check_and_advance_wb_err(file); 601 } 602 EXPORT_SYMBOL(file_fdatawait_range); 603 604 /** 605 * filemap_fdatawait_keep_errors - wait for writeback without clearing errors 606 * @mapping: address space structure to wait for 607 * 608 * Walk the list of under-writeback pages of the given address space 609 * and wait for all of them. Unlike filemap_fdatawait(), this function 610 * does not clear error status of the address space. 611 * 612 * Use this function if callers don't handle errors themselves. Expected 613 * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2), 614 * fsfreeze(8) 615 * 616 * Return: error status of the address space. 617 */ 618 int filemap_fdatawait_keep_errors(struct address_space *mapping) 619 { 620 __filemap_fdatawait_range(mapping, 0, LLONG_MAX); 621 return filemap_check_and_keep_errors(mapping); 622 } 623 EXPORT_SYMBOL(filemap_fdatawait_keep_errors); 624 625 /* Returns true if writeback might be needed or already in progress. */ 626 static bool mapping_needs_writeback(struct address_space *mapping) 627 { 628 return mapping->nrpages; 629 } 630 631 bool filemap_range_has_writeback(struct address_space *mapping, 632 loff_t start_byte, loff_t end_byte) 633 { 634 XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT); 635 pgoff_t max = end_byte >> PAGE_SHIFT; 636 struct folio *folio; 637 638 if (end_byte < start_byte) 639 return false; 640 641 rcu_read_lock(); 642 xas_for_each(&xas, folio, max) { 643 if (xas_retry(&xas, folio)) 644 continue; 645 if (xa_is_value(folio)) 646 continue; 647 if (folio_test_dirty(folio) || folio_test_locked(folio) || 648 folio_test_writeback(folio)) 649 break; 650 } 651 rcu_read_unlock(); 652 return folio != NULL; 653 } 654 EXPORT_SYMBOL_GPL(filemap_range_has_writeback); 655 656 /** 657 * filemap_write_and_wait_range - write out & wait on a file range 658 * @mapping: the address_space for the pages 659 * @lstart: offset in bytes where the range starts 660 * @lend: offset in bytes where the range ends (inclusive) 661 * 662 * Write out and wait upon file offsets lstart->lend, inclusive. 663 * 664 * Note that @lend is inclusive (describes the last byte to be written) so 665 * that this function can be used to write to the very end-of-file (end = -1). 666 * 667 * Return: error status of the address space. 668 */ 669 int filemap_write_and_wait_range(struct address_space *mapping, 670 loff_t lstart, loff_t lend) 671 { 672 int err = 0, err2; 673 674 if (lend < lstart) 675 return 0; 676 677 if (mapping_needs_writeback(mapping)) { 678 err = __filemap_fdatawrite_range(mapping, lstart, lend, 679 WB_SYNC_ALL); 680 /* 681 * Even if the above returned error, the pages may be 682 * written partially (e.g. -ENOSPC), so we wait for it. 683 * But the -EIO is special case, it may indicate the worst 684 * thing (e.g. bug) happened, so we avoid waiting for it. 685 */ 686 if (err != -EIO) 687 __filemap_fdatawait_range(mapping, lstart, lend); 688 } 689 err2 = filemap_check_errors(mapping); 690 if (!err) 691 err = err2; 692 return err; 693 } 694 EXPORT_SYMBOL(filemap_write_and_wait_range); 695 696 void __filemap_set_wb_err(struct address_space *mapping, int err) 697 { 698 errseq_t eseq = errseq_set(&mapping->wb_err, err); 699 700 trace_filemap_set_wb_err(mapping, eseq); 701 } 702 EXPORT_SYMBOL(__filemap_set_wb_err); 703 704 /** 705 * file_check_and_advance_wb_err - report wb error (if any) that was previously 706 * and advance wb_err to current one 707 * @file: struct file on which the error is being reported 708 * 709 * When userland calls fsync (or something like nfsd does the equivalent), we 710 * want to report any writeback errors that occurred since the last fsync (or 711 * since the file was opened if there haven't been any). 712 * 713 * Grab the wb_err from the mapping. If it matches what we have in the file, 714 * then just quickly return 0. The file is all caught up. 715 * 716 * If it doesn't match, then take the mapping value, set the "seen" flag in 717 * it and try to swap it into place. If it works, or another task beat us 718 * to it with the new value, then update the f_wb_err and return the error 719 * portion. The error at this point must be reported via proper channels 720 * (a'la fsync, or NFS COMMIT operation, etc.). 721 * 722 * While we handle mapping->wb_err with atomic operations, the f_wb_err 723 * value is protected by the f_lock since we must ensure that it reflects 724 * the latest value swapped in for this file descriptor. 725 * 726 * Return: %0 on success, negative error code otherwise. 727 */ 728 int file_check_and_advance_wb_err(struct file *file) 729 { 730 int err = 0; 731 errseq_t old = READ_ONCE(file->f_wb_err); 732 struct address_space *mapping = file->f_mapping; 733 734 /* Locklessly handle the common case where nothing has changed */ 735 if (errseq_check(&mapping->wb_err, old)) { 736 /* Something changed, must use slow path */ 737 spin_lock(&file->f_lock); 738 old = file->f_wb_err; 739 err = errseq_check_and_advance(&mapping->wb_err, 740 &file->f_wb_err); 741 trace_file_check_and_advance_wb_err(file, old); 742 spin_unlock(&file->f_lock); 743 } 744 745 /* 746 * We're mostly using this function as a drop in replacement for 747 * filemap_check_errors. Clear AS_EIO/AS_ENOSPC to emulate the effect 748 * that the legacy code would have had on these flags. 749 */ 750 clear_bit(AS_EIO, &mapping->flags); 751 clear_bit(AS_ENOSPC, &mapping->flags); 752 return err; 753 } 754 EXPORT_SYMBOL(file_check_and_advance_wb_err); 755 756 /** 757 * file_write_and_wait_range - write out & wait on a file range 758 * @file: file pointing to address_space with pages 759 * @lstart: offset in bytes where the range starts 760 * @lend: offset in bytes where the range ends (inclusive) 761 * 762 * Write out and wait upon file offsets lstart->lend, inclusive. 763 * 764 * Note that @lend is inclusive (describes the last byte to be written) so 765 * that this function can be used to write to the very end-of-file (end = -1). 766 * 767 * After writing out and waiting on the data, we check and advance the 768 * f_wb_err cursor to the latest value, and return any errors detected there. 769 * 770 * Return: %0 on success, negative error code otherwise. 771 */ 772 int file_write_and_wait_range(struct file *file, loff_t lstart, loff_t lend) 773 { 774 int err = 0, err2; 775 struct address_space *mapping = file->f_mapping; 776 777 if (lend < lstart) 778 return 0; 779 780 if (mapping_needs_writeback(mapping)) { 781 err = __filemap_fdatawrite_range(mapping, lstart, lend, 782 WB_SYNC_ALL); 783 /* See comment of filemap_write_and_wait() */ 784 if (err != -EIO) 785 __filemap_fdatawait_range(mapping, lstart, lend); 786 } 787 err2 = file_check_and_advance_wb_err(file); 788 if (!err) 789 err = err2; 790 return err; 791 } 792 EXPORT_SYMBOL(file_write_and_wait_range); 793 794 /** 795 * replace_page_cache_folio - replace a pagecache folio with a new one 796 * @old: folio to be replaced 797 * @new: folio to replace with 798 * 799 * This function replaces a folio in the pagecache with a new one. On 800 * success it acquires the pagecache reference for the new folio and 801 * drops it for the old folio. Both the old and new folios must be 802 * locked. This function does not add the new folio to the LRU, the 803 * caller must do that. 804 * 805 * The remove + add is atomic. This function cannot fail. 806 */ 807 void replace_page_cache_folio(struct folio *old, struct folio *new) 808 { 809 struct address_space *mapping = old->mapping; 810 void (*free_folio)(struct folio *) = mapping->a_ops->free_folio; 811 pgoff_t offset = old->index; 812 XA_STATE(xas, &mapping->i_pages, offset); 813 814 VM_BUG_ON_FOLIO(!folio_test_locked(old), old); 815 VM_BUG_ON_FOLIO(!folio_test_locked(new), new); 816 VM_BUG_ON_FOLIO(new->mapping, new); 817 818 folio_get(new); 819 new->mapping = mapping; 820 new->index = offset; 821 822 mem_cgroup_migrate(old, new); 823 824 xas_lock_irq(&xas); 825 xas_store(&xas, new); 826 827 old->mapping = NULL; 828 /* hugetlb pages do not participate in page cache accounting. */ 829 if (!folio_test_hugetlb(old)) 830 __lruvec_stat_sub_folio(old, NR_FILE_PAGES); 831 if (!folio_test_hugetlb(new)) 832 __lruvec_stat_add_folio(new, NR_FILE_PAGES); 833 if (folio_test_swapbacked(old)) 834 __lruvec_stat_sub_folio(old, NR_SHMEM); 835 if (folio_test_swapbacked(new)) 836 __lruvec_stat_add_folio(new, NR_SHMEM); 837 xas_unlock_irq(&xas); 838 if (free_folio) 839 free_folio(old); 840 folio_put(old); 841 } 842 EXPORT_SYMBOL_GPL(replace_page_cache_folio); 843 844 noinline int __filemap_add_folio(struct address_space *mapping, 845 struct folio *folio, pgoff_t index, gfp_t gfp, void **shadowp) 846 { 847 XA_STATE(xas, &mapping->i_pages, index); 848 int huge = folio_test_hugetlb(folio); 849 bool charged = false; 850 long nr = 1; 851 852 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 853 VM_BUG_ON_FOLIO(folio_test_swapbacked(folio), folio); 854 mapping_set_update(&xas, mapping); 855 856 if (!huge) { 857 int error = mem_cgroup_charge(folio, NULL, gfp); 858 VM_BUG_ON_FOLIO(index & (folio_nr_pages(folio) - 1), folio); 859 if (error) 860 return error; 861 charged = true; 862 xas_set_order(&xas, index, folio_order(folio)); 863 nr = folio_nr_pages(folio); 864 } 865 866 gfp &= GFP_RECLAIM_MASK; 867 folio_ref_add(folio, nr); 868 folio->mapping = mapping; 869 folio->index = xas.xa_index; 870 871 do { 872 unsigned int order = xa_get_order(xas.xa, xas.xa_index); 873 void *entry, *old = NULL; 874 875 if (order > folio_order(folio)) 876 xas_split_alloc(&xas, xa_load(xas.xa, xas.xa_index), 877 order, gfp); 878 xas_lock_irq(&xas); 879 xas_for_each_conflict(&xas, entry) { 880 old = entry; 881 if (!xa_is_value(entry)) { 882 xas_set_err(&xas, -EEXIST); 883 goto unlock; 884 } 885 } 886 887 if (old) { 888 if (shadowp) 889 *shadowp = old; 890 /* entry may have been split before we acquired lock */ 891 order = xa_get_order(xas.xa, xas.xa_index); 892 if (order > folio_order(folio)) { 893 /* How to handle large swap entries? */ 894 BUG_ON(shmem_mapping(mapping)); 895 xas_split(&xas, old, order); 896 xas_reset(&xas); 897 } 898 } 899 900 xas_store(&xas, folio); 901 if (xas_error(&xas)) 902 goto unlock; 903 904 mapping->nrpages += nr; 905 906 /* hugetlb pages do not participate in page cache accounting */ 907 if (!huge) { 908 __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, nr); 909 if (folio_test_pmd_mappable(folio)) 910 __lruvec_stat_mod_folio(folio, 911 NR_FILE_THPS, nr); 912 } 913 unlock: 914 xas_unlock_irq(&xas); 915 } while (xas_nomem(&xas, gfp)); 916 917 if (xas_error(&xas)) 918 goto error; 919 920 trace_mm_filemap_add_to_page_cache(folio); 921 return 0; 922 error: 923 if (charged) 924 mem_cgroup_uncharge(folio); 925 folio->mapping = NULL; 926 /* Leave page->index set: truncation relies upon it */ 927 folio_put_refs(folio, nr); 928 return xas_error(&xas); 929 } 930 ALLOW_ERROR_INJECTION(__filemap_add_folio, ERRNO); 931 932 int filemap_add_folio(struct address_space *mapping, struct folio *folio, 933 pgoff_t index, gfp_t gfp) 934 { 935 void *shadow = NULL; 936 int ret; 937 938 __folio_set_locked(folio); 939 ret = __filemap_add_folio(mapping, folio, index, gfp, &shadow); 940 if (unlikely(ret)) 941 __folio_clear_locked(folio); 942 else { 943 /* 944 * The folio might have been evicted from cache only 945 * recently, in which case it should be activated like 946 * any other repeatedly accessed folio. 947 * The exception is folios getting rewritten; evicting other 948 * data from the working set, only to cache data that will 949 * get overwritten with something else, is a waste of memory. 950 */ 951 WARN_ON_ONCE(folio_test_active(folio)); 952 if (!(gfp & __GFP_WRITE) && shadow) 953 workingset_refault(folio, shadow); 954 folio_add_lru(folio); 955 } 956 return ret; 957 } 958 EXPORT_SYMBOL_GPL(filemap_add_folio); 959 960 #ifdef CONFIG_NUMA 961 struct folio *filemap_alloc_folio(gfp_t gfp, unsigned int order) 962 { 963 int n; 964 struct folio *folio; 965 966 if (cpuset_do_page_mem_spread()) { 967 unsigned int cpuset_mems_cookie; 968 do { 969 cpuset_mems_cookie = read_mems_allowed_begin(); 970 n = cpuset_mem_spread_node(); 971 folio = __folio_alloc_node(gfp, order, n); 972 } while (!folio && read_mems_allowed_retry(cpuset_mems_cookie)); 973 974 return folio; 975 } 976 return folio_alloc(gfp, order); 977 } 978 EXPORT_SYMBOL(filemap_alloc_folio); 979 #endif 980 981 /* 982 * filemap_invalidate_lock_two - lock invalidate_lock for two mappings 983 * 984 * Lock exclusively invalidate_lock of any passed mapping that is not NULL. 985 * 986 * @mapping1: the first mapping to lock 987 * @mapping2: the second mapping to lock 988 */ 989 void filemap_invalidate_lock_two(struct address_space *mapping1, 990 struct address_space *mapping2) 991 { 992 if (mapping1 > mapping2) 993 swap(mapping1, mapping2); 994 if (mapping1) 995 down_write(&mapping1->invalidate_lock); 996 if (mapping2 && mapping1 != mapping2) 997 down_write_nested(&mapping2->invalidate_lock, 1); 998 } 999 EXPORT_SYMBOL(filemap_invalidate_lock_two); 1000 1001 /* 1002 * filemap_invalidate_unlock_two - unlock invalidate_lock for two mappings 1003 * 1004 * Unlock exclusive invalidate_lock of any passed mapping that is not NULL. 1005 * 1006 * @mapping1: the first mapping to unlock 1007 * @mapping2: the second mapping to unlock 1008 */ 1009 void filemap_invalidate_unlock_two(struct address_space *mapping1, 1010 struct address_space *mapping2) 1011 { 1012 if (mapping1) 1013 up_write(&mapping1->invalidate_lock); 1014 if (mapping2 && mapping1 != mapping2) 1015 up_write(&mapping2->invalidate_lock); 1016 } 1017 EXPORT_SYMBOL(filemap_invalidate_unlock_two); 1018 1019 /* 1020 * In order to wait for pages to become available there must be 1021 * waitqueues associated with pages. By using a hash table of 1022 * waitqueues where the bucket discipline is to maintain all 1023 * waiters on the same queue and wake all when any of the pages 1024 * become available, and for the woken contexts to check to be 1025 * sure the appropriate page became available, this saves space 1026 * at a cost of "thundering herd" phenomena during rare hash 1027 * collisions. 1028 */ 1029 #define PAGE_WAIT_TABLE_BITS 8 1030 #define PAGE_WAIT_TABLE_SIZE (1 << PAGE_WAIT_TABLE_BITS) 1031 static wait_queue_head_t folio_wait_table[PAGE_WAIT_TABLE_SIZE] __cacheline_aligned; 1032 1033 static wait_queue_head_t *folio_waitqueue(struct folio *folio) 1034 { 1035 return &folio_wait_table[hash_ptr(folio, PAGE_WAIT_TABLE_BITS)]; 1036 } 1037 1038 void __init pagecache_init(void) 1039 { 1040 int i; 1041 1042 for (i = 0; i < PAGE_WAIT_TABLE_SIZE; i++) 1043 init_waitqueue_head(&folio_wait_table[i]); 1044 1045 page_writeback_init(); 1046 } 1047 1048 /* 1049 * The page wait code treats the "wait->flags" somewhat unusually, because 1050 * we have multiple different kinds of waits, not just the usual "exclusive" 1051 * one. 1052 * 1053 * We have: 1054 * 1055 * (a) no special bits set: 1056 * 1057 * We're just waiting for the bit to be released, and when a waker 1058 * calls the wakeup function, we set WQ_FLAG_WOKEN and wake it up, 1059 * and remove it from the wait queue. 1060 * 1061 * Simple and straightforward. 1062 * 1063 * (b) WQ_FLAG_EXCLUSIVE: 1064 * 1065 * The waiter is waiting to get the lock, and only one waiter should 1066 * be woken up to avoid any thundering herd behavior. We'll set the 1067 * WQ_FLAG_WOKEN bit, wake it up, and remove it from the wait queue. 1068 * 1069 * This is the traditional exclusive wait. 1070 * 1071 * (c) WQ_FLAG_EXCLUSIVE | WQ_FLAG_CUSTOM: 1072 * 1073 * The waiter is waiting to get the bit, and additionally wants the 1074 * lock to be transferred to it for fair lock behavior. If the lock 1075 * cannot be taken, we stop walking the wait queue without waking 1076 * the waiter. 1077 * 1078 * This is the "fair lock handoff" case, and in addition to setting 1079 * WQ_FLAG_WOKEN, we set WQ_FLAG_DONE to let the waiter easily see 1080 * that it now has the lock. 1081 */ 1082 static int wake_page_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *arg) 1083 { 1084 unsigned int flags; 1085 struct wait_page_key *key = arg; 1086 struct wait_page_queue *wait_page 1087 = container_of(wait, struct wait_page_queue, wait); 1088 1089 if (!wake_page_match(wait_page, key)) 1090 return 0; 1091 1092 /* 1093 * If it's a lock handoff wait, we get the bit for it, and 1094 * stop walking (and do not wake it up) if we can't. 1095 */ 1096 flags = wait->flags; 1097 if (flags & WQ_FLAG_EXCLUSIVE) { 1098 if (test_bit(key->bit_nr, &key->folio->flags)) 1099 return -1; 1100 if (flags & WQ_FLAG_CUSTOM) { 1101 if (test_and_set_bit(key->bit_nr, &key->folio->flags)) 1102 return -1; 1103 flags |= WQ_FLAG_DONE; 1104 } 1105 } 1106 1107 /* 1108 * We are holding the wait-queue lock, but the waiter that 1109 * is waiting for this will be checking the flags without 1110 * any locking. 1111 * 1112 * So update the flags atomically, and wake up the waiter 1113 * afterwards to avoid any races. This store-release pairs 1114 * with the load-acquire in folio_wait_bit_common(). 1115 */ 1116 smp_store_release(&wait->flags, flags | WQ_FLAG_WOKEN); 1117 wake_up_state(wait->private, mode); 1118 1119 /* 1120 * Ok, we have successfully done what we're waiting for, 1121 * and we can unconditionally remove the wait entry. 1122 * 1123 * Note that this pairs with the "finish_wait()" in the 1124 * waiter, and has to be the absolute last thing we do. 1125 * After this list_del_init(&wait->entry) the wait entry 1126 * might be de-allocated and the process might even have 1127 * exited. 1128 */ 1129 list_del_init_careful(&wait->entry); 1130 return (flags & WQ_FLAG_EXCLUSIVE) != 0; 1131 } 1132 1133 static void folio_wake_bit(struct folio *folio, int bit_nr) 1134 { 1135 wait_queue_head_t *q = folio_waitqueue(folio); 1136 struct wait_page_key key; 1137 unsigned long flags; 1138 wait_queue_entry_t bookmark; 1139 1140 key.folio = folio; 1141 key.bit_nr = bit_nr; 1142 key.page_match = 0; 1143 1144 bookmark.flags = 0; 1145 bookmark.private = NULL; 1146 bookmark.func = NULL; 1147 INIT_LIST_HEAD(&bookmark.entry); 1148 1149 spin_lock_irqsave(&q->lock, flags); 1150 __wake_up_locked_key_bookmark(q, TASK_NORMAL, &key, &bookmark); 1151 1152 while (bookmark.flags & WQ_FLAG_BOOKMARK) { 1153 /* 1154 * Take a breather from holding the lock, 1155 * allow pages that finish wake up asynchronously 1156 * to acquire the lock and remove themselves 1157 * from wait queue 1158 */ 1159 spin_unlock_irqrestore(&q->lock, flags); 1160 cpu_relax(); 1161 spin_lock_irqsave(&q->lock, flags); 1162 __wake_up_locked_key_bookmark(q, TASK_NORMAL, &key, &bookmark); 1163 } 1164 1165 /* 1166 * It's possible to miss clearing waiters here, when we woke our page 1167 * waiters, but the hashed waitqueue has waiters for other pages on it. 1168 * That's okay, it's a rare case. The next waker will clear it. 1169 * 1170 * Note that, depending on the page pool (buddy, hugetlb, ZONE_DEVICE, 1171 * other), the flag may be cleared in the course of freeing the page; 1172 * but that is not required for correctness. 1173 */ 1174 if (!waitqueue_active(q) || !key.page_match) 1175 folio_clear_waiters(folio); 1176 1177 spin_unlock_irqrestore(&q->lock, flags); 1178 } 1179 1180 static void folio_wake(struct folio *folio, int bit) 1181 { 1182 if (!folio_test_waiters(folio)) 1183 return; 1184 folio_wake_bit(folio, bit); 1185 } 1186 1187 /* 1188 * A choice of three behaviors for folio_wait_bit_common(): 1189 */ 1190 enum behavior { 1191 EXCLUSIVE, /* Hold ref to page and take the bit when woken, like 1192 * __folio_lock() waiting on then setting PG_locked. 1193 */ 1194 SHARED, /* Hold ref to page and check the bit when woken, like 1195 * folio_wait_writeback() waiting on PG_writeback. 1196 */ 1197 DROP, /* Drop ref to page before wait, no check when woken, 1198 * like folio_put_wait_locked() on PG_locked. 1199 */ 1200 }; 1201 1202 /* 1203 * Attempt to check (or get) the folio flag, and mark us done 1204 * if successful. 1205 */ 1206 static inline bool folio_trylock_flag(struct folio *folio, int bit_nr, 1207 struct wait_queue_entry *wait) 1208 { 1209 if (wait->flags & WQ_FLAG_EXCLUSIVE) { 1210 if (test_and_set_bit(bit_nr, &folio->flags)) 1211 return false; 1212 } else if (test_bit(bit_nr, &folio->flags)) 1213 return false; 1214 1215 wait->flags |= WQ_FLAG_WOKEN | WQ_FLAG_DONE; 1216 return true; 1217 } 1218 1219 /* How many times do we accept lock stealing from under a waiter? */ 1220 int sysctl_page_lock_unfairness = 5; 1221 1222 static inline int folio_wait_bit_common(struct folio *folio, int bit_nr, 1223 int state, enum behavior behavior) 1224 { 1225 wait_queue_head_t *q = folio_waitqueue(folio); 1226 int unfairness = sysctl_page_lock_unfairness; 1227 struct wait_page_queue wait_page; 1228 wait_queue_entry_t *wait = &wait_page.wait; 1229 bool thrashing = false; 1230 unsigned long pflags; 1231 bool in_thrashing; 1232 1233 if (bit_nr == PG_locked && 1234 !folio_test_uptodate(folio) && folio_test_workingset(folio)) { 1235 delayacct_thrashing_start(&in_thrashing); 1236 psi_memstall_enter(&pflags); 1237 thrashing = true; 1238 } 1239 1240 init_wait(wait); 1241 wait->func = wake_page_function; 1242 wait_page.folio = folio; 1243 wait_page.bit_nr = bit_nr; 1244 1245 repeat: 1246 wait->flags = 0; 1247 if (behavior == EXCLUSIVE) { 1248 wait->flags = WQ_FLAG_EXCLUSIVE; 1249 if (--unfairness < 0) 1250 wait->flags |= WQ_FLAG_CUSTOM; 1251 } 1252 1253 /* 1254 * Do one last check whether we can get the 1255 * page bit synchronously. 1256 * 1257 * Do the folio_set_waiters() marking before that 1258 * to let any waker we _just_ missed know they 1259 * need to wake us up (otherwise they'll never 1260 * even go to the slow case that looks at the 1261 * page queue), and add ourselves to the wait 1262 * queue if we need to sleep. 1263 * 1264 * This part needs to be done under the queue 1265 * lock to avoid races. 1266 */ 1267 spin_lock_irq(&q->lock); 1268 folio_set_waiters(folio); 1269 if (!folio_trylock_flag(folio, bit_nr, wait)) 1270 __add_wait_queue_entry_tail(q, wait); 1271 spin_unlock_irq(&q->lock); 1272 1273 /* 1274 * From now on, all the logic will be based on 1275 * the WQ_FLAG_WOKEN and WQ_FLAG_DONE flag, to 1276 * see whether the page bit testing has already 1277 * been done by the wake function. 1278 * 1279 * We can drop our reference to the folio. 1280 */ 1281 if (behavior == DROP) 1282 folio_put(folio); 1283 1284 /* 1285 * Note that until the "finish_wait()", or until 1286 * we see the WQ_FLAG_WOKEN flag, we need to 1287 * be very careful with the 'wait->flags', because 1288 * we may race with a waker that sets them. 1289 */ 1290 for (;;) { 1291 unsigned int flags; 1292 1293 set_current_state(state); 1294 1295 /* Loop until we've been woken or interrupted */ 1296 flags = smp_load_acquire(&wait->flags); 1297 if (!(flags & WQ_FLAG_WOKEN)) { 1298 if (signal_pending_state(state, current)) 1299 break; 1300 1301 io_schedule(); 1302 continue; 1303 } 1304 1305 /* If we were non-exclusive, we're done */ 1306 if (behavior != EXCLUSIVE) 1307 break; 1308 1309 /* If the waker got the lock for us, we're done */ 1310 if (flags & WQ_FLAG_DONE) 1311 break; 1312 1313 /* 1314 * Otherwise, if we're getting the lock, we need to 1315 * try to get it ourselves. 1316 * 1317 * And if that fails, we'll have to retry this all. 1318 */ 1319 if (unlikely(test_and_set_bit(bit_nr, folio_flags(folio, 0)))) 1320 goto repeat; 1321 1322 wait->flags |= WQ_FLAG_DONE; 1323 break; 1324 } 1325 1326 /* 1327 * If a signal happened, this 'finish_wait()' may remove the last 1328 * waiter from the wait-queues, but the folio waiters bit will remain 1329 * set. That's ok. The next wakeup will take care of it, and trying 1330 * to do it here would be difficult and prone to races. 1331 */ 1332 finish_wait(q, wait); 1333 1334 if (thrashing) { 1335 delayacct_thrashing_end(&in_thrashing); 1336 psi_memstall_leave(&pflags); 1337 } 1338 1339 /* 1340 * NOTE! The wait->flags weren't stable until we've done the 1341 * 'finish_wait()', and we could have exited the loop above due 1342 * to a signal, and had a wakeup event happen after the signal 1343 * test but before the 'finish_wait()'. 1344 * 1345 * So only after the finish_wait() can we reliably determine 1346 * if we got woken up or not, so we can now figure out the final 1347 * return value based on that state without races. 1348 * 1349 * Also note that WQ_FLAG_WOKEN is sufficient for a non-exclusive 1350 * waiter, but an exclusive one requires WQ_FLAG_DONE. 1351 */ 1352 if (behavior == EXCLUSIVE) 1353 return wait->flags & WQ_FLAG_DONE ? 0 : -EINTR; 1354 1355 return wait->flags & WQ_FLAG_WOKEN ? 0 : -EINTR; 1356 } 1357 1358 #ifdef CONFIG_MIGRATION 1359 /** 1360 * migration_entry_wait_on_locked - Wait for a migration entry to be removed 1361 * @entry: migration swap entry. 1362 * @ptl: already locked ptl. This function will drop the lock. 1363 * 1364 * Wait for a migration entry referencing the given page to be removed. This is 1365 * equivalent to put_and_wait_on_page_locked(page, TASK_UNINTERRUPTIBLE) except 1366 * this can be called without taking a reference on the page. Instead this 1367 * should be called while holding the ptl for the migration entry referencing 1368 * the page. 1369 * 1370 * Returns after unlocking the ptl. 1371 * 1372 * This follows the same logic as folio_wait_bit_common() so see the comments 1373 * there. 1374 */ 1375 void migration_entry_wait_on_locked(swp_entry_t entry, spinlock_t *ptl) 1376 __releases(ptl) 1377 { 1378 struct wait_page_queue wait_page; 1379 wait_queue_entry_t *wait = &wait_page.wait; 1380 bool thrashing = false; 1381 unsigned long pflags; 1382 bool in_thrashing; 1383 wait_queue_head_t *q; 1384 struct folio *folio = page_folio(pfn_swap_entry_to_page(entry)); 1385 1386 q = folio_waitqueue(folio); 1387 if (!folio_test_uptodate(folio) && folio_test_workingset(folio)) { 1388 delayacct_thrashing_start(&in_thrashing); 1389 psi_memstall_enter(&pflags); 1390 thrashing = true; 1391 } 1392 1393 init_wait(wait); 1394 wait->func = wake_page_function; 1395 wait_page.folio = folio; 1396 wait_page.bit_nr = PG_locked; 1397 wait->flags = 0; 1398 1399 spin_lock_irq(&q->lock); 1400 folio_set_waiters(folio); 1401 if (!folio_trylock_flag(folio, PG_locked, wait)) 1402 __add_wait_queue_entry_tail(q, wait); 1403 spin_unlock_irq(&q->lock); 1404 1405 /* 1406 * If a migration entry exists for the page the migration path must hold 1407 * a valid reference to the page, and it must take the ptl to remove the 1408 * migration entry. So the page is valid until the ptl is dropped. 1409 */ 1410 spin_unlock(ptl); 1411 1412 for (;;) { 1413 unsigned int flags; 1414 1415 set_current_state(TASK_UNINTERRUPTIBLE); 1416 1417 /* Loop until we've been woken or interrupted */ 1418 flags = smp_load_acquire(&wait->flags); 1419 if (!(flags & WQ_FLAG_WOKEN)) { 1420 if (signal_pending_state(TASK_UNINTERRUPTIBLE, current)) 1421 break; 1422 1423 io_schedule(); 1424 continue; 1425 } 1426 break; 1427 } 1428 1429 finish_wait(q, wait); 1430 1431 if (thrashing) { 1432 delayacct_thrashing_end(&in_thrashing); 1433 psi_memstall_leave(&pflags); 1434 } 1435 } 1436 #endif 1437 1438 void folio_wait_bit(struct folio *folio, int bit_nr) 1439 { 1440 folio_wait_bit_common(folio, bit_nr, TASK_UNINTERRUPTIBLE, SHARED); 1441 } 1442 EXPORT_SYMBOL(folio_wait_bit); 1443 1444 int folio_wait_bit_killable(struct folio *folio, int bit_nr) 1445 { 1446 return folio_wait_bit_common(folio, bit_nr, TASK_KILLABLE, SHARED); 1447 } 1448 EXPORT_SYMBOL(folio_wait_bit_killable); 1449 1450 /** 1451 * folio_put_wait_locked - Drop a reference and wait for it to be unlocked 1452 * @folio: The folio to wait for. 1453 * @state: The sleep state (TASK_KILLABLE, TASK_UNINTERRUPTIBLE, etc). 1454 * 1455 * The caller should hold a reference on @folio. They expect the page to 1456 * become unlocked relatively soon, but do not wish to hold up migration 1457 * (for example) by holding the reference while waiting for the folio to 1458 * come unlocked. After this function returns, the caller should not 1459 * dereference @folio. 1460 * 1461 * Return: 0 if the folio was unlocked or -EINTR if interrupted by a signal. 1462 */ 1463 static int folio_put_wait_locked(struct folio *folio, int state) 1464 { 1465 return folio_wait_bit_common(folio, PG_locked, state, DROP); 1466 } 1467 1468 /** 1469 * folio_add_wait_queue - Add an arbitrary waiter to a folio's wait queue 1470 * @folio: Folio defining the wait queue of interest 1471 * @waiter: Waiter to add to the queue 1472 * 1473 * Add an arbitrary @waiter to the wait queue for the nominated @folio. 1474 */ 1475 void folio_add_wait_queue(struct folio *folio, wait_queue_entry_t *waiter) 1476 { 1477 wait_queue_head_t *q = folio_waitqueue(folio); 1478 unsigned long flags; 1479 1480 spin_lock_irqsave(&q->lock, flags); 1481 __add_wait_queue_entry_tail(q, waiter); 1482 folio_set_waiters(folio); 1483 spin_unlock_irqrestore(&q->lock, flags); 1484 } 1485 EXPORT_SYMBOL_GPL(folio_add_wait_queue); 1486 1487 #ifndef clear_bit_unlock_is_negative_byte 1488 1489 /* 1490 * PG_waiters is the high bit in the same byte as PG_lock. 1491 * 1492 * On x86 (and on many other architectures), we can clear PG_lock and 1493 * test the sign bit at the same time. But if the architecture does 1494 * not support that special operation, we just do this all by hand 1495 * instead. 1496 * 1497 * The read of PG_waiters has to be after (or concurrently with) PG_locked 1498 * being cleared, but a memory barrier should be unnecessary since it is 1499 * in the same byte as PG_locked. 1500 */ 1501 static inline bool clear_bit_unlock_is_negative_byte(long nr, volatile void *mem) 1502 { 1503 clear_bit_unlock(nr, mem); 1504 /* smp_mb__after_atomic(); */ 1505 return test_bit(PG_waiters, mem); 1506 } 1507 1508 #endif 1509 1510 /** 1511 * folio_unlock - Unlock a locked folio. 1512 * @folio: The folio. 1513 * 1514 * Unlocks the folio and wakes up any thread sleeping on the page lock. 1515 * 1516 * Context: May be called from interrupt or process context. May not be 1517 * called from NMI context. 1518 */ 1519 void folio_unlock(struct folio *folio) 1520 { 1521 /* Bit 7 allows x86 to check the byte's sign bit */ 1522 BUILD_BUG_ON(PG_waiters != 7); 1523 BUILD_BUG_ON(PG_locked > 7); 1524 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 1525 if (clear_bit_unlock_is_negative_byte(PG_locked, folio_flags(folio, 0))) 1526 folio_wake_bit(folio, PG_locked); 1527 } 1528 EXPORT_SYMBOL(folio_unlock); 1529 1530 /** 1531 * folio_end_private_2 - Clear PG_private_2 and wake any waiters. 1532 * @folio: The folio. 1533 * 1534 * Clear the PG_private_2 bit on a folio and wake up any sleepers waiting for 1535 * it. The folio reference held for PG_private_2 being set is released. 1536 * 1537 * This is, for example, used when a netfs folio is being written to a local 1538 * disk cache, thereby allowing writes to the cache for the same folio to be 1539 * serialised. 1540 */ 1541 void folio_end_private_2(struct folio *folio) 1542 { 1543 VM_BUG_ON_FOLIO(!folio_test_private_2(folio), folio); 1544 clear_bit_unlock(PG_private_2, folio_flags(folio, 0)); 1545 folio_wake_bit(folio, PG_private_2); 1546 folio_put(folio); 1547 } 1548 EXPORT_SYMBOL(folio_end_private_2); 1549 1550 /** 1551 * folio_wait_private_2 - Wait for PG_private_2 to be cleared on a folio. 1552 * @folio: The folio to wait on. 1553 * 1554 * Wait for PG_private_2 (aka PG_fscache) to be cleared on a folio. 1555 */ 1556 void folio_wait_private_2(struct folio *folio) 1557 { 1558 while (folio_test_private_2(folio)) 1559 folio_wait_bit(folio, PG_private_2); 1560 } 1561 EXPORT_SYMBOL(folio_wait_private_2); 1562 1563 /** 1564 * folio_wait_private_2_killable - Wait for PG_private_2 to be cleared on a folio. 1565 * @folio: The folio to wait on. 1566 * 1567 * Wait for PG_private_2 (aka PG_fscache) to be cleared on a folio or until a 1568 * fatal signal is received by the calling task. 1569 * 1570 * Return: 1571 * - 0 if successful. 1572 * - -EINTR if a fatal signal was encountered. 1573 */ 1574 int folio_wait_private_2_killable(struct folio *folio) 1575 { 1576 int ret = 0; 1577 1578 while (folio_test_private_2(folio)) { 1579 ret = folio_wait_bit_killable(folio, PG_private_2); 1580 if (ret < 0) 1581 break; 1582 } 1583 1584 return ret; 1585 } 1586 EXPORT_SYMBOL(folio_wait_private_2_killable); 1587 1588 /** 1589 * folio_end_writeback - End writeback against a folio. 1590 * @folio: The folio. 1591 */ 1592 void folio_end_writeback(struct folio *folio) 1593 { 1594 /* 1595 * folio_test_clear_reclaim() could be used here but it is an 1596 * atomic operation and overkill in this particular case. Failing 1597 * to shuffle a folio marked for immediate reclaim is too mild 1598 * a gain to justify taking an atomic operation penalty at the 1599 * end of every folio writeback. 1600 */ 1601 if (folio_test_reclaim(folio)) { 1602 folio_clear_reclaim(folio); 1603 folio_rotate_reclaimable(folio); 1604 } 1605 1606 /* 1607 * Writeback does not hold a folio reference of its own, relying 1608 * on truncation to wait for the clearing of PG_writeback. 1609 * But here we must make sure that the folio is not freed and 1610 * reused before the folio_wake(). 1611 */ 1612 folio_get(folio); 1613 if (!__folio_end_writeback(folio)) 1614 BUG(); 1615 1616 smp_mb__after_atomic(); 1617 folio_wake(folio, PG_writeback); 1618 acct_reclaim_writeback(folio); 1619 folio_put(folio); 1620 } 1621 EXPORT_SYMBOL(folio_end_writeback); 1622 1623 /** 1624 * __folio_lock - Get a lock on the folio, assuming we need to sleep to get it. 1625 * @folio: The folio to lock 1626 */ 1627 void __folio_lock(struct folio *folio) 1628 { 1629 folio_wait_bit_common(folio, PG_locked, TASK_UNINTERRUPTIBLE, 1630 EXCLUSIVE); 1631 } 1632 EXPORT_SYMBOL(__folio_lock); 1633 1634 int __folio_lock_killable(struct folio *folio) 1635 { 1636 return folio_wait_bit_common(folio, PG_locked, TASK_KILLABLE, 1637 EXCLUSIVE); 1638 } 1639 EXPORT_SYMBOL_GPL(__folio_lock_killable); 1640 1641 static int __folio_lock_async(struct folio *folio, struct wait_page_queue *wait) 1642 { 1643 struct wait_queue_head *q = folio_waitqueue(folio); 1644 int ret = 0; 1645 1646 wait->folio = folio; 1647 wait->bit_nr = PG_locked; 1648 1649 spin_lock_irq(&q->lock); 1650 __add_wait_queue_entry_tail(q, &wait->wait); 1651 folio_set_waiters(folio); 1652 ret = !folio_trylock(folio); 1653 /* 1654 * If we were successful now, we know we're still on the 1655 * waitqueue as we're still under the lock. This means it's 1656 * safe to remove and return success, we know the callback 1657 * isn't going to trigger. 1658 */ 1659 if (!ret) 1660 __remove_wait_queue(q, &wait->wait); 1661 else 1662 ret = -EIOCBQUEUED; 1663 spin_unlock_irq(&q->lock); 1664 return ret; 1665 } 1666 1667 /* 1668 * Return values: 1669 * 0 - folio is locked. 1670 * non-zero - folio is not locked. 1671 * mmap_lock or per-VMA lock has been released (mmap_read_unlock() or 1672 * vma_end_read()), unless flags had both FAULT_FLAG_ALLOW_RETRY and 1673 * FAULT_FLAG_RETRY_NOWAIT set, in which case the lock is still held. 1674 * 1675 * If neither ALLOW_RETRY nor KILLABLE are set, will always return 0 1676 * with the folio locked and the mmap_lock/per-VMA lock is left unperturbed. 1677 */ 1678 vm_fault_t __folio_lock_or_retry(struct folio *folio, struct vm_fault *vmf) 1679 { 1680 unsigned int flags = vmf->flags; 1681 1682 if (fault_flag_allow_retry_first(flags)) { 1683 /* 1684 * CAUTION! In this case, mmap_lock/per-VMA lock is not 1685 * released even though returning VM_FAULT_RETRY. 1686 */ 1687 if (flags & FAULT_FLAG_RETRY_NOWAIT) 1688 return VM_FAULT_RETRY; 1689 1690 release_fault_lock(vmf); 1691 if (flags & FAULT_FLAG_KILLABLE) 1692 folio_wait_locked_killable(folio); 1693 else 1694 folio_wait_locked(folio); 1695 return VM_FAULT_RETRY; 1696 } 1697 if (flags & FAULT_FLAG_KILLABLE) { 1698 bool ret; 1699 1700 ret = __folio_lock_killable(folio); 1701 if (ret) { 1702 release_fault_lock(vmf); 1703 return VM_FAULT_RETRY; 1704 } 1705 } else { 1706 __folio_lock(folio); 1707 } 1708 1709 return 0; 1710 } 1711 1712 /** 1713 * page_cache_next_miss() - Find the next gap in the page cache. 1714 * @mapping: Mapping. 1715 * @index: Index. 1716 * @max_scan: Maximum range to search. 1717 * 1718 * Search the range [index, min(index + max_scan - 1, ULONG_MAX)] for the 1719 * gap with the lowest index. 1720 * 1721 * This function may be called under the rcu_read_lock. However, this will 1722 * not atomically search a snapshot of the cache at a single point in time. 1723 * For example, if a gap is created at index 5, then subsequently a gap is 1724 * created at index 10, page_cache_next_miss covering both indices may 1725 * return 10 if called under the rcu_read_lock. 1726 * 1727 * Return: The index of the gap if found, otherwise an index outside the 1728 * range specified (in which case 'return - index >= max_scan' will be true). 1729 * In the rare case of index wrap-around, 0 will be returned. 1730 */ 1731 pgoff_t page_cache_next_miss(struct address_space *mapping, 1732 pgoff_t index, unsigned long max_scan) 1733 { 1734 XA_STATE(xas, &mapping->i_pages, index); 1735 1736 while (max_scan--) { 1737 void *entry = xas_next(&xas); 1738 if (!entry || xa_is_value(entry)) 1739 break; 1740 if (xas.xa_index == 0) 1741 break; 1742 } 1743 1744 return xas.xa_index; 1745 } 1746 EXPORT_SYMBOL(page_cache_next_miss); 1747 1748 /** 1749 * page_cache_prev_miss() - Find the previous gap in the page cache. 1750 * @mapping: Mapping. 1751 * @index: Index. 1752 * @max_scan: Maximum range to search. 1753 * 1754 * Search the range [max(index - max_scan + 1, 0), index] for the 1755 * gap with the highest index. 1756 * 1757 * This function may be called under the rcu_read_lock. However, this will 1758 * not atomically search a snapshot of the cache at a single point in time. 1759 * For example, if a gap is created at index 10, then subsequently a gap is 1760 * created at index 5, page_cache_prev_miss() covering both indices may 1761 * return 5 if called under the rcu_read_lock. 1762 * 1763 * Return: The index of the gap if found, otherwise an index outside the 1764 * range specified (in which case 'index - return >= max_scan' will be true). 1765 * In the rare case of wrap-around, ULONG_MAX will be returned. 1766 */ 1767 pgoff_t page_cache_prev_miss(struct address_space *mapping, 1768 pgoff_t index, unsigned long max_scan) 1769 { 1770 XA_STATE(xas, &mapping->i_pages, index); 1771 1772 while (max_scan--) { 1773 void *entry = xas_prev(&xas); 1774 if (!entry || xa_is_value(entry)) 1775 break; 1776 if (xas.xa_index == ULONG_MAX) 1777 break; 1778 } 1779 1780 return xas.xa_index; 1781 } 1782 EXPORT_SYMBOL(page_cache_prev_miss); 1783 1784 /* 1785 * Lockless page cache protocol: 1786 * On the lookup side: 1787 * 1. Load the folio from i_pages 1788 * 2. Increment the refcount if it's not zero 1789 * 3. If the folio is not found by xas_reload(), put the refcount and retry 1790 * 1791 * On the removal side: 1792 * A. Freeze the page (by zeroing the refcount if nobody else has a reference) 1793 * B. Remove the page from i_pages 1794 * C. Return the page to the page allocator 1795 * 1796 * This means that any page may have its reference count temporarily 1797 * increased by a speculative page cache (or fast GUP) lookup as it can 1798 * be allocated by another user before the RCU grace period expires. 1799 * Because the refcount temporarily acquired here may end up being the 1800 * last refcount on the page, any page allocation must be freeable by 1801 * folio_put(). 1802 */ 1803 1804 /* 1805 * filemap_get_entry - Get a page cache entry. 1806 * @mapping: the address_space to search 1807 * @index: The page cache index. 1808 * 1809 * Looks up the page cache entry at @mapping & @index. If it is a folio, 1810 * it is returned with an increased refcount. If it is a shadow entry 1811 * of a previously evicted folio, or a swap entry from shmem/tmpfs, 1812 * it is returned without further action. 1813 * 1814 * Return: The folio, swap or shadow entry, %NULL if nothing is found. 1815 */ 1816 void *filemap_get_entry(struct address_space *mapping, pgoff_t index) 1817 { 1818 XA_STATE(xas, &mapping->i_pages, index); 1819 struct folio *folio; 1820 1821 rcu_read_lock(); 1822 repeat: 1823 xas_reset(&xas); 1824 folio = xas_load(&xas); 1825 if (xas_retry(&xas, folio)) 1826 goto repeat; 1827 /* 1828 * A shadow entry of a recently evicted page, or a swap entry from 1829 * shmem/tmpfs. Return it without attempting to raise page count. 1830 */ 1831 if (!folio || xa_is_value(folio)) 1832 goto out; 1833 1834 if (!folio_try_get_rcu(folio)) 1835 goto repeat; 1836 1837 if (unlikely(folio != xas_reload(&xas))) { 1838 folio_put(folio); 1839 goto repeat; 1840 } 1841 out: 1842 rcu_read_unlock(); 1843 1844 return folio; 1845 } 1846 1847 /** 1848 * __filemap_get_folio - Find and get a reference to a folio. 1849 * @mapping: The address_space to search. 1850 * @index: The page index. 1851 * @fgp_flags: %FGP flags modify how the folio is returned. 1852 * @gfp: Memory allocation flags to use if %FGP_CREAT is specified. 1853 * 1854 * Looks up the page cache entry at @mapping & @index. 1855 * 1856 * If %FGP_LOCK or %FGP_CREAT are specified then the function may sleep even 1857 * if the %GFP flags specified for %FGP_CREAT are atomic. 1858 * 1859 * If this function returns a folio, it is returned with an increased refcount. 1860 * 1861 * Return: The found folio or an ERR_PTR() otherwise. 1862 */ 1863 struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index, 1864 fgf_t fgp_flags, gfp_t gfp) 1865 { 1866 struct folio *folio; 1867 1868 repeat: 1869 folio = filemap_get_entry(mapping, index); 1870 if (xa_is_value(folio)) 1871 folio = NULL; 1872 if (!folio) 1873 goto no_page; 1874 1875 if (fgp_flags & FGP_LOCK) { 1876 if (fgp_flags & FGP_NOWAIT) { 1877 if (!folio_trylock(folio)) { 1878 folio_put(folio); 1879 return ERR_PTR(-EAGAIN); 1880 } 1881 } else { 1882 folio_lock(folio); 1883 } 1884 1885 /* Has the page been truncated? */ 1886 if (unlikely(folio->mapping != mapping)) { 1887 folio_unlock(folio); 1888 folio_put(folio); 1889 goto repeat; 1890 } 1891 VM_BUG_ON_FOLIO(!folio_contains(folio, index), folio); 1892 } 1893 1894 if (fgp_flags & FGP_ACCESSED) 1895 folio_mark_accessed(folio); 1896 else if (fgp_flags & FGP_WRITE) { 1897 /* Clear idle flag for buffer write */ 1898 if (folio_test_idle(folio)) 1899 folio_clear_idle(folio); 1900 } 1901 1902 if (fgp_flags & FGP_STABLE) 1903 folio_wait_stable(folio); 1904 no_page: 1905 if (!folio && (fgp_flags & FGP_CREAT)) { 1906 unsigned order = FGF_GET_ORDER(fgp_flags); 1907 int err; 1908 1909 if ((fgp_flags & FGP_WRITE) && mapping_can_writeback(mapping)) 1910 gfp |= __GFP_WRITE; 1911 if (fgp_flags & FGP_NOFS) 1912 gfp &= ~__GFP_FS; 1913 if (fgp_flags & FGP_NOWAIT) { 1914 gfp &= ~GFP_KERNEL; 1915 gfp |= GFP_NOWAIT | __GFP_NOWARN; 1916 } 1917 if (WARN_ON_ONCE(!(fgp_flags & (FGP_LOCK | FGP_FOR_MMAP)))) 1918 fgp_flags |= FGP_LOCK; 1919 1920 if (!mapping_large_folio_support(mapping)) 1921 order = 0; 1922 if (order > MAX_PAGECACHE_ORDER) 1923 order = MAX_PAGECACHE_ORDER; 1924 /* If we're not aligned, allocate a smaller folio */ 1925 if (index & ((1UL << order) - 1)) 1926 order = __ffs(index); 1927 1928 do { 1929 gfp_t alloc_gfp = gfp; 1930 1931 err = -ENOMEM; 1932 if (order == 1) 1933 order = 0; 1934 if (order > 0) 1935 alloc_gfp |= __GFP_NORETRY | __GFP_NOWARN; 1936 folio = filemap_alloc_folio(alloc_gfp, order); 1937 if (!folio) 1938 continue; 1939 1940 /* Init accessed so avoid atomic mark_page_accessed later */ 1941 if (fgp_flags & FGP_ACCESSED) 1942 __folio_set_referenced(folio); 1943 1944 err = filemap_add_folio(mapping, folio, index, gfp); 1945 if (!err) 1946 break; 1947 folio_put(folio); 1948 folio = NULL; 1949 } while (order-- > 0); 1950 1951 if (err == -EEXIST) 1952 goto repeat; 1953 if (err) 1954 return ERR_PTR(err); 1955 /* 1956 * filemap_add_folio locks the page, and for mmap 1957 * we expect an unlocked page. 1958 */ 1959 if (folio && (fgp_flags & FGP_FOR_MMAP)) 1960 folio_unlock(folio); 1961 } 1962 1963 if (!folio) 1964 return ERR_PTR(-ENOENT); 1965 return folio; 1966 } 1967 EXPORT_SYMBOL(__filemap_get_folio); 1968 1969 static inline struct folio *find_get_entry(struct xa_state *xas, pgoff_t max, 1970 xa_mark_t mark) 1971 { 1972 struct folio *folio; 1973 1974 retry: 1975 if (mark == XA_PRESENT) 1976 folio = xas_find(xas, max); 1977 else 1978 folio = xas_find_marked(xas, max, mark); 1979 1980 if (xas_retry(xas, folio)) 1981 goto retry; 1982 /* 1983 * A shadow entry of a recently evicted page, a swap 1984 * entry from shmem/tmpfs or a DAX entry. Return it 1985 * without attempting to raise page count. 1986 */ 1987 if (!folio || xa_is_value(folio)) 1988 return folio; 1989 1990 if (!folio_try_get_rcu(folio)) 1991 goto reset; 1992 1993 if (unlikely(folio != xas_reload(xas))) { 1994 folio_put(folio); 1995 goto reset; 1996 } 1997 1998 return folio; 1999 reset: 2000 xas_reset(xas); 2001 goto retry; 2002 } 2003 2004 /** 2005 * find_get_entries - gang pagecache lookup 2006 * @mapping: The address_space to search 2007 * @start: The starting page cache index 2008 * @end: The final page index (inclusive). 2009 * @fbatch: Where the resulting entries are placed. 2010 * @indices: The cache indices corresponding to the entries in @entries 2011 * 2012 * find_get_entries() will search for and return a batch of entries in 2013 * the mapping. The entries are placed in @fbatch. find_get_entries() 2014 * takes a reference on any actual folios it returns. 2015 * 2016 * The entries have ascending indexes. The indices may not be consecutive 2017 * due to not-present entries or large folios. 2018 * 2019 * Any shadow entries of evicted folios, or swap entries from 2020 * shmem/tmpfs, are included in the returned array. 2021 * 2022 * Return: The number of entries which were found. 2023 */ 2024 unsigned find_get_entries(struct address_space *mapping, pgoff_t *start, 2025 pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices) 2026 { 2027 XA_STATE(xas, &mapping->i_pages, *start); 2028 struct folio *folio; 2029 2030 rcu_read_lock(); 2031 while ((folio = find_get_entry(&xas, end, XA_PRESENT)) != NULL) { 2032 indices[fbatch->nr] = xas.xa_index; 2033 if (!folio_batch_add(fbatch, folio)) 2034 break; 2035 } 2036 rcu_read_unlock(); 2037 2038 if (folio_batch_count(fbatch)) { 2039 unsigned long nr = 1; 2040 int idx = folio_batch_count(fbatch) - 1; 2041 2042 folio = fbatch->folios[idx]; 2043 if (!xa_is_value(folio) && !folio_test_hugetlb(folio)) 2044 nr = folio_nr_pages(folio); 2045 *start = indices[idx] + nr; 2046 } 2047 return folio_batch_count(fbatch); 2048 } 2049 2050 /** 2051 * find_lock_entries - Find a batch of pagecache entries. 2052 * @mapping: The address_space to search. 2053 * @start: The starting page cache index. 2054 * @end: The final page index (inclusive). 2055 * @fbatch: Where the resulting entries are placed. 2056 * @indices: The cache indices of the entries in @fbatch. 2057 * 2058 * find_lock_entries() will return a batch of entries from @mapping. 2059 * Swap, shadow and DAX entries are included. Folios are returned 2060 * locked and with an incremented refcount. Folios which are locked 2061 * by somebody else or under writeback are skipped. Folios which are 2062 * partially outside the range are not returned. 2063 * 2064 * The entries have ascending indexes. The indices may not be consecutive 2065 * due to not-present entries, large folios, folios which could not be 2066 * locked or folios under writeback. 2067 * 2068 * Return: The number of entries which were found. 2069 */ 2070 unsigned find_lock_entries(struct address_space *mapping, pgoff_t *start, 2071 pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices) 2072 { 2073 XA_STATE(xas, &mapping->i_pages, *start); 2074 struct folio *folio; 2075 2076 rcu_read_lock(); 2077 while ((folio = find_get_entry(&xas, end, XA_PRESENT))) { 2078 if (!xa_is_value(folio)) { 2079 if (folio->index < *start) 2080 goto put; 2081 if (folio_next_index(folio) - 1 > end) 2082 goto put; 2083 if (!folio_trylock(folio)) 2084 goto put; 2085 if (folio->mapping != mapping || 2086 folio_test_writeback(folio)) 2087 goto unlock; 2088 VM_BUG_ON_FOLIO(!folio_contains(folio, xas.xa_index), 2089 folio); 2090 } 2091 indices[fbatch->nr] = xas.xa_index; 2092 if (!folio_batch_add(fbatch, folio)) 2093 break; 2094 continue; 2095 unlock: 2096 folio_unlock(folio); 2097 put: 2098 folio_put(folio); 2099 } 2100 rcu_read_unlock(); 2101 2102 if (folio_batch_count(fbatch)) { 2103 unsigned long nr = 1; 2104 int idx = folio_batch_count(fbatch) - 1; 2105 2106 folio = fbatch->folios[idx]; 2107 if (!xa_is_value(folio) && !folio_test_hugetlb(folio)) 2108 nr = folio_nr_pages(folio); 2109 *start = indices[idx] + nr; 2110 } 2111 return folio_batch_count(fbatch); 2112 } 2113 2114 /** 2115 * filemap_get_folios - Get a batch of folios 2116 * @mapping: The address_space to search 2117 * @start: The starting page index 2118 * @end: The final page index (inclusive) 2119 * @fbatch: The batch to fill. 2120 * 2121 * Search for and return a batch of folios in the mapping starting at 2122 * index @start and up to index @end (inclusive). The folios are returned 2123 * in @fbatch with an elevated reference count. 2124 * 2125 * The first folio may start before @start; if it does, it will contain 2126 * @start. The final folio may extend beyond @end; if it does, it will 2127 * contain @end. The folios have ascending indices. There may be gaps 2128 * between the folios if there are indices which have no folio in the 2129 * page cache. If folios are added to or removed from the page cache 2130 * while this is running, they may or may not be found by this call. 2131 * 2132 * Return: The number of folios which were found. 2133 * We also update @start to index the next folio for the traversal. 2134 */ 2135 unsigned filemap_get_folios(struct address_space *mapping, pgoff_t *start, 2136 pgoff_t end, struct folio_batch *fbatch) 2137 { 2138 XA_STATE(xas, &mapping->i_pages, *start); 2139 struct folio *folio; 2140 2141 rcu_read_lock(); 2142 while ((folio = find_get_entry(&xas, end, XA_PRESENT)) != NULL) { 2143 /* Skip over shadow, swap and DAX entries */ 2144 if (xa_is_value(folio)) 2145 continue; 2146 if (!folio_batch_add(fbatch, folio)) { 2147 unsigned long nr = folio_nr_pages(folio); 2148 2149 if (folio_test_hugetlb(folio)) 2150 nr = 1; 2151 *start = folio->index + nr; 2152 goto out; 2153 } 2154 } 2155 2156 /* 2157 * We come here when there is no page beyond @end. We take care to not 2158 * overflow the index @start as it confuses some of the callers. This 2159 * breaks the iteration when there is a page at index -1 but that is 2160 * already broken anyway. 2161 */ 2162 if (end == (pgoff_t)-1) 2163 *start = (pgoff_t)-1; 2164 else 2165 *start = end + 1; 2166 out: 2167 rcu_read_unlock(); 2168 2169 return folio_batch_count(fbatch); 2170 } 2171 EXPORT_SYMBOL(filemap_get_folios); 2172 2173 /** 2174 * filemap_get_folios_contig - Get a batch of contiguous folios 2175 * @mapping: The address_space to search 2176 * @start: The starting page index 2177 * @end: The final page index (inclusive) 2178 * @fbatch: The batch to fill 2179 * 2180 * filemap_get_folios_contig() works exactly like filemap_get_folios(), 2181 * except the returned folios are guaranteed to be contiguous. This may 2182 * not return all contiguous folios if the batch gets filled up. 2183 * 2184 * Return: The number of folios found. 2185 * Also update @start to be positioned for traversal of the next folio. 2186 */ 2187 2188 unsigned filemap_get_folios_contig(struct address_space *mapping, 2189 pgoff_t *start, pgoff_t end, struct folio_batch *fbatch) 2190 { 2191 XA_STATE(xas, &mapping->i_pages, *start); 2192 unsigned long nr; 2193 struct folio *folio; 2194 2195 rcu_read_lock(); 2196 2197 for (folio = xas_load(&xas); folio && xas.xa_index <= end; 2198 folio = xas_next(&xas)) { 2199 if (xas_retry(&xas, folio)) 2200 continue; 2201 /* 2202 * If the entry has been swapped out, we can stop looking. 2203 * No current caller is looking for DAX entries. 2204 */ 2205 if (xa_is_value(folio)) 2206 goto update_start; 2207 2208 if (!folio_try_get_rcu(folio)) 2209 goto retry; 2210 2211 if (unlikely(folio != xas_reload(&xas))) 2212 goto put_folio; 2213 2214 if (!folio_batch_add(fbatch, folio)) { 2215 nr = folio_nr_pages(folio); 2216 2217 if (folio_test_hugetlb(folio)) 2218 nr = 1; 2219 *start = folio->index + nr; 2220 goto out; 2221 } 2222 continue; 2223 put_folio: 2224 folio_put(folio); 2225 2226 retry: 2227 xas_reset(&xas); 2228 } 2229 2230 update_start: 2231 nr = folio_batch_count(fbatch); 2232 2233 if (nr) { 2234 folio = fbatch->folios[nr - 1]; 2235 if (folio_test_hugetlb(folio)) 2236 *start = folio->index + 1; 2237 else 2238 *start = folio_next_index(folio); 2239 } 2240 out: 2241 rcu_read_unlock(); 2242 return folio_batch_count(fbatch); 2243 } 2244 EXPORT_SYMBOL(filemap_get_folios_contig); 2245 2246 /** 2247 * filemap_get_folios_tag - Get a batch of folios matching @tag 2248 * @mapping: The address_space to search 2249 * @start: The starting page index 2250 * @end: The final page index (inclusive) 2251 * @tag: The tag index 2252 * @fbatch: The batch to fill 2253 * 2254 * Same as filemap_get_folios(), but only returning folios tagged with @tag. 2255 * 2256 * Return: The number of folios found. 2257 * Also update @start to index the next folio for traversal. 2258 */ 2259 unsigned filemap_get_folios_tag(struct address_space *mapping, pgoff_t *start, 2260 pgoff_t end, xa_mark_t tag, struct folio_batch *fbatch) 2261 { 2262 XA_STATE(xas, &mapping->i_pages, *start); 2263 struct folio *folio; 2264 2265 rcu_read_lock(); 2266 while ((folio = find_get_entry(&xas, end, tag)) != NULL) { 2267 /* 2268 * Shadow entries should never be tagged, but this iteration 2269 * is lockless so there is a window for page reclaim to evict 2270 * a page we saw tagged. Skip over it. 2271 */ 2272 if (xa_is_value(folio)) 2273 continue; 2274 if (!folio_batch_add(fbatch, folio)) { 2275 unsigned long nr = folio_nr_pages(folio); 2276 2277 if (folio_test_hugetlb(folio)) 2278 nr = 1; 2279 *start = folio->index + nr; 2280 goto out; 2281 } 2282 } 2283 /* 2284 * We come here when there is no page beyond @end. We take care to not 2285 * overflow the index @start as it confuses some of the callers. This 2286 * breaks the iteration when there is a page at index -1 but that is 2287 * already broke anyway. 2288 */ 2289 if (end == (pgoff_t)-1) 2290 *start = (pgoff_t)-1; 2291 else 2292 *start = end + 1; 2293 out: 2294 rcu_read_unlock(); 2295 2296 return folio_batch_count(fbatch); 2297 } 2298 EXPORT_SYMBOL(filemap_get_folios_tag); 2299 2300 /* 2301 * CD/DVDs are error prone. When a medium error occurs, the driver may fail 2302 * a _large_ part of the i/o request. Imagine the worst scenario: 2303 * 2304 * ---R__________________________________________B__________ 2305 * ^ reading here ^ bad block(assume 4k) 2306 * 2307 * read(R) => miss => readahead(R...B) => media error => frustrating retries 2308 * => failing the whole request => read(R) => read(R+1) => 2309 * readahead(R+1...B+1) => bang => read(R+2) => read(R+3) => 2310 * readahead(R+3...B+2) => bang => read(R+3) => read(R+4) => 2311 * readahead(R+4...B+3) => bang => read(R+4) => read(R+5) => ...... 2312 * 2313 * It is going insane. Fix it by quickly scaling down the readahead size. 2314 */ 2315 static void shrink_readahead_size_eio(struct file_ra_state *ra) 2316 { 2317 ra->ra_pages /= 4; 2318 } 2319 2320 /* 2321 * filemap_get_read_batch - Get a batch of folios for read 2322 * 2323 * Get a batch of folios which represent a contiguous range of bytes in 2324 * the file. No exceptional entries will be returned. If @index is in 2325 * the middle of a folio, the entire folio will be returned. The last 2326 * folio in the batch may have the readahead flag set or the uptodate flag 2327 * clear so that the caller can take the appropriate action. 2328 */ 2329 static void filemap_get_read_batch(struct address_space *mapping, 2330 pgoff_t index, pgoff_t max, struct folio_batch *fbatch) 2331 { 2332 XA_STATE(xas, &mapping->i_pages, index); 2333 struct folio *folio; 2334 2335 rcu_read_lock(); 2336 for (folio = xas_load(&xas); folio; folio = xas_next(&xas)) { 2337 if (xas_retry(&xas, folio)) 2338 continue; 2339 if (xas.xa_index > max || xa_is_value(folio)) 2340 break; 2341 if (xa_is_sibling(folio)) 2342 break; 2343 if (!folio_try_get_rcu(folio)) 2344 goto retry; 2345 2346 if (unlikely(folio != xas_reload(&xas))) 2347 goto put_folio; 2348 2349 if (!folio_batch_add(fbatch, folio)) 2350 break; 2351 if (!folio_test_uptodate(folio)) 2352 break; 2353 if (folio_test_readahead(folio)) 2354 break; 2355 xas_advance(&xas, folio_next_index(folio) - 1); 2356 continue; 2357 put_folio: 2358 folio_put(folio); 2359 retry: 2360 xas_reset(&xas); 2361 } 2362 rcu_read_unlock(); 2363 } 2364 2365 static int filemap_read_folio(struct file *file, filler_t filler, 2366 struct folio *folio) 2367 { 2368 bool workingset = folio_test_workingset(folio); 2369 unsigned long pflags; 2370 int error; 2371 2372 /* 2373 * A previous I/O error may have been due to temporary failures, 2374 * eg. multipath errors. PG_error will be set again if read_folio 2375 * fails. 2376 */ 2377 folio_clear_error(folio); 2378 2379 /* Start the actual read. The read will unlock the page. */ 2380 if (unlikely(workingset)) 2381 psi_memstall_enter(&pflags); 2382 error = filler(file, folio); 2383 if (unlikely(workingset)) 2384 psi_memstall_leave(&pflags); 2385 if (error) 2386 return error; 2387 2388 error = folio_wait_locked_killable(folio); 2389 if (error) 2390 return error; 2391 if (folio_test_uptodate(folio)) 2392 return 0; 2393 if (file) 2394 shrink_readahead_size_eio(&file->f_ra); 2395 return -EIO; 2396 } 2397 2398 static bool filemap_range_uptodate(struct address_space *mapping, 2399 loff_t pos, size_t count, struct folio *folio, 2400 bool need_uptodate) 2401 { 2402 if (folio_test_uptodate(folio)) 2403 return true; 2404 /* pipes can't handle partially uptodate pages */ 2405 if (need_uptodate) 2406 return false; 2407 if (!mapping->a_ops->is_partially_uptodate) 2408 return false; 2409 if (mapping->host->i_blkbits >= folio_shift(folio)) 2410 return false; 2411 2412 if (folio_pos(folio) > pos) { 2413 count -= folio_pos(folio) - pos; 2414 pos = 0; 2415 } else { 2416 pos -= folio_pos(folio); 2417 } 2418 2419 return mapping->a_ops->is_partially_uptodate(folio, pos, count); 2420 } 2421 2422 static int filemap_update_page(struct kiocb *iocb, 2423 struct address_space *mapping, size_t count, 2424 struct folio *folio, bool need_uptodate) 2425 { 2426 int error; 2427 2428 if (iocb->ki_flags & IOCB_NOWAIT) { 2429 if (!filemap_invalidate_trylock_shared(mapping)) 2430 return -EAGAIN; 2431 } else { 2432 filemap_invalidate_lock_shared(mapping); 2433 } 2434 2435 if (!folio_trylock(folio)) { 2436 error = -EAGAIN; 2437 if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_NOIO)) 2438 goto unlock_mapping; 2439 if (!(iocb->ki_flags & IOCB_WAITQ)) { 2440 filemap_invalidate_unlock_shared(mapping); 2441 /* 2442 * This is where we usually end up waiting for a 2443 * previously submitted readahead to finish. 2444 */ 2445 folio_put_wait_locked(folio, TASK_KILLABLE); 2446 return AOP_TRUNCATED_PAGE; 2447 } 2448 error = __folio_lock_async(folio, iocb->ki_waitq); 2449 if (error) 2450 goto unlock_mapping; 2451 } 2452 2453 error = AOP_TRUNCATED_PAGE; 2454 if (!folio->mapping) 2455 goto unlock; 2456 2457 error = 0; 2458 if (filemap_range_uptodate(mapping, iocb->ki_pos, count, folio, 2459 need_uptodate)) 2460 goto unlock; 2461 2462 error = -EAGAIN; 2463 if (iocb->ki_flags & (IOCB_NOIO | IOCB_NOWAIT | IOCB_WAITQ)) 2464 goto unlock; 2465 2466 error = filemap_read_folio(iocb->ki_filp, mapping->a_ops->read_folio, 2467 folio); 2468 goto unlock_mapping; 2469 unlock: 2470 folio_unlock(folio); 2471 unlock_mapping: 2472 filemap_invalidate_unlock_shared(mapping); 2473 if (error == AOP_TRUNCATED_PAGE) 2474 folio_put(folio); 2475 return error; 2476 } 2477 2478 static int filemap_create_folio(struct file *file, 2479 struct address_space *mapping, pgoff_t index, 2480 struct folio_batch *fbatch) 2481 { 2482 struct folio *folio; 2483 int error; 2484 2485 folio = filemap_alloc_folio(mapping_gfp_mask(mapping), 0); 2486 if (!folio) 2487 return -ENOMEM; 2488 2489 /* 2490 * Protect against truncate / hole punch. Grabbing invalidate_lock 2491 * here assures we cannot instantiate and bring uptodate new 2492 * pagecache folios after evicting page cache during truncate 2493 * and before actually freeing blocks. Note that we could 2494 * release invalidate_lock after inserting the folio into 2495 * the page cache as the locked folio would then be enough to 2496 * synchronize with hole punching. But there are code paths 2497 * such as filemap_update_page() filling in partially uptodate 2498 * pages or ->readahead() that need to hold invalidate_lock 2499 * while mapping blocks for IO so let's hold the lock here as 2500 * well to keep locking rules simple. 2501 */ 2502 filemap_invalidate_lock_shared(mapping); 2503 error = filemap_add_folio(mapping, folio, index, 2504 mapping_gfp_constraint(mapping, GFP_KERNEL)); 2505 if (error == -EEXIST) 2506 error = AOP_TRUNCATED_PAGE; 2507 if (error) 2508 goto error; 2509 2510 error = filemap_read_folio(file, mapping->a_ops->read_folio, folio); 2511 if (error) 2512 goto error; 2513 2514 filemap_invalidate_unlock_shared(mapping); 2515 folio_batch_add(fbatch, folio); 2516 return 0; 2517 error: 2518 filemap_invalidate_unlock_shared(mapping); 2519 folio_put(folio); 2520 return error; 2521 } 2522 2523 static int filemap_readahead(struct kiocb *iocb, struct file *file, 2524 struct address_space *mapping, struct folio *folio, 2525 pgoff_t last_index) 2526 { 2527 DEFINE_READAHEAD(ractl, file, &file->f_ra, mapping, folio->index); 2528 2529 if (iocb->ki_flags & IOCB_NOIO) 2530 return -EAGAIN; 2531 page_cache_async_ra(&ractl, folio, last_index - folio->index); 2532 return 0; 2533 } 2534 2535 static int filemap_get_pages(struct kiocb *iocb, size_t count, 2536 struct folio_batch *fbatch, bool need_uptodate) 2537 { 2538 struct file *filp = iocb->ki_filp; 2539 struct address_space *mapping = filp->f_mapping; 2540 struct file_ra_state *ra = &filp->f_ra; 2541 pgoff_t index = iocb->ki_pos >> PAGE_SHIFT; 2542 pgoff_t last_index; 2543 struct folio *folio; 2544 int err = 0; 2545 2546 /* "last_index" is the index of the page beyond the end of the read */ 2547 last_index = DIV_ROUND_UP(iocb->ki_pos + count, PAGE_SIZE); 2548 retry: 2549 if (fatal_signal_pending(current)) 2550 return -EINTR; 2551 2552 filemap_get_read_batch(mapping, index, last_index - 1, fbatch); 2553 if (!folio_batch_count(fbatch)) { 2554 if (iocb->ki_flags & IOCB_NOIO) 2555 return -EAGAIN; 2556 page_cache_sync_readahead(mapping, ra, filp, index, 2557 last_index - index); 2558 filemap_get_read_batch(mapping, index, last_index - 1, fbatch); 2559 } 2560 if (!folio_batch_count(fbatch)) { 2561 if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_WAITQ)) 2562 return -EAGAIN; 2563 err = filemap_create_folio(filp, mapping, 2564 iocb->ki_pos >> PAGE_SHIFT, fbatch); 2565 if (err == AOP_TRUNCATED_PAGE) 2566 goto retry; 2567 return err; 2568 } 2569 2570 folio = fbatch->folios[folio_batch_count(fbatch) - 1]; 2571 if (folio_test_readahead(folio)) { 2572 err = filemap_readahead(iocb, filp, mapping, folio, last_index); 2573 if (err) 2574 goto err; 2575 } 2576 if (!folio_test_uptodate(folio)) { 2577 if ((iocb->ki_flags & IOCB_WAITQ) && 2578 folio_batch_count(fbatch) > 1) 2579 iocb->ki_flags |= IOCB_NOWAIT; 2580 err = filemap_update_page(iocb, mapping, count, folio, 2581 need_uptodate); 2582 if (err) 2583 goto err; 2584 } 2585 2586 return 0; 2587 err: 2588 if (err < 0) 2589 folio_put(folio); 2590 if (likely(--fbatch->nr)) 2591 return 0; 2592 if (err == AOP_TRUNCATED_PAGE) 2593 goto retry; 2594 return err; 2595 } 2596 2597 static inline bool pos_same_folio(loff_t pos1, loff_t pos2, struct folio *folio) 2598 { 2599 unsigned int shift = folio_shift(folio); 2600 2601 return (pos1 >> shift == pos2 >> shift); 2602 } 2603 2604 /** 2605 * filemap_read - Read data from the page cache. 2606 * @iocb: The iocb to read. 2607 * @iter: Destination for the data. 2608 * @already_read: Number of bytes already read by the caller. 2609 * 2610 * Copies data from the page cache. If the data is not currently present, 2611 * uses the readahead and read_folio address_space operations to fetch it. 2612 * 2613 * Return: Total number of bytes copied, including those already read by 2614 * the caller. If an error happens before any bytes are copied, returns 2615 * a negative error number. 2616 */ 2617 ssize_t filemap_read(struct kiocb *iocb, struct iov_iter *iter, 2618 ssize_t already_read) 2619 { 2620 struct file *filp = iocb->ki_filp; 2621 struct file_ra_state *ra = &filp->f_ra; 2622 struct address_space *mapping = filp->f_mapping; 2623 struct inode *inode = mapping->host; 2624 struct folio_batch fbatch; 2625 int i, error = 0; 2626 bool writably_mapped; 2627 loff_t isize, end_offset; 2628 loff_t last_pos = ra->prev_pos; 2629 2630 if (unlikely(iocb->ki_pos >= inode->i_sb->s_maxbytes)) 2631 return 0; 2632 if (unlikely(!iov_iter_count(iter))) 2633 return 0; 2634 2635 iov_iter_truncate(iter, inode->i_sb->s_maxbytes); 2636 folio_batch_init(&fbatch); 2637 2638 do { 2639 cond_resched(); 2640 2641 /* 2642 * If we've already successfully copied some data, then we 2643 * can no longer safely return -EIOCBQUEUED. Hence mark 2644 * an async read NOWAIT at that point. 2645 */ 2646 if ((iocb->ki_flags & IOCB_WAITQ) && already_read) 2647 iocb->ki_flags |= IOCB_NOWAIT; 2648 2649 if (unlikely(iocb->ki_pos >= i_size_read(inode))) 2650 break; 2651 2652 error = filemap_get_pages(iocb, iter->count, &fbatch, false); 2653 if (error < 0) 2654 break; 2655 2656 /* 2657 * i_size must be checked after we know the pages are Uptodate. 2658 * 2659 * Checking i_size after the check allows us to calculate 2660 * the correct value for "nr", which means the zero-filled 2661 * part of the page is not copied back to userspace (unless 2662 * another truncate extends the file - this is desired though). 2663 */ 2664 isize = i_size_read(inode); 2665 if (unlikely(iocb->ki_pos >= isize)) 2666 goto put_folios; 2667 end_offset = min_t(loff_t, isize, iocb->ki_pos + iter->count); 2668 2669 /* 2670 * Once we start copying data, we don't want to be touching any 2671 * cachelines that might be contended: 2672 */ 2673 writably_mapped = mapping_writably_mapped(mapping); 2674 2675 /* 2676 * When a read accesses the same folio several times, only 2677 * mark it as accessed the first time. 2678 */ 2679 if (!pos_same_folio(iocb->ki_pos, last_pos - 1, 2680 fbatch.folios[0])) 2681 folio_mark_accessed(fbatch.folios[0]); 2682 2683 for (i = 0; i < folio_batch_count(&fbatch); i++) { 2684 struct folio *folio = fbatch.folios[i]; 2685 size_t fsize = folio_size(folio); 2686 size_t offset = iocb->ki_pos & (fsize - 1); 2687 size_t bytes = min_t(loff_t, end_offset - iocb->ki_pos, 2688 fsize - offset); 2689 size_t copied; 2690 2691 if (end_offset < folio_pos(folio)) 2692 break; 2693 if (i > 0) 2694 folio_mark_accessed(folio); 2695 /* 2696 * If users can be writing to this folio using arbitrary 2697 * virtual addresses, take care of potential aliasing 2698 * before reading the folio on the kernel side. 2699 */ 2700 if (writably_mapped) 2701 flush_dcache_folio(folio); 2702 2703 copied = copy_folio_to_iter(folio, offset, bytes, iter); 2704 2705 already_read += copied; 2706 iocb->ki_pos += copied; 2707 last_pos = iocb->ki_pos; 2708 2709 if (copied < bytes) { 2710 error = -EFAULT; 2711 break; 2712 } 2713 } 2714 put_folios: 2715 for (i = 0; i < folio_batch_count(&fbatch); i++) 2716 folio_put(fbatch.folios[i]); 2717 folio_batch_init(&fbatch); 2718 } while (iov_iter_count(iter) && iocb->ki_pos < isize && !error); 2719 2720 file_accessed(filp); 2721 ra->prev_pos = last_pos; 2722 return already_read ? already_read : error; 2723 } 2724 EXPORT_SYMBOL_GPL(filemap_read); 2725 2726 int kiocb_write_and_wait(struct kiocb *iocb, size_t count) 2727 { 2728 struct address_space *mapping = iocb->ki_filp->f_mapping; 2729 loff_t pos = iocb->ki_pos; 2730 loff_t end = pos + count - 1; 2731 2732 if (iocb->ki_flags & IOCB_NOWAIT) { 2733 if (filemap_range_needs_writeback(mapping, pos, end)) 2734 return -EAGAIN; 2735 return 0; 2736 } 2737 2738 return filemap_write_and_wait_range(mapping, pos, end); 2739 } 2740 2741 int kiocb_invalidate_pages(struct kiocb *iocb, size_t count) 2742 { 2743 struct address_space *mapping = iocb->ki_filp->f_mapping; 2744 loff_t pos = iocb->ki_pos; 2745 loff_t end = pos + count - 1; 2746 int ret; 2747 2748 if (iocb->ki_flags & IOCB_NOWAIT) { 2749 /* we could block if there are any pages in the range */ 2750 if (filemap_range_has_page(mapping, pos, end)) 2751 return -EAGAIN; 2752 } else { 2753 ret = filemap_write_and_wait_range(mapping, pos, end); 2754 if (ret) 2755 return ret; 2756 } 2757 2758 /* 2759 * After a write we want buffered reads to be sure to go to disk to get 2760 * the new data. We invalidate clean cached page from the region we're 2761 * about to write. We do this *before* the write so that we can return 2762 * without clobbering -EIOCBQUEUED from ->direct_IO(). 2763 */ 2764 return invalidate_inode_pages2_range(mapping, pos >> PAGE_SHIFT, 2765 end >> PAGE_SHIFT); 2766 } 2767 2768 /** 2769 * generic_file_read_iter - generic filesystem read routine 2770 * @iocb: kernel I/O control block 2771 * @iter: destination for the data read 2772 * 2773 * This is the "read_iter()" routine for all filesystems 2774 * that can use the page cache directly. 2775 * 2776 * The IOCB_NOWAIT flag in iocb->ki_flags indicates that -EAGAIN shall 2777 * be returned when no data can be read without waiting for I/O requests 2778 * to complete; it doesn't prevent readahead. 2779 * 2780 * The IOCB_NOIO flag in iocb->ki_flags indicates that no new I/O 2781 * requests shall be made for the read or for readahead. When no data 2782 * can be read, -EAGAIN shall be returned. When readahead would be 2783 * triggered, a partial, possibly empty read shall be returned. 2784 * 2785 * Return: 2786 * * number of bytes copied, even for partial reads 2787 * * negative error code (or 0 if IOCB_NOIO) if nothing was read 2788 */ 2789 ssize_t 2790 generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) 2791 { 2792 size_t count = iov_iter_count(iter); 2793 ssize_t retval = 0; 2794 2795 if (!count) 2796 return 0; /* skip atime */ 2797 2798 if (iocb->ki_flags & IOCB_DIRECT) { 2799 struct file *file = iocb->ki_filp; 2800 struct address_space *mapping = file->f_mapping; 2801 struct inode *inode = mapping->host; 2802 2803 retval = kiocb_write_and_wait(iocb, count); 2804 if (retval < 0) 2805 return retval; 2806 file_accessed(file); 2807 2808 retval = mapping->a_ops->direct_IO(iocb, iter); 2809 if (retval >= 0) { 2810 iocb->ki_pos += retval; 2811 count -= retval; 2812 } 2813 if (retval != -EIOCBQUEUED) 2814 iov_iter_revert(iter, count - iov_iter_count(iter)); 2815 2816 /* 2817 * Btrfs can have a short DIO read if we encounter 2818 * compressed extents, so if there was an error, or if 2819 * we've already read everything we wanted to, or if 2820 * there was a short read because we hit EOF, go ahead 2821 * and return. Otherwise fallthrough to buffered io for 2822 * the rest of the read. Buffered reads will not work for 2823 * DAX files, so don't bother trying. 2824 */ 2825 if (retval < 0 || !count || IS_DAX(inode)) 2826 return retval; 2827 if (iocb->ki_pos >= i_size_read(inode)) 2828 return retval; 2829 } 2830 2831 return filemap_read(iocb, iter, retval); 2832 } 2833 EXPORT_SYMBOL(generic_file_read_iter); 2834 2835 /* 2836 * Splice subpages from a folio into a pipe. 2837 */ 2838 size_t splice_folio_into_pipe(struct pipe_inode_info *pipe, 2839 struct folio *folio, loff_t fpos, size_t size) 2840 { 2841 struct page *page; 2842 size_t spliced = 0, offset = offset_in_folio(folio, fpos); 2843 2844 page = folio_page(folio, offset / PAGE_SIZE); 2845 size = min(size, folio_size(folio) - offset); 2846 offset %= PAGE_SIZE; 2847 2848 while (spliced < size && 2849 !pipe_full(pipe->head, pipe->tail, pipe->max_usage)) { 2850 struct pipe_buffer *buf = pipe_head_buf(pipe); 2851 size_t part = min_t(size_t, PAGE_SIZE - offset, size - spliced); 2852 2853 *buf = (struct pipe_buffer) { 2854 .ops = &page_cache_pipe_buf_ops, 2855 .page = page, 2856 .offset = offset, 2857 .len = part, 2858 }; 2859 folio_get(folio); 2860 pipe->head++; 2861 page++; 2862 spliced += part; 2863 offset = 0; 2864 } 2865 2866 return spliced; 2867 } 2868 2869 /** 2870 * filemap_splice_read - Splice data from a file's pagecache into a pipe 2871 * @in: The file to read from 2872 * @ppos: Pointer to the file position to read from 2873 * @pipe: The pipe to splice into 2874 * @len: The amount to splice 2875 * @flags: The SPLICE_F_* flags 2876 * 2877 * This function gets folios from a file's pagecache and splices them into the 2878 * pipe. Readahead will be called as necessary to fill more folios. This may 2879 * be used for blockdevs also. 2880 * 2881 * Return: On success, the number of bytes read will be returned and *@ppos 2882 * will be updated if appropriate; 0 will be returned if there is no more data 2883 * to be read; -EAGAIN will be returned if the pipe had no space, and some 2884 * other negative error code will be returned on error. A short read may occur 2885 * if the pipe has insufficient space, we reach the end of the data or we hit a 2886 * hole. 2887 */ 2888 ssize_t filemap_splice_read(struct file *in, loff_t *ppos, 2889 struct pipe_inode_info *pipe, 2890 size_t len, unsigned int flags) 2891 { 2892 struct folio_batch fbatch; 2893 struct kiocb iocb; 2894 size_t total_spliced = 0, used, npages; 2895 loff_t isize, end_offset; 2896 bool writably_mapped; 2897 int i, error = 0; 2898 2899 if (unlikely(*ppos >= in->f_mapping->host->i_sb->s_maxbytes)) 2900 return 0; 2901 2902 init_sync_kiocb(&iocb, in); 2903 iocb.ki_pos = *ppos; 2904 2905 /* Work out how much data we can actually add into the pipe */ 2906 used = pipe_occupancy(pipe->head, pipe->tail); 2907 npages = max_t(ssize_t, pipe->max_usage - used, 0); 2908 len = min_t(size_t, len, npages * PAGE_SIZE); 2909 2910 folio_batch_init(&fbatch); 2911 2912 do { 2913 cond_resched(); 2914 2915 if (*ppos >= i_size_read(in->f_mapping->host)) 2916 break; 2917 2918 iocb.ki_pos = *ppos; 2919 error = filemap_get_pages(&iocb, len, &fbatch, true); 2920 if (error < 0) 2921 break; 2922 2923 /* 2924 * i_size must be checked after we know the pages are Uptodate. 2925 * 2926 * Checking i_size after the check allows us to calculate 2927 * the correct value for "nr", which means the zero-filled 2928 * part of the page is not copied back to userspace (unless 2929 * another truncate extends the file - this is desired though). 2930 */ 2931 isize = i_size_read(in->f_mapping->host); 2932 if (unlikely(*ppos >= isize)) 2933 break; 2934 end_offset = min_t(loff_t, isize, *ppos + len); 2935 2936 /* 2937 * Once we start copying data, we don't want to be touching any 2938 * cachelines that might be contended: 2939 */ 2940 writably_mapped = mapping_writably_mapped(in->f_mapping); 2941 2942 for (i = 0; i < folio_batch_count(&fbatch); i++) { 2943 struct folio *folio = fbatch.folios[i]; 2944 size_t n; 2945 2946 if (folio_pos(folio) >= end_offset) 2947 goto out; 2948 folio_mark_accessed(folio); 2949 2950 /* 2951 * If users can be writing to this folio using arbitrary 2952 * virtual addresses, take care of potential aliasing 2953 * before reading the folio on the kernel side. 2954 */ 2955 if (writably_mapped) 2956 flush_dcache_folio(folio); 2957 2958 n = min_t(loff_t, len, isize - *ppos); 2959 n = splice_folio_into_pipe(pipe, folio, *ppos, n); 2960 if (!n) 2961 goto out; 2962 len -= n; 2963 total_spliced += n; 2964 *ppos += n; 2965 in->f_ra.prev_pos = *ppos; 2966 if (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) 2967 goto out; 2968 } 2969 2970 folio_batch_release(&fbatch); 2971 } while (len); 2972 2973 out: 2974 folio_batch_release(&fbatch); 2975 file_accessed(in); 2976 2977 return total_spliced ? total_spliced : error; 2978 } 2979 EXPORT_SYMBOL(filemap_splice_read); 2980 2981 static inline loff_t folio_seek_hole_data(struct xa_state *xas, 2982 struct address_space *mapping, struct folio *folio, 2983 loff_t start, loff_t end, bool seek_data) 2984 { 2985 const struct address_space_operations *ops = mapping->a_ops; 2986 size_t offset, bsz = i_blocksize(mapping->host); 2987 2988 if (xa_is_value(folio) || folio_test_uptodate(folio)) 2989 return seek_data ? start : end; 2990 if (!ops->is_partially_uptodate) 2991 return seek_data ? end : start; 2992 2993 xas_pause(xas); 2994 rcu_read_unlock(); 2995 folio_lock(folio); 2996 if (unlikely(folio->mapping != mapping)) 2997 goto unlock; 2998 2999 offset = offset_in_folio(folio, start) & ~(bsz - 1); 3000 3001 do { 3002 if (ops->is_partially_uptodate(folio, offset, bsz) == 3003 seek_data) 3004 break; 3005 start = (start + bsz) & ~(bsz - 1); 3006 offset += bsz; 3007 } while (offset < folio_size(folio)); 3008 unlock: 3009 folio_unlock(folio); 3010 rcu_read_lock(); 3011 return start; 3012 } 3013 3014 static inline size_t seek_folio_size(struct xa_state *xas, struct folio *folio) 3015 { 3016 if (xa_is_value(folio)) 3017 return PAGE_SIZE << xa_get_order(xas->xa, xas->xa_index); 3018 return folio_size(folio); 3019 } 3020 3021 /** 3022 * mapping_seek_hole_data - Seek for SEEK_DATA / SEEK_HOLE in the page cache. 3023 * @mapping: Address space to search. 3024 * @start: First byte to consider. 3025 * @end: Limit of search (exclusive). 3026 * @whence: Either SEEK_HOLE or SEEK_DATA. 3027 * 3028 * If the page cache knows which blocks contain holes and which blocks 3029 * contain data, your filesystem can use this function to implement 3030 * SEEK_HOLE and SEEK_DATA. This is useful for filesystems which are 3031 * entirely memory-based such as tmpfs, and filesystems which support 3032 * unwritten extents. 3033 * 3034 * Return: The requested offset on success, or -ENXIO if @whence specifies 3035 * SEEK_DATA and there is no data after @start. There is an implicit hole 3036 * after @end - 1, so SEEK_HOLE returns @end if all the bytes between @start 3037 * and @end contain data. 3038 */ 3039 loff_t mapping_seek_hole_data(struct address_space *mapping, loff_t start, 3040 loff_t end, int whence) 3041 { 3042 XA_STATE(xas, &mapping->i_pages, start >> PAGE_SHIFT); 3043 pgoff_t max = (end - 1) >> PAGE_SHIFT; 3044 bool seek_data = (whence == SEEK_DATA); 3045 struct folio *folio; 3046 3047 if (end <= start) 3048 return -ENXIO; 3049 3050 rcu_read_lock(); 3051 while ((folio = find_get_entry(&xas, max, XA_PRESENT))) { 3052 loff_t pos = (u64)xas.xa_index << PAGE_SHIFT; 3053 size_t seek_size; 3054 3055 if (start < pos) { 3056 if (!seek_data) 3057 goto unlock; 3058 start = pos; 3059 } 3060 3061 seek_size = seek_folio_size(&xas, folio); 3062 pos = round_up((u64)pos + 1, seek_size); 3063 start = folio_seek_hole_data(&xas, mapping, folio, start, pos, 3064 seek_data); 3065 if (start < pos) 3066 goto unlock; 3067 if (start >= end) 3068 break; 3069 if (seek_size > PAGE_SIZE) 3070 xas_set(&xas, pos >> PAGE_SHIFT); 3071 if (!xa_is_value(folio)) 3072 folio_put(folio); 3073 } 3074 if (seek_data) 3075 start = -ENXIO; 3076 unlock: 3077 rcu_read_unlock(); 3078 if (folio && !xa_is_value(folio)) 3079 folio_put(folio); 3080 if (start > end) 3081 return end; 3082 return start; 3083 } 3084 3085 #ifdef CONFIG_MMU 3086 #define MMAP_LOTSAMISS (100) 3087 /* 3088 * lock_folio_maybe_drop_mmap - lock the page, possibly dropping the mmap_lock 3089 * @vmf - the vm_fault for this fault. 3090 * @folio - the folio to lock. 3091 * @fpin - the pointer to the file we may pin (or is already pinned). 3092 * 3093 * This works similar to lock_folio_or_retry in that it can drop the 3094 * mmap_lock. It differs in that it actually returns the folio locked 3095 * if it returns 1 and 0 if it couldn't lock the folio. If we did have 3096 * to drop the mmap_lock then fpin will point to the pinned file and 3097 * needs to be fput()'ed at a later point. 3098 */ 3099 static int lock_folio_maybe_drop_mmap(struct vm_fault *vmf, struct folio *folio, 3100 struct file **fpin) 3101 { 3102 if (folio_trylock(folio)) 3103 return 1; 3104 3105 /* 3106 * NOTE! This will make us return with VM_FAULT_RETRY, but with 3107 * the mmap_lock still held. That's how FAULT_FLAG_RETRY_NOWAIT 3108 * is supposed to work. We have way too many special cases.. 3109 */ 3110 if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) 3111 return 0; 3112 3113 *fpin = maybe_unlock_mmap_for_io(vmf, *fpin); 3114 if (vmf->flags & FAULT_FLAG_KILLABLE) { 3115 if (__folio_lock_killable(folio)) { 3116 /* 3117 * We didn't have the right flags to drop the mmap_lock, 3118 * but all fault_handlers only check for fatal signals 3119 * if we return VM_FAULT_RETRY, so we need to drop the 3120 * mmap_lock here and return 0 if we don't have a fpin. 3121 */ 3122 if (*fpin == NULL) 3123 mmap_read_unlock(vmf->vma->vm_mm); 3124 return 0; 3125 } 3126 } else 3127 __folio_lock(folio); 3128 3129 return 1; 3130 } 3131 3132 /* 3133 * Synchronous readahead happens when we don't even find a page in the page 3134 * cache at all. We don't want to perform IO under the mmap sem, so if we have 3135 * to drop the mmap sem we return the file that was pinned in order for us to do 3136 * that. If we didn't pin a file then we return NULL. The file that is 3137 * returned needs to be fput()'ed when we're done with it. 3138 */ 3139 static struct file *do_sync_mmap_readahead(struct vm_fault *vmf) 3140 { 3141 struct file *file = vmf->vma->vm_file; 3142 struct file_ra_state *ra = &file->f_ra; 3143 struct address_space *mapping = file->f_mapping; 3144 DEFINE_READAHEAD(ractl, file, ra, mapping, vmf->pgoff); 3145 struct file *fpin = NULL; 3146 unsigned long vm_flags = vmf->vma->vm_flags; 3147 unsigned int mmap_miss; 3148 3149 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3150 /* Use the readahead code, even if readahead is disabled */ 3151 if (vm_flags & VM_HUGEPAGE) { 3152 fpin = maybe_unlock_mmap_for_io(vmf, fpin); 3153 ractl._index &= ~((unsigned long)HPAGE_PMD_NR - 1); 3154 ra->size = HPAGE_PMD_NR; 3155 /* 3156 * Fetch two PMD folios, so we get the chance to actually 3157 * readahead, unless we've been told not to. 3158 */ 3159 if (!(vm_flags & VM_RAND_READ)) 3160 ra->size *= 2; 3161 ra->async_size = HPAGE_PMD_NR; 3162 page_cache_ra_order(&ractl, ra, HPAGE_PMD_ORDER); 3163 return fpin; 3164 } 3165 #endif 3166 3167 /* If we don't want any read-ahead, don't bother */ 3168 if (vm_flags & VM_RAND_READ) 3169 return fpin; 3170 if (!ra->ra_pages) 3171 return fpin; 3172 3173 if (vm_flags & VM_SEQ_READ) { 3174 fpin = maybe_unlock_mmap_for_io(vmf, fpin); 3175 page_cache_sync_ra(&ractl, ra->ra_pages); 3176 return fpin; 3177 } 3178 3179 /* Avoid banging the cache line if not needed */ 3180 mmap_miss = READ_ONCE(ra->mmap_miss); 3181 if (mmap_miss < MMAP_LOTSAMISS * 10) 3182 WRITE_ONCE(ra->mmap_miss, ++mmap_miss); 3183 3184 /* 3185 * Do we miss much more than hit in this file? If so, 3186 * stop bothering with read-ahead. It will only hurt. 3187 */ 3188 if (mmap_miss > MMAP_LOTSAMISS) 3189 return fpin; 3190 3191 /* 3192 * mmap read-around 3193 */ 3194 fpin = maybe_unlock_mmap_for_io(vmf, fpin); 3195 ra->start = max_t(long, 0, vmf->pgoff - ra->ra_pages / 2); 3196 ra->size = ra->ra_pages; 3197 ra->async_size = ra->ra_pages / 4; 3198 ractl._index = ra->start; 3199 page_cache_ra_order(&ractl, ra, 0); 3200 return fpin; 3201 } 3202 3203 /* 3204 * Asynchronous readahead happens when we find the page and PG_readahead, 3205 * so we want to possibly extend the readahead further. We return the file that 3206 * was pinned if we have to drop the mmap_lock in order to do IO. 3207 */ 3208 static struct file *do_async_mmap_readahead(struct vm_fault *vmf, 3209 struct folio *folio) 3210 { 3211 struct file *file = vmf->vma->vm_file; 3212 struct file_ra_state *ra = &file->f_ra; 3213 DEFINE_READAHEAD(ractl, file, ra, file->f_mapping, vmf->pgoff); 3214 struct file *fpin = NULL; 3215 unsigned int mmap_miss; 3216 3217 /* If we don't want any read-ahead, don't bother */ 3218 if (vmf->vma->vm_flags & VM_RAND_READ || !ra->ra_pages) 3219 return fpin; 3220 3221 mmap_miss = READ_ONCE(ra->mmap_miss); 3222 if (mmap_miss) 3223 WRITE_ONCE(ra->mmap_miss, --mmap_miss); 3224 3225 if (folio_test_readahead(folio)) { 3226 fpin = maybe_unlock_mmap_for_io(vmf, fpin); 3227 page_cache_async_ra(&ractl, folio, ra->ra_pages); 3228 } 3229 return fpin; 3230 } 3231 3232 /** 3233 * filemap_fault - read in file data for page fault handling 3234 * @vmf: struct vm_fault containing details of the fault 3235 * 3236 * filemap_fault() is invoked via the vma operations vector for a 3237 * mapped memory region to read in file data during a page fault. 3238 * 3239 * The goto's are kind of ugly, but this streamlines the normal case of having 3240 * it in the page cache, and handles the special cases reasonably without 3241 * having a lot of duplicated code. 3242 * 3243 * vma->vm_mm->mmap_lock must be held on entry. 3244 * 3245 * If our return value has VM_FAULT_RETRY set, it's because the mmap_lock 3246 * may be dropped before doing I/O or by lock_folio_maybe_drop_mmap(). 3247 * 3248 * If our return value does not have VM_FAULT_RETRY set, the mmap_lock 3249 * has not been released. 3250 * 3251 * We never return with VM_FAULT_RETRY and a bit from VM_FAULT_ERROR set. 3252 * 3253 * Return: bitwise-OR of %VM_FAULT_ codes. 3254 */ 3255 vm_fault_t filemap_fault(struct vm_fault *vmf) 3256 { 3257 int error; 3258 struct file *file = vmf->vma->vm_file; 3259 struct file *fpin = NULL; 3260 struct address_space *mapping = file->f_mapping; 3261 struct inode *inode = mapping->host; 3262 pgoff_t max_idx, index = vmf->pgoff; 3263 struct folio *folio; 3264 vm_fault_t ret = 0; 3265 bool mapping_locked = false; 3266 3267 max_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); 3268 if (unlikely(index >= max_idx)) 3269 return VM_FAULT_SIGBUS; 3270 3271 /* 3272 * Do we have something in the page cache already? 3273 */ 3274 folio = filemap_get_folio(mapping, index); 3275 if (likely(!IS_ERR(folio))) { 3276 /* 3277 * We found the page, so try async readahead before waiting for 3278 * the lock. 3279 */ 3280 if (!(vmf->flags & FAULT_FLAG_TRIED)) 3281 fpin = do_async_mmap_readahead(vmf, folio); 3282 if (unlikely(!folio_test_uptodate(folio))) { 3283 filemap_invalidate_lock_shared(mapping); 3284 mapping_locked = true; 3285 } 3286 } else { 3287 /* No page in the page cache at all */ 3288 count_vm_event(PGMAJFAULT); 3289 count_memcg_event_mm(vmf->vma->vm_mm, PGMAJFAULT); 3290 ret = VM_FAULT_MAJOR; 3291 fpin = do_sync_mmap_readahead(vmf); 3292 retry_find: 3293 /* 3294 * See comment in filemap_create_folio() why we need 3295 * invalidate_lock 3296 */ 3297 if (!mapping_locked) { 3298 filemap_invalidate_lock_shared(mapping); 3299 mapping_locked = true; 3300 } 3301 folio = __filemap_get_folio(mapping, index, 3302 FGP_CREAT|FGP_FOR_MMAP, 3303 vmf->gfp_mask); 3304 if (IS_ERR(folio)) { 3305 if (fpin) 3306 goto out_retry; 3307 filemap_invalidate_unlock_shared(mapping); 3308 return VM_FAULT_OOM; 3309 } 3310 } 3311 3312 if (!lock_folio_maybe_drop_mmap(vmf, folio, &fpin)) 3313 goto out_retry; 3314 3315 /* Did it get truncated? */ 3316 if (unlikely(folio->mapping != mapping)) { 3317 folio_unlock(folio); 3318 folio_put(folio); 3319 goto retry_find; 3320 } 3321 VM_BUG_ON_FOLIO(!folio_contains(folio, index), folio); 3322 3323 /* 3324 * We have a locked page in the page cache, now we need to check 3325 * that it's up-to-date. If not, it is going to be due to an error. 3326 */ 3327 if (unlikely(!folio_test_uptodate(folio))) { 3328 /* 3329 * The page was in cache and uptodate and now it is not. 3330 * Strange but possible since we didn't hold the page lock all 3331 * the time. Let's drop everything get the invalidate lock and 3332 * try again. 3333 */ 3334 if (!mapping_locked) { 3335 folio_unlock(folio); 3336 folio_put(folio); 3337 goto retry_find; 3338 } 3339 goto page_not_uptodate; 3340 } 3341 3342 /* 3343 * We've made it this far and we had to drop our mmap_lock, now is the 3344 * time to return to the upper layer and have it re-find the vma and 3345 * redo the fault. 3346 */ 3347 if (fpin) { 3348 folio_unlock(folio); 3349 goto out_retry; 3350 } 3351 if (mapping_locked) 3352 filemap_invalidate_unlock_shared(mapping); 3353 3354 /* 3355 * Found the page and have a reference on it. 3356 * We must recheck i_size under page lock. 3357 */ 3358 max_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); 3359 if (unlikely(index >= max_idx)) { 3360 folio_unlock(folio); 3361 folio_put(folio); 3362 return VM_FAULT_SIGBUS; 3363 } 3364 3365 vmf->page = folio_file_page(folio, index); 3366 return ret | VM_FAULT_LOCKED; 3367 3368 page_not_uptodate: 3369 /* 3370 * Umm, take care of errors if the page isn't up-to-date. 3371 * Try to re-read it _once_. We do this synchronously, 3372 * because there really aren't any performance issues here 3373 * and we need to check for errors. 3374 */ 3375 fpin = maybe_unlock_mmap_for_io(vmf, fpin); 3376 error = filemap_read_folio(file, mapping->a_ops->read_folio, folio); 3377 if (fpin) 3378 goto out_retry; 3379 folio_put(folio); 3380 3381 if (!error || error == AOP_TRUNCATED_PAGE) 3382 goto retry_find; 3383 filemap_invalidate_unlock_shared(mapping); 3384 3385 return VM_FAULT_SIGBUS; 3386 3387 out_retry: 3388 /* 3389 * We dropped the mmap_lock, we need to return to the fault handler to 3390 * re-find the vma and come back and find our hopefully still populated 3391 * page. 3392 */ 3393 if (!IS_ERR(folio)) 3394 folio_put(folio); 3395 if (mapping_locked) 3396 filemap_invalidate_unlock_shared(mapping); 3397 if (fpin) 3398 fput(fpin); 3399 return ret | VM_FAULT_RETRY; 3400 } 3401 EXPORT_SYMBOL(filemap_fault); 3402 3403 static bool filemap_map_pmd(struct vm_fault *vmf, struct folio *folio, 3404 pgoff_t start) 3405 { 3406 struct mm_struct *mm = vmf->vma->vm_mm; 3407 3408 /* Huge page is mapped? No need to proceed. */ 3409 if (pmd_trans_huge(*vmf->pmd)) { 3410 folio_unlock(folio); 3411 folio_put(folio); 3412 return true; 3413 } 3414 3415 if (pmd_none(*vmf->pmd) && folio_test_pmd_mappable(folio)) { 3416 struct page *page = folio_file_page(folio, start); 3417 vm_fault_t ret = do_set_pmd(vmf, page); 3418 if (!ret) { 3419 /* The page is mapped successfully, reference consumed. */ 3420 folio_unlock(folio); 3421 return true; 3422 } 3423 } 3424 3425 if (pmd_none(*vmf->pmd)) 3426 pmd_install(mm, vmf->pmd, &vmf->prealloc_pte); 3427 3428 return false; 3429 } 3430 3431 static struct folio *next_uptodate_folio(struct xa_state *xas, 3432 struct address_space *mapping, pgoff_t end_pgoff) 3433 { 3434 struct folio *folio = xas_next_entry(xas, end_pgoff); 3435 unsigned long max_idx; 3436 3437 do { 3438 if (!folio) 3439 return NULL; 3440 if (xas_retry(xas, folio)) 3441 continue; 3442 if (xa_is_value(folio)) 3443 continue; 3444 if (folio_test_locked(folio)) 3445 continue; 3446 if (!folio_try_get_rcu(folio)) 3447 continue; 3448 /* Has the page moved or been split? */ 3449 if (unlikely(folio != xas_reload(xas))) 3450 goto skip; 3451 if (!folio_test_uptodate(folio) || folio_test_readahead(folio)) 3452 goto skip; 3453 if (!folio_trylock(folio)) 3454 goto skip; 3455 if (folio->mapping != mapping) 3456 goto unlock; 3457 if (!folio_test_uptodate(folio)) 3458 goto unlock; 3459 max_idx = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); 3460 if (xas->xa_index >= max_idx) 3461 goto unlock; 3462 return folio; 3463 unlock: 3464 folio_unlock(folio); 3465 skip: 3466 folio_put(folio); 3467 } while ((folio = xas_next_entry(xas, end_pgoff)) != NULL); 3468 3469 return NULL; 3470 } 3471 3472 /* 3473 * Map page range [start_page, start_page + nr_pages) of folio. 3474 * start_page is gotten from start by folio_page(folio, start) 3475 */ 3476 static vm_fault_t filemap_map_folio_range(struct vm_fault *vmf, 3477 struct folio *folio, unsigned long start, 3478 unsigned long addr, unsigned int nr_pages, 3479 unsigned int *mmap_miss) 3480 { 3481 vm_fault_t ret = 0; 3482 struct page *page = folio_page(folio, start); 3483 unsigned int count = 0; 3484 pte_t *old_ptep = vmf->pte; 3485 3486 do { 3487 if (PageHWPoison(page + count)) 3488 goto skip; 3489 3490 (*mmap_miss)++; 3491 3492 /* 3493 * NOTE: If there're PTE markers, we'll leave them to be 3494 * handled in the specific fault path, and it'll prohibit the 3495 * fault-around logic. 3496 */ 3497 if (!pte_none(vmf->pte[count])) 3498 goto skip; 3499 3500 count++; 3501 continue; 3502 skip: 3503 if (count) { 3504 set_pte_range(vmf, folio, page, count, addr); 3505 folio_ref_add(folio, count); 3506 if (in_range(vmf->address, addr, count * PAGE_SIZE)) 3507 ret = VM_FAULT_NOPAGE; 3508 } 3509 3510 count++; 3511 page += count; 3512 vmf->pte += count; 3513 addr += count * PAGE_SIZE; 3514 count = 0; 3515 } while (--nr_pages > 0); 3516 3517 if (count) { 3518 set_pte_range(vmf, folio, page, count, addr); 3519 folio_ref_add(folio, count); 3520 if (in_range(vmf->address, addr, count * PAGE_SIZE)) 3521 ret = VM_FAULT_NOPAGE; 3522 } 3523 3524 vmf->pte = old_ptep; 3525 3526 return ret; 3527 } 3528 3529 static vm_fault_t filemap_map_order0_folio(struct vm_fault *vmf, 3530 struct folio *folio, unsigned long addr, 3531 unsigned int *mmap_miss) 3532 { 3533 vm_fault_t ret = 0; 3534 struct page *page = &folio->page; 3535 3536 if (PageHWPoison(page)) 3537 return ret; 3538 3539 (*mmap_miss)++; 3540 3541 /* 3542 * NOTE: If there're PTE markers, we'll leave them to be 3543 * handled in the specific fault path, and it'll prohibit 3544 * the fault-around logic. 3545 */ 3546 if (!pte_none(ptep_get(vmf->pte))) 3547 return ret; 3548 3549 if (vmf->address == addr) 3550 ret = VM_FAULT_NOPAGE; 3551 3552 set_pte_range(vmf, folio, page, 1, addr); 3553 folio_ref_inc(folio); 3554 3555 return ret; 3556 } 3557 3558 vm_fault_t filemap_map_pages(struct vm_fault *vmf, 3559 pgoff_t start_pgoff, pgoff_t end_pgoff) 3560 { 3561 struct vm_area_struct *vma = vmf->vma; 3562 struct file *file = vma->vm_file; 3563 struct address_space *mapping = file->f_mapping; 3564 pgoff_t last_pgoff = start_pgoff; 3565 unsigned long addr; 3566 XA_STATE(xas, &mapping->i_pages, start_pgoff); 3567 struct folio *folio; 3568 vm_fault_t ret = 0; 3569 unsigned int nr_pages = 0, mmap_miss = 0, mmap_miss_saved; 3570 3571 rcu_read_lock(); 3572 folio = next_uptodate_folio(&xas, mapping, end_pgoff); 3573 if (!folio) 3574 goto out; 3575 3576 if (filemap_map_pmd(vmf, folio, start_pgoff)) { 3577 ret = VM_FAULT_NOPAGE; 3578 goto out; 3579 } 3580 3581 addr = vma->vm_start + ((start_pgoff - vma->vm_pgoff) << PAGE_SHIFT); 3582 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, addr, &vmf->ptl); 3583 if (!vmf->pte) { 3584 folio_unlock(folio); 3585 folio_put(folio); 3586 goto out; 3587 } 3588 do { 3589 unsigned long end; 3590 3591 addr += (xas.xa_index - last_pgoff) << PAGE_SHIFT; 3592 vmf->pte += xas.xa_index - last_pgoff; 3593 last_pgoff = xas.xa_index; 3594 end = folio->index + folio_nr_pages(folio) - 1; 3595 nr_pages = min(end, end_pgoff) - xas.xa_index + 1; 3596 3597 if (!folio_test_large(folio)) 3598 ret |= filemap_map_order0_folio(vmf, 3599 folio, addr, &mmap_miss); 3600 else 3601 ret |= filemap_map_folio_range(vmf, folio, 3602 xas.xa_index - folio->index, addr, 3603 nr_pages, &mmap_miss); 3604 3605 folio_unlock(folio); 3606 folio_put(folio); 3607 } while ((folio = next_uptodate_folio(&xas, mapping, end_pgoff)) != NULL); 3608 pte_unmap_unlock(vmf->pte, vmf->ptl); 3609 out: 3610 rcu_read_unlock(); 3611 3612 mmap_miss_saved = READ_ONCE(file->f_ra.mmap_miss); 3613 if (mmap_miss >= mmap_miss_saved) 3614 WRITE_ONCE(file->f_ra.mmap_miss, 0); 3615 else 3616 WRITE_ONCE(file->f_ra.mmap_miss, mmap_miss_saved - mmap_miss); 3617 3618 return ret; 3619 } 3620 EXPORT_SYMBOL(filemap_map_pages); 3621 3622 vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf) 3623 { 3624 struct address_space *mapping = vmf->vma->vm_file->f_mapping; 3625 struct folio *folio = page_folio(vmf->page); 3626 vm_fault_t ret = VM_FAULT_LOCKED; 3627 3628 sb_start_pagefault(mapping->host->i_sb); 3629 file_update_time(vmf->vma->vm_file); 3630 folio_lock(folio); 3631 if (folio->mapping != mapping) { 3632 folio_unlock(folio); 3633 ret = VM_FAULT_NOPAGE; 3634 goto out; 3635 } 3636 /* 3637 * We mark the folio dirty already here so that when freeze is in 3638 * progress, we are guaranteed that writeback during freezing will 3639 * see the dirty folio and writeprotect it again. 3640 */ 3641 folio_mark_dirty(folio); 3642 folio_wait_stable(folio); 3643 out: 3644 sb_end_pagefault(mapping->host->i_sb); 3645 return ret; 3646 } 3647 3648 const struct vm_operations_struct generic_file_vm_ops = { 3649 .fault = filemap_fault, 3650 .map_pages = filemap_map_pages, 3651 .page_mkwrite = filemap_page_mkwrite, 3652 }; 3653 3654 /* This is used for a general mmap of a disk file */ 3655 3656 int generic_file_mmap(struct file *file, struct vm_area_struct *vma) 3657 { 3658 struct address_space *mapping = file->f_mapping; 3659 3660 if (!mapping->a_ops->read_folio) 3661 return -ENOEXEC; 3662 file_accessed(file); 3663 vma->vm_ops = &generic_file_vm_ops; 3664 return 0; 3665 } 3666 3667 /* 3668 * This is for filesystems which do not implement ->writepage. 3669 */ 3670 int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma) 3671 { 3672 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE)) 3673 return -EINVAL; 3674 return generic_file_mmap(file, vma); 3675 } 3676 #else 3677 vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf) 3678 { 3679 return VM_FAULT_SIGBUS; 3680 } 3681 int generic_file_mmap(struct file *file, struct vm_area_struct *vma) 3682 { 3683 return -ENOSYS; 3684 } 3685 int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma) 3686 { 3687 return -ENOSYS; 3688 } 3689 #endif /* CONFIG_MMU */ 3690 3691 EXPORT_SYMBOL(filemap_page_mkwrite); 3692 EXPORT_SYMBOL(generic_file_mmap); 3693 EXPORT_SYMBOL(generic_file_readonly_mmap); 3694 3695 static struct folio *do_read_cache_folio(struct address_space *mapping, 3696 pgoff_t index, filler_t filler, struct file *file, gfp_t gfp) 3697 { 3698 struct folio *folio; 3699 int err; 3700 3701 if (!filler) 3702 filler = mapping->a_ops->read_folio; 3703 repeat: 3704 folio = filemap_get_folio(mapping, index); 3705 if (IS_ERR(folio)) { 3706 folio = filemap_alloc_folio(gfp, 0); 3707 if (!folio) 3708 return ERR_PTR(-ENOMEM); 3709 err = filemap_add_folio(mapping, folio, index, gfp); 3710 if (unlikely(err)) { 3711 folio_put(folio); 3712 if (err == -EEXIST) 3713 goto repeat; 3714 /* Presumably ENOMEM for xarray node */ 3715 return ERR_PTR(err); 3716 } 3717 3718 goto filler; 3719 } 3720 if (folio_test_uptodate(folio)) 3721 goto out; 3722 3723 if (!folio_trylock(folio)) { 3724 folio_put_wait_locked(folio, TASK_UNINTERRUPTIBLE); 3725 goto repeat; 3726 } 3727 3728 /* Folio was truncated from mapping */ 3729 if (!folio->mapping) { 3730 folio_unlock(folio); 3731 folio_put(folio); 3732 goto repeat; 3733 } 3734 3735 /* Someone else locked and filled the page in a very small window */ 3736 if (folio_test_uptodate(folio)) { 3737 folio_unlock(folio); 3738 goto out; 3739 } 3740 3741 filler: 3742 err = filemap_read_folio(file, filler, folio); 3743 if (err) { 3744 folio_put(folio); 3745 if (err == AOP_TRUNCATED_PAGE) 3746 goto repeat; 3747 return ERR_PTR(err); 3748 } 3749 3750 out: 3751 folio_mark_accessed(folio); 3752 return folio; 3753 } 3754 3755 /** 3756 * read_cache_folio - Read into page cache, fill it if needed. 3757 * @mapping: The address_space to read from. 3758 * @index: The index to read. 3759 * @filler: Function to perform the read, or NULL to use aops->read_folio(). 3760 * @file: Passed to filler function, may be NULL if not required. 3761 * 3762 * Read one page into the page cache. If it succeeds, the folio returned 3763 * will contain @index, but it may not be the first page of the folio. 3764 * 3765 * If the filler function returns an error, it will be returned to the 3766 * caller. 3767 * 3768 * Context: May sleep. Expects mapping->invalidate_lock to be held. 3769 * Return: An uptodate folio on success, ERR_PTR() on failure. 3770 */ 3771 struct folio *read_cache_folio(struct address_space *mapping, pgoff_t index, 3772 filler_t filler, struct file *file) 3773 { 3774 return do_read_cache_folio(mapping, index, filler, file, 3775 mapping_gfp_mask(mapping)); 3776 } 3777 EXPORT_SYMBOL(read_cache_folio); 3778 3779 /** 3780 * mapping_read_folio_gfp - Read into page cache, using specified allocation flags. 3781 * @mapping: The address_space for the folio. 3782 * @index: The index that the allocated folio will contain. 3783 * @gfp: The page allocator flags to use if allocating. 3784 * 3785 * This is the same as "read_cache_folio(mapping, index, NULL, NULL)", but with 3786 * any new memory allocations done using the specified allocation flags. 3787 * 3788 * The most likely error from this function is EIO, but ENOMEM is 3789 * possible and so is EINTR. If ->read_folio returns another error, 3790 * that will be returned to the caller. 3791 * 3792 * The function expects mapping->invalidate_lock to be already held. 3793 * 3794 * Return: Uptodate folio on success, ERR_PTR() on failure. 3795 */ 3796 struct folio *mapping_read_folio_gfp(struct address_space *mapping, 3797 pgoff_t index, gfp_t gfp) 3798 { 3799 return do_read_cache_folio(mapping, index, NULL, NULL, gfp); 3800 } 3801 EXPORT_SYMBOL(mapping_read_folio_gfp); 3802 3803 static struct page *do_read_cache_page(struct address_space *mapping, 3804 pgoff_t index, filler_t *filler, struct file *file, gfp_t gfp) 3805 { 3806 struct folio *folio; 3807 3808 folio = do_read_cache_folio(mapping, index, filler, file, gfp); 3809 if (IS_ERR(folio)) 3810 return &folio->page; 3811 return folio_file_page(folio, index); 3812 } 3813 3814 struct page *read_cache_page(struct address_space *mapping, 3815 pgoff_t index, filler_t *filler, struct file *file) 3816 { 3817 return do_read_cache_page(mapping, index, filler, file, 3818 mapping_gfp_mask(mapping)); 3819 } 3820 EXPORT_SYMBOL(read_cache_page); 3821 3822 /** 3823 * read_cache_page_gfp - read into page cache, using specified page allocation flags. 3824 * @mapping: the page's address_space 3825 * @index: the page index 3826 * @gfp: the page allocator flags to use if allocating 3827 * 3828 * This is the same as "read_mapping_page(mapping, index, NULL)", but with 3829 * any new page allocations done using the specified allocation flags. 3830 * 3831 * If the page does not get brought uptodate, return -EIO. 3832 * 3833 * The function expects mapping->invalidate_lock to be already held. 3834 * 3835 * Return: up to date page on success, ERR_PTR() on failure. 3836 */ 3837 struct page *read_cache_page_gfp(struct address_space *mapping, 3838 pgoff_t index, 3839 gfp_t gfp) 3840 { 3841 return do_read_cache_page(mapping, index, NULL, NULL, gfp); 3842 } 3843 EXPORT_SYMBOL(read_cache_page_gfp); 3844 3845 /* 3846 * Warn about a page cache invalidation failure during a direct I/O write. 3847 */ 3848 static void dio_warn_stale_pagecache(struct file *filp) 3849 { 3850 static DEFINE_RATELIMIT_STATE(_rs, 86400 * HZ, DEFAULT_RATELIMIT_BURST); 3851 char pathname[128]; 3852 char *path; 3853 3854 errseq_set(&filp->f_mapping->wb_err, -EIO); 3855 if (__ratelimit(&_rs)) { 3856 path = file_path(filp, pathname, sizeof(pathname)); 3857 if (IS_ERR(path)) 3858 path = "(unknown)"; 3859 pr_crit("Page cache invalidation failure on direct I/O. Possible data corruption due to collision with buffered I/O!\n"); 3860 pr_crit("File: %s PID: %d Comm: %.20s\n", path, current->pid, 3861 current->comm); 3862 } 3863 } 3864 3865 void kiocb_invalidate_post_direct_write(struct kiocb *iocb, size_t count) 3866 { 3867 struct address_space *mapping = iocb->ki_filp->f_mapping; 3868 3869 if (mapping->nrpages && 3870 invalidate_inode_pages2_range(mapping, 3871 iocb->ki_pos >> PAGE_SHIFT, 3872 (iocb->ki_pos + count - 1) >> PAGE_SHIFT)) 3873 dio_warn_stale_pagecache(iocb->ki_filp); 3874 } 3875 3876 ssize_t 3877 generic_file_direct_write(struct kiocb *iocb, struct iov_iter *from) 3878 { 3879 struct address_space *mapping = iocb->ki_filp->f_mapping; 3880 size_t write_len = iov_iter_count(from); 3881 ssize_t written; 3882 3883 /* 3884 * If a page can not be invalidated, return 0 to fall back 3885 * to buffered write. 3886 */ 3887 written = kiocb_invalidate_pages(iocb, write_len); 3888 if (written) { 3889 if (written == -EBUSY) 3890 return 0; 3891 return written; 3892 } 3893 3894 written = mapping->a_ops->direct_IO(iocb, from); 3895 3896 /* 3897 * Finally, try again to invalidate clean pages which might have been 3898 * cached by non-direct readahead, or faulted in by get_user_pages() 3899 * if the source of the write was an mmap'ed region of the file 3900 * we're writing. Either one is a pretty crazy thing to do, 3901 * so we don't support it 100%. If this invalidation 3902 * fails, tough, the write still worked... 3903 * 3904 * Most of the time we do not need this since dio_complete() will do 3905 * the invalidation for us. However there are some file systems that 3906 * do not end up with dio_complete() being called, so let's not break 3907 * them by removing it completely. 3908 * 3909 * Noticeable example is a blkdev_direct_IO(). 3910 * 3911 * Skip invalidation for async writes or if mapping has no pages. 3912 */ 3913 if (written > 0) { 3914 struct inode *inode = mapping->host; 3915 loff_t pos = iocb->ki_pos; 3916 3917 kiocb_invalidate_post_direct_write(iocb, written); 3918 pos += written; 3919 write_len -= written; 3920 if (pos > i_size_read(inode) && !S_ISBLK(inode->i_mode)) { 3921 i_size_write(inode, pos); 3922 mark_inode_dirty(inode); 3923 } 3924 iocb->ki_pos = pos; 3925 } 3926 if (written != -EIOCBQUEUED) 3927 iov_iter_revert(from, write_len - iov_iter_count(from)); 3928 return written; 3929 } 3930 EXPORT_SYMBOL(generic_file_direct_write); 3931 3932 ssize_t generic_perform_write(struct kiocb *iocb, struct iov_iter *i) 3933 { 3934 struct file *file = iocb->ki_filp; 3935 loff_t pos = iocb->ki_pos; 3936 struct address_space *mapping = file->f_mapping; 3937 const struct address_space_operations *a_ops = mapping->a_ops; 3938 long status = 0; 3939 ssize_t written = 0; 3940 3941 do { 3942 struct page *page; 3943 unsigned long offset; /* Offset into pagecache page */ 3944 unsigned long bytes; /* Bytes to write to page */ 3945 size_t copied; /* Bytes copied from user */ 3946 void *fsdata = NULL; 3947 3948 offset = (pos & (PAGE_SIZE - 1)); 3949 bytes = min_t(unsigned long, PAGE_SIZE - offset, 3950 iov_iter_count(i)); 3951 3952 again: 3953 /* 3954 * Bring in the user page that we will copy from _first_. 3955 * Otherwise there's a nasty deadlock on copying from the 3956 * same page as we're writing to, without it being marked 3957 * up-to-date. 3958 */ 3959 if (unlikely(fault_in_iov_iter_readable(i, bytes) == bytes)) { 3960 status = -EFAULT; 3961 break; 3962 } 3963 3964 if (fatal_signal_pending(current)) { 3965 status = -EINTR; 3966 break; 3967 } 3968 3969 status = a_ops->write_begin(file, mapping, pos, bytes, 3970 &page, &fsdata); 3971 if (unlikely(status < 0)) 3972 break; 3973 3974 if (mapping_writably_mapped(mapping)) 3975 flush_dcache_page(page); 3976 3977 copied = copy_page_from_iter_atomic(page, offset, bytes, i); 3978 flush_dcache_page(page); 3979 3980 status = a_ops->write_end(file, mapping, pos, bytes, copied, 3981 page, fsdata); 3982 if (unlikely(status != copied)) { 3983 iov_iter_revert(i, copied - max(status, 0L)); 3984 if (unlikely(status < 0)) 3985 break; 3986 } 3987 cond_resched(); 3988 3989 if (unlikely(status == 0)) { 3990 /* 3991 * A short copy made ->write_end() reject the 3992 * thing entirely. Might be memory poisoning 3993 * halfway through, might be a race with munmap, 3994 * might be severe memory pressure. 3995 */ 3996 if (copied) 3997 bytes = copied; 3998 goto again; 3999 } 4000 pos += status; 4001 written += status; 4002 4003 balance_dirty_pages_ratelimited(mapping); 4004 } while (iov_iter_count(i)); 4005 4006 if (!written) 4007 return status; 4008 iocb->ki_pos += written; 4009 return written; 4010 } 4011 EXPORT_SYMBOL(generic_perform_write); 4012 4013 /** 4014 * __generic_file_write_iter - write data to a file 4015 * @iocb: IO state structure (file, offset, etc.) 4016 * @from: iov_iter with data to write 4017 * 4018 * This function does all the work needed for actually writing data to a 4019 * file. It does all basic checks, removes SUID from the file, updates 4020 * modification times and calls proper subroutines depending on whether we 4021 * do direct IO or a standard buffered write. 4022 * 4023 * It expects i_rwsem to be grabbed unless we work on a block device or similar 4024 * object which does not need locking at all. 4025 * 4026 * This function does *not* take care of syncing data in case of O_SYNC write. 4027 * A caller has to handle it. This is mainly due to the fact that we want to 4028 * avoid syncing under i_rwsem. 4029 * 4030 * Return: 4031 * * number of bytes written, even for truncated writes 4032 * * negative error code if no data has been written at all 4033 */ 4034 ssize_t __generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from) 4035 { 4036 struct file *file = iocb->ki_filp; 4037 struct address_space *mapping = file->f_mapping; 4038 struct inode *inode = mapping->host; 4039 ssize_t ret; 4040 4041 ret = file_remove_privs(file); 4042 if (ret) 4043 return ret; 4044 4045 ret = file_update_time(file); 4046 if (ret) 4047 return ret; 4048 4049 if (iocb->ki_flags & IOCB_DIRECT) { 4050 ret = generic_file_direct_write(iocb, from); 4051 /* 4052 * If the write stopped short of completing, fall back to 4053 * buffered writes. Some filesystems do this for writes to 4054 * holes, for example. For DAX files, a buffered write will 4055 * not succeed (even if it did, DAX does not handle dirty 4056 * page-cache pages correctly). 4057 */ 4058 if (ret < 0 || !iov_iter_count(from) || IS_DAX(inode)) 4059 return ret; 4060 return direct_write_fallback(iocb, from, ret, 4061 generic_perform_write(iocb, from)); 4062 } 4063 4064 return generic_perform_write(iocb, from); 4065 } 4066 EXPORT_SYMBOL(__generic_file_write_iter); 4067 4068 /** 4069 * generic_file_write_iter - write data to a file 4070 * @iocb: IO state structure 4071 * @from: iov_iter with data to write 4072 * 4073 * This is a wrapper around __generic_file_write_iter() to be used by most 4074 * filesystems. It takes care of syncing the file in case of O_SYNC file 4075 * and acquires i_rwsem as needed. 4076 * Return: 4077 * * negative error code if no data has been written at all of 4078 * vfs_fsync_range() failed for a synchronous write 4079 * * number of bytes written, even for truncated writes 4080 */ 4081 ssize_t generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from) 4082 { 4083 struct file *file = iocb->ki_filp; 4084 struct inode *inode = file->f_mapping->host; 4085 ssize_t ret; 4086 4087 inode_lock(inode); 4088 ret = generic_write_checks(iocb, from); 4089 if (ret > 0) 4090 ret = __generic_file_write_iter(iocb, from); 4091 inode_unlock(inode); 4092 4093 if (ret > 0) 4094 ret = generic_write_sync(iocb, ret); 4095 return ret; 4096 } 4097 EXPORT_SYMBOL(generic_file_write_iter); 4098 4099 /** 4100 * filemap_release_folio() - Release fs-specific metadata on a folio. 4101 * @folio: The folio which the kernel is trying to free. 4102 * @gfp: Memory allocation flags (and I/O mode). 4103 * 4104 * The address_space is trying to release any data attached to a folio 4105 * (presumably at folio->private). 4106 * 4107 * This will also be called if the private_2 flag is set on a page, 4108 * indicating that the folio has other metadata associated with it. 4109 * 4110 * The @gfp argument specifies whether I/O may be performed to release 4111 * this page (__GFP_IO), and whether the call may block 4112 * (__GFP_RECLAIM & __GFP_FS). 4113 * 4114 * Return: %true if the release was successful, otherwise %false. 4115 */ 4116 bool filemap_release_folio(struct folio *folio, gfp_t gfp) 4117 { 4118 struct address_space * const mapping = folio->mapping; 4119 4120 BUG_ON(!folio_test_locked(folio)); 4121 if (!folio_needs_release(folio)) 4122 return true; 4123 if (folio_test_writeback(folio)) 4124 return false; 4125 4126 if (mapping && mapping->a_ops->release_folio) 4127 return mapping->a_ops->release_folio(folio, gfp); 4128 return try_to_free_buffers(folio); 4129 } 4130 EXPORT_SYMBOL(filemap_release_folio); 4131 4132 #ifdef CONFIG_CACHESTAT_SYSCALL 4133 /** 4134 * filemap_cachestat() - compute the page cache statistics of a mapping 4135 * @mapping: The mapping to compute the statistics for. 4136 * @first_index: The starting page cache index. 4137 * @last_index: The final page index (inclusive). 4138 * @cs: the cachestat struct to write the result to. 4139 * 4140 * This will query the page cache statistics of a mapping in the 4141 * page range of [first_index, last_index] (inclusive). The statistics 4142 * queried include: number of dirty pages, number of pages marked for 4143 * writeback, and the number of (recently) evicted pages. 4144 */ 4145 static void filemap_cachestat(struct address_space *mapping, 4146 pgoff_t first_index, pgoff_t last_index, struct cachestat *cs) 4147 { 4148 XA_STATE(xas, &mapping->i_pages, first_index); 4149 struct folio *folio; 4150 4151 rcu_read_lock(); 4152 xas_for_each(&xas, folio, last_index) { 4153 unsigned long nr_pages; 4154 pgoff_t folio_first_index, folio_last_index; 4155 4156 if (xas_retry(&xas, folio)) 4157 continue; 4158 4159 if (xa_is_value(folio)) { 4160 /* page is evicted */ 4161 void *shadow = (void *)folio; 4162 bool workingset; /* not used */ 4163 int order = xa_get_order(xas.xa, xas.xa_index); 4164 4165 nr_pages = 1 << order; 4166 folio_first_index = round_down(xas.xa_index, 1 << order); 4167 folio_last_index = folio_first_index + nr_pages - 1; 4168 4169 /* Folios might straddle the range boundaries, only count covered pages */ 4170 if (folio_first_index < first_index) 4171 nr_pages -= first_index - folio_first_index; 4172 4173 if (folio_last_index > last_index) 4174 nr_pages -= folio_last_index - last_index; 4175 4176 cs->nr_evicted += nr_pages; 4177 4178 #ifdef CONFIG_SWAP /* implies CONFIG_MMU */ 4179 if (shmem_mapping(mapping)) { 4180 /* shmem file - in swap cache */ 4181 swp_entry_t swp = radix_to_swp_entry(folio); 4182 4183 shadow = get_shadow_from_swap_cache(swp); 4184 } 4185 #endif 4186 if (workingset_test_recent(shadow, true, &workingset)) 4187 cs->nr_recently_evicted += nr_pages; 4188 4189 goto resched; 4190 } 4191 4192 nr_pages = folio_nr_pages(folio); 4193 folio_first_index = folio_pgoff(folio); 4194 folio_last_index = folio_first_index + nr_pages - 1; 4195 4196 /* Folios might straddle the range boundaries, only count covered pages */ 4197 if (folio_first_index < first_index) 4198 nr_pages -= first_index - folio_first_index; 4199 4200 if (folio_last_index > last_index) 4201 nr_pages -= folio_last_index - last_index; 4202 4203 /* page is in cache */ 4204 cs->nr_cache += nr_pages; 4205 4206 if (folio_test_dirty(folio)) 4207 cs->nr_dirty += nr_pages; 4208 4209 if (folio_test_writeback(folio)) 4210 cs->nr_writeback += nr_pages; 4211 4212 resched: 4213 if (need_resched()) { 4214 xas_pause(&xas); 4215 cond_resched_rcu(); 4216 } 4217 } 4218 rcu_read_unlock(); 4219 } 4220 4221 /* 4222 * The cachestat(2) system call. 4223 * 4224 * cachestat() returns the page cache statistics of a file in the 4225 * bytes range specified by `off` and `len`: number of cached pages, 4226 * number of dirty pages, number of pages marked for writeback, 4227 * number of evicted pages, and number of recently evicted pages. 4228 * 4229 * An evicted page is a page that is previously in the page cache 4230 * but has been evicted since. A page is recently evicted if its last 4231 * eviction was recent enough that its reentry to the cache would 4232 * indicate that it is actively being used by the system, and that 4233 * there is memory pressure on the system. 4234 * 4235 * `off` and `len` must be non-negative integers. If `len` > 0, 4236 * the queried range is [`off`, `off` + `len`]. If `len` == 0, 4237 * we will query in the range from `off` to the end of the file. 4238 * 4239 * The `flags` argument is unused for now, but is included for future 4240 * extensibility. User should pass 0 (i.e no flag specified). 4241 * 4242 * Currently, hugetlbfs is not supported. 4243 * 4244 * Because the status of a page can change after cachestat() checks it 4245 * but before it returns to the application, the returned values may 4246 * contain stale information. 4247 * 4248 * return values: 4249 * zero - success 4250 * -EFAULT - cstat or cstat_range points to an illegal address 4251 * -EINVAL - invalid flags 4252 * -EBADF - invalid file descriptor 4253 * -EOPNOTSUPP - file descriptor is of a hugetlbfs file 4254 */ 4255 SYSCALL_DEFINE4(cachestat, unsigned int, fd, 4256 struct cachestat_range __user *, cstat_range, 4257 struct cachestat __user *, cstat, unsigned int, flags) 4258 { 4259 struct fd f = fdget(fd); 4260 struct address_space *mapping; 4261 struct cachestat_range csr; 4262 struct cachestat cs; 4263 pgoff_t first_index, last_index; 4264 4265 if (!f.file) 4266 return -EBADF; 4267 4268 if (copy_from_user(&csr, cstat_range, 4269 sizeof(struct cachestat_range))) { 4270 fdput(f); 4271 return -EFAULT; 4272 } 4273 4274 /* hugetlbfs is not supported */ 4275 if (is_file_hugepages(f.file)) { 4276 fdput(f); 4277 return -EOPNOTSUPP; 4278 } 4279 4280 if (flags != 0) { 4281 fdput(f); 4282 return -EINVAL; 4283 } 4284 4285 first_index = csr.off >> PAGE_SHIFT; 4286 last_index = 4287 csr.len == 0 ? ULONG_MAX : (csr.off + csr.len - 1) >> PAGE_SHIFT; 4288 memset(&cs, 0, sizeof(struct cachestat)); 4289 mapping = f.file->f_mapping; 4290 filemap_cachestat(mapping, first_index, last_index, &cs); 4291 fdput(f); 4292 4293 if (copy_to_user(cstat, &cs, sizeof(struct cachestat))) 4294 return -EFAULT; 4295 4296 return 0; 4297 } 4298 #endif /* CONFIG_CACHESTAT_SYSCALL */