개념 설명 전체 · v6.18.37 / kernel/time/timekeeping.c
1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Kernel timekeeping code and accessor functions. Based on code from 4 * timer.c, moved in commit 8524070b7982. 5 */ 6 #include <linux/timekeeper_internal.h> 7 #include <linux/module.h> 8 #include <linux/interrupt.h> 9 #include <linux/kobject.h> 10 #include <linux/percpu.h> 11 #include <linux/init.h> 12 #include <linux/mm.h> 13 #include <linux/nmi.h> 14 #include <linux/sched.h> 15 #include <linux/sched/loadavg.h> 16 #include <linux/sched/clock.h> 17 #include <linux/syscore_ops.h> 18 #include <linux/clocksource.h> 19 #include <linux/jiffies.h> 20 #include <linux/time.h> 21 #include <linux/timex.h> 22 #include <linux/tick.h> 23 #include <linux/stop_machine.h> 24 #include <linux/pvclock_gtod.h> 25 #include <linux/compiler.h> 26 #include <linux/audit.h> 27 #include <linux/random.h> 28 29 #include <vdso/auxclock.h> 30 31 #include "tick-internal.h" 32 #include "ntp_internal.h" 33 #include "timekeeping_internal.h" 34 35 #define TK_CLEAR_NTP (1 << 0) 36 #define TK_CLOCK_WAS_SET (1 << 1) 37 38 #define TK_UPDATE_ALL (TK_CLEAR_NTP | TK_CLOCK_WAS_SET) 39 40 enum timekeeping_adv_mode { 41 /* Update timekeeper when a tick has passed */ 42 TK_ADV_TICK, 43 44 /* Update timekeeper on a direct frequency change */ 45 TK_ADV_FREQ 46 }; 47 48 /* 49 * The most important data for readout fits into a single 64 byte 50 * cache line. 51 */ 52 struct tk_data { 53 seqcount_raw_spinlock_t seq; 54 struct timekeeper timekeeper; 55 struct timekeeper shadow_timekeeper; 56 raw_spinlock_t lock; 57 } ____cacheline_aligned; 58 59 static struct tk_data timekeeper_data[TIMEKEEPERS_MAX]; 60 61 /* The core timekeeper */ 62 #define tk_core (timekeeper_data[TIMEKEEPER_CORE]) 63 64 #ifdef CONFIG_POSIX_AUX_CLOCKS 65 static inline bool tk_get_aux_ts64(unsigned int tkid, struct timespec64 *ts) 66 { 67 return ktime_get_aux_ts64(CLOCK_AUX + tkid - TIMEKEEPER_AUX_FIRST, ts); 68 } 69 70 static inline bool tk_is_aux(const struct timekeeper *tk) 71 { 72 return tk->id >= TIMEKEEPER_AUX_FIRST && tk->id <= TIMEKEEPER_AUX_LAST; 73 } 74 #else 75 static inline bool tk_get_aux_ts64(unsigned int tkid, struct timespec64 *ts) 76 { 77 return false; 78 } 79 80 static inline bool tk_is_aux(const struct timekeeper *tk) 81 { 82 return false; 83 } 84 #endif 85 86 static inline void tk_update_aux_offs(struct timekeeper *tk, ktime_t offs) 87 { 88 tk->offs_aux = offs; 89 tk->monotonic_to_aux = ktime_to_timespec64(offs); 90 } 91 92 /* flag for if timekeeping is suspended */ 93 int __read_mostly timekeeping_suspended; 94 95 /** 96 * struct tk_fast - NMI safe timekeeper 97 * @seq: Sequence counter for protecting updates. The lowest bit 98 * is the index for the tk_read_base array 99 * @base: tk_read_base array. Access is indexed by the lowest bit of 100 * @seq. 101 * 102 * See @update_fast_timekeeper() below. 103 */ 104 struct tk_fast { 105 seqcount_latch_t seq; 106 struct tk_read_base base[2]; 107 }; 108 109 /* Suspend-time cycles value for halted fast timekeeper. */ 110 static u64 cycles_at_suspend; 111 112 static u64 dummy_clock_read(struct clocksource *cs) 113 { 114 if (timekeeping_suspended) 115 return cycles_at_suspend; 116 return local_clock(); 117 } 118 119 static struct clocksource dummy_clock = { 120 .read = dummy_clock_read, 121 }; 122 123 /* 124 * Boot time initialization which allows local_clock() to be utilized 125 * during early boot when clocksources are not available. local_clock() 126 * returns nanoseconds already so no conversion is required, hence mult=1 127 * and shift=0. When the first proper clocksource is installed then 128 * the fast time keepers are updated with the correct values. 129 */ 130 #define FAST_TK_INIT \ 131 { \ 132 .clock = &dummy_clock, \ 133 .mask = CLOCKSOURCE_MASK(64), \ 134 .mult = 1, \ 135 .shift = 0, \ 136 } 137 138 static struct tk_fast tk_fast_mono ____cacheline_aligned = { 139 .seq = SEQCNT_LATCH_ZERO(tk_fast_mono.seq), 140 .base[0] = FAST_TK_INIT, 141 .base[1] = FAST_TK_INIT, 142 }; 143 144 static struct tk_fast tk_fast_raw ____cacheline_aligned = { 145 .seq = SEQCNT_LATCH_ZERO(tk_fast_raw.seq), 146 .base[0] = FAST_TK_INIT, 147 .base[1] = FAST_TK_INIT, 148 }; 149 150 #ifdef CONFIG_POSIX_AUX_CLOCKS 151 static __init void tk_aux_setup(void); 152 static void tk_aux_update_clocksource(void); 153 static void tk_aux_advance(void); 154 #else 155 static inline void tk_aux_setup(void) { } 156 static inline void tk_aux_update_clocksource(void) { } 157 static inline void tk_aux_advance(void) { } 158 #endif 159 160 unsigned long timekeeper_lock_irqsave(void) 161 { 162 unsigned long flags; 163 164 raw_spin_lock_irqsave(&tk_core.lock, flags); 165 return flags; 166 } 167 168 void timekeeper_unlock_irqrestore(unsigned long flags) 169 { 170 raw_spin_unlock_irqrestore(&tk_core.lock, flags); 171 } 172 173 /* 174 * Multigrain timestamps require tracking the latest fine-grained timestamp 175 * that has been issued, and never returning a coarse-grained timestamp that is 176 * earlier than that value. 177 * 178 * mg_floor represents the latest fine-grained time that has been handed out as 179 * a file timestamp on the system. This is tracked as a monotonic ktime_t, and 180 * converted to a realtime clock value on an as-needed basis. 181 * 182 * Maintaining mg_floor ensures the multigrain interfaces never issue a 183 * timestamp earlier than one that has been previously issued. 184 * 185 * The exception to this rule is when there is a backward realtime clock jump. If 186 * such an event occurs, a timestamp can appear to be earlier than a previous one. 187 */ 188 static __cacheline_aligned_in_smp atomic64_t mg_floor; 189 190 static inline void tk_normalize_xtime(struct timekeeper *tk) 191 { 192 while (tk->tkr_mono.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_mono.shift)) { 193 tk->tkr_mono.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_mono.shift; 194 tk->xtime_sec++; 195 } 196 while (tk->tkr_raw.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_raw.shift)) { 197 tk->tkr_raw.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_raw.shift; 198 tk->raw_sec++; 199 } 200 } 201 202 static inline struct timespec64 tk_xtime(const struct timekeeper *tk) 203 { 204 struct timespec64 ts; 205 206 ts.tv_sec = tk->xtime_sec; 207 ts.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift); 208 return ts; 209 } 210 211 static inline struct timespec64 tk_xtime_coarse(const struct timekeeper *tk) 212 { 213 struct timespec64 ts; 214 215 ts.tv_sec = tk->xtime_sec; 216 ts.tv_nsec = tk->coarse_nsec; 217 return ts; 218 } 219 220 /* 221 * Update the nanoseconds part for the coarse time keepers. They can't rely 222 * on xtime_nsec because xtime_nsec could be adjusted by a small negative 223 * amount when the multiplication factor of the clock is adjusted, which 224 * could cause the coarse clocks to go slightly backwards. See 225 * timekeeping_apply_adjustment(). Thus we keep a separate copy for the coarse 226 * clockids which only is updated when the clock has been set or we have 227 * accumulated time. 228 */ 229 static inline void tk_update_coarse_nsecs(struct timekeeper *tk) 230 { 231 tk->coarse_nsec = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift; 232 } 233 234 static void tk_set_xtime(struct timekeeper *tk, const struct timespec64 *ts) 235 { 236 tk->xtime_sec = ts->tv_sec; 237 tk->tkr_mono.xtime_nsec = (u64)ts->tv_nsec << tk->tkr_mono.shift; 238 tk_update_coarse_nsecs(tk); 239 } 240 241 static void tk_xtime_add(struct timekeeper *tk, const struct timespec64 *ts) 242 { 243 tk->xtime_sec += ts->tv_sec; 244 tk->tkr_mono.xtime_nsec += (u64)ts->tv_nsec << tk->tkr_mono.shift; 245 tk_normalize_xtime(tk); 246 tk_update_coarse_nsecs(tk); 247 } 248 249 static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm) 250 { 251 struct timespec64 tmp; 252 253 /* 254 * Verify consistency of: offset_real = -wall_to_monotonic 255 * before modifying anything 256 */ 257 set_normalized_timespec64(&tmp, -tk->wall_to_monotonic.tv_sec, 258 -tk->wall_to_monotonic.tv_nsec); 259 WARN_ON_ONCE(tk->offs_real != timespec64_to_ktime(tmp)); 260 tk->wall_to_monotonic = wtm; 261 set_normalized_timespec64(&tmp, -wtm.tv_sec, -wtm.tv_nsec); 262 /* Paired with READ_ONCE() in ktime_mono_to_any() */ 263 WRITE_ONCE(tk->offs_real, timespec64_to_ktime(tmp)); 264 WRITE_ONCE(tk->offs_tai, ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0))); 265 } 266 267 static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta) 268 { 269 /* Paired with READ_ONCE() in ktime_mono_to_any() */ 270 WRITE_ONCE(tk->offs_boot, ktime_add(tk->offs_boot, delta)); 271 /* 272 * Timespec representation for VDSO update to avoid 64bit division 273 * on every update. 274 */ 275 tk->monotonic_to_boot = ktime_to_timespec64(tk->offs_boot); 276 } 277 278 /* 279 * tk_clock_read - atomic clocksource read() helper 280 * 281 * This helper is necessary to use in the read paths because, while the 282 * seqcount ensures we don't return a bad value while structures are updated, 283 * it doesn't protect from potential crashes. There is the possibility that 284 * the tkr's clocksource may change between the read reference, and the 285 * clock reference passed to the read function. This can cause crashes if 286 * the wrong clocksource is passed to the wrong read function. 287 * This isn't necessary to use when holding the tk_core.lock or doing 288 * a read of the fast-timekeeper tkrs (which is protected by its own locking 289 * and update logic). 290 */ 291 static inline u64 tk_clock_read(const struct tk_read_base *tkr) 292 { 293 struct clocksource *clock = READ_ONCE(tkr->clock); 294 295 return clock->read(clock); 296 } 297 298 /** 299 * tk_setup_internals - Set up internals to use clocksource clock. 300 * 301 * @tk: The target timekeeper to setup. 302 * @clock: Pointer to clocksource. 303 * 304 * Calculates a fixed cycle/nsec interval for a given clocksource/adjustment 305 * pair and interval request. 306 * 307 * Unless you're the timekeeping code, you should not be using this! 308 */ 309 static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock) 310 { 311 u64 interval; 312 u64 tmp, ntpinterval; 313 struct clocksource *old_clock; 314 315 ++tk->cs_was_changed_seq; 316 old_clock = tk->tkr_mono.clock; 317 tk->tkr_mono.clock = clock; 318 tk->tkr_mono.mask = clock->mask; 319 tk->tkr_mono.cycle_last = tk_clock_read(&tk->tkr_mono); 320 321 tk->tkr_raw.clock = clock; 322 tk->tkr_raw.mask = clock->mask; 323 tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last; 324 325 /* Do the ns -> cycle conversion first, using original mult */ 326 tmp = NTP_INTERVAL_LENGTH; 327 tmp <<= clock->shift; 328 ntpinterval = tmp; 329 tmp += clock->mult/2; 330 do_div(tmp, clock->mult); 331 if (tmp == 0) 332 tmp = 1; 333 334 interval = (u64) tmp; 335 tk->cycle_interval = interval; 336 337 /* Go back from cycles -> shifted ns */ 338 tk->xtime_interval = interval * clock->mult; 339 tk->xtime_remainder = ntpinterval - tk->xtime_interval; 340 tk->raw_interval = interval * clock->mult; 341 342 /* if changing clocks, convert xtime_nsec shift units */ 343 if (old_clock) { 344 int shift_change = clock->shift - old_clock->shift; 345 if (shift_change < 0) { 346 tk->tkr_mono.xtime_nsec >>= -shift_change; 347 tk->tkr_raw.xtime_nsec >>= -shift_change; 348 } else { 349 tk->tkr_mono.xtime_nsec <<= shift_change; 350 tk->tkr_raw.xtime_nsec <<= shift_change; 351 } 352 } 353 354 tk->tkr_mono.shift = clock->shift; 355 tk->tkr_raw.shift = clock->shift; 356 357 tk->ntp_error = 0; 358 tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift; 359 tk->ntp_tick = ntpinterval << tk->ntp_error_shift; 360 361 /* 362 * The timekeeper keeps its own mult values for the currently 363 * active clocksource. These value will be adjusted via NTP 364 * to counteract clock drifting. 365 */ 366 tk->tkr_mono.mult = clock->mult; 367 tk->tkr_raw.mult = clock->mult; 368 tk->ntp_err_mult = 0; 369 tk->skip_second_overflow = 0; 370 } 371 372 /* Timekeeper helper functions. */ 373 static noinline u64 delta_to_ns_safe(const struct tk_read_base *tkr, u64 delta) 374 { 375 return mul_u64_u32_add_u64_shr(delta, tkr->mult, tkr->xtime_nsec, tkr->shift); 376 } 377 378 static inline u64 timekeeping_cycles_to_ns(const struct tk_read_base *tkr, u64 cycles) 379 { 380 /* Calculate the delta since the last update_wall_time() */ 381 u64 mask = tkr->mask, delta = (cycles - tkr->cycle_last) & mask; 382 383 /* 384 * This detects both negative motion and the case where the delta 385 * overflows the multiplication with tkr->mult. 386 */ 387 if (unlikely(delta > tkr->clock->max_cycles)) { 388 /* 389 * Handle clocksource inconsistency between CPUs to prevent 390 * time from going backwards by checking for the MSB of the 391 * mask being set in the delta. 392 */ 393 if (delta & ~(mask >> 1)) 394 return tkr->xtime_nsec >> tkr->shift; 395 396 return delta_to_ns_safe(tkr, delta); 397 } 398 399 return ((delta * tkr->mult) + tkr->xtime_nsec) >> tkr->shift; 400 } 401 402 static __always_inline u64 timekeeping_get_ns(const struct tk_read_base *tkr) 403 { 404 return timekeeping_cycles_to_ns(tkr, tk_clock_read(tkr)); 405 } 406 407 /** 408 * update_fast_timekeeper - Update the fast and NMI safe monotonic timekeeper. 409 * @tkr: Timekeeping readout base from which we take the update 410 * @tkf: Pointer to NMI safe timekeeper 411 * 412 * We want to use this from any context including NMI and tracing / 413 * instrumenting the timekeeping code itself. 414 * 415 * Employ the latch technique; see @write_seqcount_latch. 416 * 417 * So if a NMI hits the update of base[0] then it will use base[1] 418 * which is still consistent. In the worst case this can result is a 419 * slightly wrong timestamp (a few nanoseconds). See 420 * @ktime_get_mono_fast_ns. 421 */ 422 static void update_fast_timekeeper(const struct tk_read_base *tkr, 423 struct tk_fast *tkf) 424 { 425 struct tk_read_base *base = tkf->base; 426 427 /* Force readers off to base[1] */ 428 write_seqcount_latch_begin(&tkf->seq); 429 430 /* Update base[0] */ 431 memcpy(base, tkr, sizeof(*base)); 432 433 /* Force readers back to base[0] */ 434 write_seqcount_latch(&tkf->seq); 435 436 /* Update base[1] */ 437 memcpy(base + 1, base, sizeof(*base)); 438 439 write_seqcount_latch_end(&tkf->seq); 440 } 441 442 static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf) 443 { 444 struct tk_read_base *tkr; 445 unsigned int seq; 446 u64 now; 447 448 do { 449 seq = read_seqcount_latch(&tkf->seq); 450 tkr = tkf->base + (seq & 0x01); 451 now = ktime_to_ns(tkr->base); 452 now += timekeeping_get_ns(tkr); 453 } while (read_seqcount_latch_retry(&tkf->seq, seq)); 454 455 return now; 456 } 457 458 /** 459 * ktime_get_mono_fast_ns - Fast NMI safe access to clock monotonic 460 * 461 * This timestamp is not guaranteed to be monotonic across an update. 462 * The timestamp is calculated by: 463 * 464 * now = base_mono + clock_delta * slope 465 * 466 * So if the update lowers the slope, readers who are forced to the 467 * not yet updated second array are still using the old steeper slope. 468 * 469 * tmono 470 * ^ 471 * | o n 472 * | o n 473 * | u 474 * | o 475 * |o 476 * |12345678---> reader order 477 * 478 * o = old slope 479 * u = update 480 * n = new slope 481 * 482 * So reader 6 will observe time going backwards versus reader 5. 483 * 484 * While other CPUs are likely to be able to observe that, the only way 485 * for a CPU local observation is when an NMI hits in the middle of 486 * the update. Timestamps taken from that NMI context might be ahead 487 * of the following timestamps. Callers need to be aware of that and 488 * deal with it. 489 */ 490 u64 notrace ktime_get_mono_fast_ns(void) 491 { 492 return __ktime_get_fast_ns(&tk_fast_mono); 493 } 494 EXPORT_SYMBOL_GPL(ktime_get_mono_fast_ns); 495 496 /** 497 * ktime_get_raw_fast_ns - Fast NMI safe access to clock monotonic raw 498 * 499 * Contrary to ktime_get_mono_fast_ns() this is always correct because the 500 * conversion factor is not affected by NTP/PTP correction. 501 */ 502 u64 notrace ktime_get_raw_fast_ns(void) 503 { 504 return __ktime_get_fast_ns(&tk_fast_raw); 505 } 506 EXPORT_SYMBOL_GPL(ktime_get_raw_fast_ns); 507 508 /** 509 * ktime_get_boot_fast_ns - NMI safe and fast access to boot clock. 510 * 511 * To keep it NMI safe since we're accessing from tracing, we're not using a 512 * separate timekeeper with updates to monotonic clock and boot offset 513 * protected with seqcounts. This has the following minor side effects: 514 * 515 * (1) Its possible that a timestamp be taken after the boot offset is updated 516 * but before the timekeeper is updated. If this happens, the new boot offset 517 * is added to the old timekeeping making the clock appear to update slightly 518 * earlier: 519 * CPU 0 CPU 1 520 * timekeeping_inject_sleeptime64() 521 * __timekeeping_inject_sleeptime(tk, delta); 522 * timestamp(); 523 * timekeeping_update_staged(tkd, TK_CLEAR_NTP...); 524 * 525 * (2) On 32-bit systems, the 64-bit boot offset (tk->offs_boot) may be 526 * partially updated. Since the tk->offs_boot update is a rare event, this 527 * should be a rare occurrence which postprocessing should be able to handle. 528 * 529 * The caveats vs. timestamp ordering as documented for ktime_get_mono_fast_ns() 530 * apply as well. 531 */ 532 u64 notrace ktime_get_boot_fast_ns(void) 533 { 534 struct timekeeper *tk = &tk_core.timekeeper; 535 536 return (ktime_get_mono_fast_ns() + ktime_to_ns(data_race(tk->offs_boot))); 537 } 538 EXPORT_SYMBOL_GPL(ktime_get_boot_fast_ns); 539 540 /** 541 * ktime_get_tai_fast_ns - NMI safe and fast access to tai clock. 542 * 543 * The same limitations as described for ktime_get_boot_fast_ns() apply. The 544 * mono time and the TAI offset are not read atomically which may yield wrong 545 * readouts. However, an update of the TAI offset is an rare event e.g., caused 546 * by settime or adjtimex with an offset. The user of this function has to deal 547 * with the possibility of wrong timestamps in post processing. 548 */ 549 u64 notrace ktime_get_tai_fast_ns(void) 550 { 551 struct timekeeper *tk = &tk_core.timekeeper; 552 553 return (ktime_get_mono_fast_ns() + ktime_to_ns(data_race(tk->offs_tai))); 554 } 555 EXPORT_SYMBOL_GPL(ktime_get_tai_fast_ns); 556 557 /** 558 * ktime_get_real_fast_ns: - NMI safe and fast access to clock realtime. 559 * 560 * See ktime_get_mono_fast_ns() for documentation of the time stamp ordering. 561 */ 562 u64 ktime_get_real_fast_ns(void) 563 { 564 struct tk_fast *tkf = &tk_fast_mono; 565 struct tk_read_base *tkr; 566 u64 baser, delta; 567 unsigned int seq; 568 569 do { 570 seq = raw_read_seqcount_latch(&tkf->seq); 571 tkr = tkf->base + (seq & 0x01); 572 baser = ktime_to_ns(tkr->base_real); 573 delta = timekeeping_get_ns(tkr); 574 } while (raw_read_seqcount_latch_retry(&tkf->seq, seq)); 575 576 return baser + delta; 577 } 578 EXPORT_SYMBOL_GPL(ktime_get_real_fast_ns); 579 580 /** 581 * halt_fast_timekeeper - Prevent fast timekeeper from accessing clocksource. 582 * @tk: Timekeeper to snapshot. 583 * 584 * It generally is unsafe to access the clocksource after timekeeping has been 585 * suspended, so take a snapshot of the readout base of @tk and use it as the 586 * fast timekeeper's readout base while suspended. It will return the same 587 * number of cycles every time until timekeeping is resumed at which time the 588 * proper readout base for the fast timekeeper will be restored automatically. 589 */ 590 static void halt_fast_timekeeper(const struct timekeeper *tk) 591 { 592 static struct tk_read_base tkr_dummy; 593 const struct tk_read_base *tkr = &tk->tkr_mono; 594 595 memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy)); 596 cycles_at_suspend = tk_clock_read(tkr); 597 tkr_dummy.clock = &dummy_clock; 598 tkr_dummy.base_real = tkr->base + tk->offs_real; 599 update_fast_timekeeper(&tkr_dummy, &tk_fast_mono); 600 601 tkr = &tk->tkr_raw; 602 memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy)); 603 tkr_dummy.clock = &dummy_clock; 604 update_fast_timekeeper(&tkr_dummy, &tk_fast_raw); 605 } 606 607 static RAW_NOTIFIER_HEAD(pvclock_gtod_chain); 608 609 static void update_pvclock_gtod(struct timekeeper *tk, bool was_set) 610 { 611 raw_notifier_call_chain(&pvclock_gtod_chain, was_set, tk); 612 } 613 614 /** 615 * pvclock_gtod_register_notifier - register a pvclock timedata update listener 616 * @nb: Pointer to the notifier block to register 617 */ 618 int pvclock_gtod_register_notifier(struct notifier_block *nb) 619 { 620 struct timekeeper *tk = &tk_core.timekeeper; 621 int ret; 622 623 guard(raw_spinlock_irqsave)(&tk_core.lock); 624 ret = raw_notifier_chain_register(&pvclock_gtod_chain, nb); 625 update_pvclock_gtod(tk, true); 626 627 return ret; 628 } 629 EXPORT_SYMBOL_GPL(pvclock_gtod_register_notifier); 630 631 /** 632 * pvclock_gtod_unregister_notifier - unregister a pvclock 633 * timedata update listener 634 * @nb: Pointer to the notifier block to unregister 635 */ 636 int pvclock_gtod_unregister_notifier(struct notifier_block *nb) 637 { 638 guard(raw_spinlock_irqsave)(&tk_core.lock); 639 return raw_notifier_chain_unregister(&pvclock_gtod_chain, nb); 640 } 641 EXPORT_SYMBOL_GPL(pvclock_gtod_unregister_notifier); 642 643 /* 644 * tk_update_leap_state - helper to update the next_leap_ktime 645 */ 646 static inline void tk_update_leap_state(struct timekeeper *tk) 647 { 648 tk->next_leap_ktime = ntp_get_next_leap(tk->id); 649 if (tk->next_leap_ktime != KTIME_MAX) 650 /* Convert to monotonic time */ 651 tk->next_leap_ktime = ktime_sub(tk->next_leap_ktime, tk->offs_real); 652 } 653 654 /* 655 * Leap state update for both shadow and the real timekeeper 656 * Separate to spare a full memcpy() of the timekeeper. 657 */ 658 static void tk_update_leap_state_all(struct tk_data *tkd) 659 { 660 write_seqcount_begin(&tkd->seq); 661 tk_update_leap_state(&tkd->shadow_timekeeper); 662 tkd->timekeeper.next_leap_ktime = tkd->shadow_timekeeper.next_leap_ktime; 663 write_seqcount_end(&tkd->seq); 664 } 665 666 /* 667 * Update the ktime_t based scalar nsec members of the timekeeper 668 */ 669 static inline void tk_update_ktime_data(struct timekeeper *tk) 670 { 671 u64 seconds; 672 u32 nsec; 673 674 /* 675 * The xtime based monotonic readout is: 676 * nsec = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec + now(); 677 * The ktime based monotonic readout is: 678 * nsec = base_mono + now(); 679 * ==> base_mono = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec 680 */ 681 seconds = (u64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec); 682 nsec = (u32) tk->wall_to_monotonic.tv_nsec; 683 tk->tkr_mono.base = ns_to_ktime(seconds * NSEC_PER_SEC + nsec); 684 685 /* 686 * The sum of the nanoseconds portions of xtime and 687 * wall_to_monotonic can be greater/equal one second. Take 688 * this into account before updating tk->ktime_sec. 689 */ 690 nsec += (u32)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift); 691 if (nsec >= NSEC_PER_SEC) 692 seconds++; 693 tk->ktime_sec = seconds; 694 695 /* Update the monotonic raw base */ 696 tk->tkr_raw.base = ns_to_ktime(tk->raw_sec * NSEC_PER_SEC); 697 } 698 699 /* 700 * Restore the shadow timekeeper from the real timekeeper. 701 */ 702 static void timekeeping_restore_shadow(struct tk_data *tkd) 703 { 704 lockdep_assert_held(&tkd->lock); 705 memcpy(&tkd->shadow_timekeeper, &tkd->timekeeper, sizeof(tkd->timekeeper)); 706 } 707 708 static void timekeeping_update_from_shadow(struct tk_data *tkd, unsigned int action) 709 { 710 struct timekeeper *tk = &tkd->shadow_timekeeper; 711 712 lockdep_assert_held(&tkd->lock); 713 714 /* 715 * Block out readers before running the updates below because that 716 * updates VDSO and other time related infrastructure. Not blocking 717 * the readers might let a reader see time going backwards when 718 * reading from the VDSO after the VDSO update and then reading in 719 * the kernel from the timekeeper before that got updated. 720 */ 721 write_seqcount_begin(&tkd->seq); 722 723 if (action & TK_CLEAR_NTP) { 724 tk->ntp_error = 0; 725 ntp_clear(tk->id); 726 } 727 728 tk_update_leap_state(tk); 729 tk_update_ktime_data(tk); 730 tk->tkr_mono.base_real = tk->tkr_mono.base + tk->offs_real; 731 732 if (tk->id == TIMEKEEPER_CORE) { 733 update_vsyscall(tk); 734 update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET); 735 736 update_fast_timekeeper(&tk->tkr_mono, &tk_fast_mono); 737 update_fast_timekeeper(&tk->tkr_raw, &tk_fast_raw); 738 } else if (tk_is_aux(tk)) { 739 vdso_time_update_aux(tk); 740 } 741 742 if (action & TK_CLOCK_WAS_SET) 743 tk->clock_was_set_seq++; 744 745 /* 746 * Update the real timekeeper. 747 * 748 * We could avoid this memcpy() by switching pointers, but that has 749 * the downside that the reader side does not longer benefit from 750 * the cacheline optimized data layout of the timekeeper and requires 751 * another indirection. 752 */ 753 memcpy(&tkd->timekeeper, tk, sizeof(*tk)); 754 write_seqcount_end(&tkd->seq); 755 } 756 757 /** 758 * timekeeping_forward_now - update clock to the current time 759 * @tk: Pointer to the timekeeper to update 760 * 761 * Forward the current clock to update its state since the last call to 762 * update_wall_time(). This is useful before significant clock changes, 763 * as it avoids having to deal with this time offset explicitly. 764 */ 765 static void timekeeping_forward_now(struct timekeeper *tk) 766 { 767 u64 cycle_now, delta; 768 769 cycle_now = tk_clock_read(&tk->tkr_mono); 770 delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last, tk->tkr_mono.mask, 771 tk->tkr_mono.clock->max_raw_delta); 772 tk->tkr_mono.cycle_last = cycle_now; 773 tk->tkr_raw.cycle_last = cycle_now; 774 775 while (delta > 0) { 776 u64 max = tk->tkr_mono.clock->max_cycles; 777 u64 incr = delta < max ? delta : max; 778 779 tk->tkr_mono.xtime_nsec += incr * tk->tkr_mono.mult; 780 tk->tkr_raw.xtime_nsec += incr * tk->tkr_raw.mult; 781 tk_normalize_xtime(tk); 782 delta -= incr; 783 } 784 tk_update_coarse_nsecs(tk); 785 } 786 787 /** 788 * ktime_get_real_ts64 - Returns the time of day in a timespec64. 789 * @ts: pointer to the timespec to be set 790 * 791 * Returns the time of day in a timespec64 (WARN if suspended). 792 */ 793 void ktime_get_real_ts64(struct timespec64 *ts) 794 { 795 struct timekeeper *tk = &tk_core.timekeeper; 796 unsigned int seq; 797 u64 nsecs; 798 799 WARN_ON(timekeeping_suspended); 800 801 do { 802 seq = read_seqcount_begin(&tk_core.seq); 803 804 ts->tv_sec = tk->xtime_sec; 805 nsecs = timekeeping_get_ns(&tk->tkr_mono); 806 807 } while (read_seqcount_retry(&tk_core.seq, seq)); 808 809 ts->tv_nsec = 0; 810 timespec64_add_ns(ts, nsecs); 811 } 812 EXPORT_SYMBOL(ktime_get_real_ts64); 813 814 ktime_t ktime_get(void) 815 { 816 struct timekeeper *tk = &tk_core.timekeeper; 817 unsigned int seq; 818 ktime_t base; 819 u64 nsecs; 820 821 WARN_ON(timekeeping_suspended); 822 823 do { 824 seq = read_seqcount_begin(&tk_core.seq); 825 base = tk->tkr_mono.base; 826 nsecs = timekeeping_get_ns(&tk->tkr_mono); 827 828 } while (read_seqcount_retry(&tk_core.seq, seq)); 829 830 return ktime_add_ns(base, nsecs); 831 } 832 EXPORT_SYMBOL_GPL(ktime_get); 833 834 u32 ktime_get_resolution_ns(void) 835 { 836 struct timekeeper *tk = &tk_core.timekeeper; 837 unsigned int seq; 838 u32 nsecs; 839 840 WARN_ON(timekeeping_suspended); 841 842 do { 843 seq = read_seqcount_begin(&tk_core.seq); 844 nsecs = tk->tkr_mono.mult >> tk->tkr_mono.shift; 845 } while (read_seqcount_retry(&tk_core.seq, seq)); 846 847 return nsecs; 848 } 849 EXPORT_SYMBOL_GPL(ktime_get_resolution_ns); 850 851 static ktime_t *offsets[TK_OFFS_MAX] = { 852 [TK_OFFS_REAL] = &tk_core.timekeeper.offs_real, 853 [TK_OFFS_BOOT] = &tk_core.timekeeper.offs_boot, 854 [TK_OFFS_TAI] = &tk_core.timekeeper.offs_tai, 855 }; 856 857 ktime_t ktime_get_with_offset(enum tk_offsets offs) 858 { 859 struct timekeeper *tk = &tk_core.timekeeper; 860 unsigned int seq; 861 ktime_t base, *offset = offsets[offs]; 862 u64 nsecs; 863 864 WARN_ON(timekeeping_suspended); 865 866 do { 867 seq = read_seqcount_begin(&tk_core.seq); 868 base = ktime_add(tk->tkr_mono.base, *offset); 869 nsecs = timekeeping_get_ns(&tk->tkr_mono); 870 871 } while (read_seqcount_retry(&tk_core.seq, seq)); 872 873 return ktime_add_ns(base, nsecs); 874 875 } 876 EXPORT_SYMBOL_GPL(ktime_get_with_offset); 877 878 ktime_t ktime_get_coarse_with_offset(enum tk_offsets offs) 879 { 880 struct timekeeper *tk = &tk_core.timekeeper; 881 ktime_t base, *offset = offsets[offs]; 882 unsigned int seq; 883 u64 nsecs; 884 885 WARN_ON(timekeeping_suspended); 886 887 do { 888 seq = read_seqcount_begin(&tk_core.seq); 889 base = ktime_add(tk->tkr_mono.base, *offset); 890 nsecs = tk->coarse_nsec; 891 892 } while (read_seqcount_retry(&tk_core.seq, seq)); 893 894 return ktime_add_ns(base, nsecs); 895 } 896 EXPORT_SYMBOL_GPL(ktime_get_coarse_with_offset); 897 898 /** 899 * ktime_mono_to_any() - convert monotonic time to any other time 900 * @tmono: time to convert. 901 * @offs: which offset to use 902 */ 903 ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs) 904 { 905 ktime_t *offset = offsets[offs]; 906 unsigned int seq; 907 ktime_t tconv; 908 909 if (IS_ENABLED(CONFIG_64BIT)) { 910 /* 911 * Paired with WRITE_ONCE()s in tk_set_wall_to_mono() and 912 * tk_update_sleep_time(). 913 */ 914 return ktime_add(tmono, READ_ONCE(*offset)); 915 } 916 917 do { 918 seq = read_seqcount_begin(&tk_core.seq); 919 tconv = ktime_add(tmono, *offset); 920 } while (read_seqcount_retry(&tk_core.seq, seq)); 921 922 return tconv; 923 } 924 EXPORT_SYMBOL_GPL(ktime_mono_to_any); 925 926 /** 927 * ktime_get_raw - Returns the raw monotonic time in ktime_t format 928 */ 929 ktime_t ktime_get_raw(void) 930 { 931 struct timekeeper *tk = &tk_core.timekeeper; 932 unsigned int seq; 933 ktime_t base; 934 u64 nsecs; 935 936 do { 937 seq = read_seqcount_begin(&tk_core.seq); 938 base = tk->tkr_raw.base; 939 nsecs = timekeeping_get_ns(&tk->tkr_raw); 940 941 } while (read_seqcount_retry(&tk_core.seq, seq)); 942 943 return ktime_add_ns(base, nsecs); 944 } 945 EXPORT_SYMBOL_GPL(ktime_get_raw); 946 947 /** 948 * ktime_get_ts64 - get the monotonic clock in timespec64 format 949 * @ts: pointer to timespec variable 950 * 951 * The function calculates the monotonic clock from the realtime 952 * clock and the wall_to_monotonic offset and stores the result 953 * in normalized timespec64 format in the variable pointed to by @ts. 954 */ 955 void ktime_get_ts64(struct timespec64 *ts) 956 { 957 struct timekeeper *tk = &tk_core.timekeeper; 958 struct timespec64 tomono; 959 unsigned int seq; 960 u64 nsec; 961 962 WARN_ON(timekeeping_suspended); 963 964 do { 965 seq = read_seqcount_begin(&tk_core.seq); 966 ts->tv_sec = tk->xtime_sec; 967 nsec = timekeeping_get_ns(&tk->tkr_mono); 968 tomono = tk->wall_to_monotonic; 969 970 } while (read_seqcount_retry(&tk_core.seq, seq)); 971 972 ts->tv_sec += tomono.tv_sec; 973 ts->tv_nsec = 0; 974 timespec64_add_ns(ts, nsec + tomono.tv_nsec); 975 } 976 EXPORT_SYMBOL_GPL(ktime_get_ts64); 977 978 /** 979 * ktime_get_seconds - Get the seconds portion of CLOCK_MONOTONIC 980 * 981 * Returns the seconds portion of CLOCK_MONOTONIC with a single non 982 * serialized read. tk->ktime_sec is of type 'unsigned long' so this 983 * works on both 32 and 64 bit systems. On 32 bit systems the readout 984 * covers ~136 years of uptime which should be enough to prevent 985 * premature wrap arounds. 986 */ 987 time64_t ktime_get_seconds(void) 988 { 989 struct timekeeper *tk = &tk_core.timekeeper; 990 991 WARN_ON(timekeeping_suspended); 992 return tk->ktime_sec; 993 } 994 EXPORT_SYMBOL_GPL(ktime_get_seconds); 995 996 /** 997 * ktime_get_real_seconds - Get the seconds portion of CLOCK_REALTIME 998 * 999 * Returns the wall clock seconds since 1970. 1000 * 1001 * For 64bit systems the fast access to tk->xtime_sec is preserved. On 1002 * 32bit systems the access must be protected with the sequence 1003 * counter to provide "atomic" access to the 64bit tk->xtime_sec 1004 * value. 1005 */ 1006 time64_t ktime_get_real_seconds(void) 1007 { 1008 struct timekeeper *tk = &tk_core.timekeeper; 1009 time64_t seconds; 1010 unsigned int seq; 1011 1012 if (IS_ENABLED(CONFIG_64BIT)) 1013 return tk->xtime_sec; 1014 1015 do { 1016 seq = read_seqcount_begin(&tk_core.seq); 1017 seconds = tk->xtime_sec; 1018 1019 } while (read_seqcount_retry(&tk_core.seq, seq)); 1020 1021 return seconds; 1022 } 1023 EXPORT_SYMBOL_GPL(ktime_get_real_seconds); 1024 1025 /** 1026 * __ktime_get_real_seconds - Unprotected access to CLOCK_REALTIME seconds 1027 * 1028 * The same as ktime_get_real_seconds() but without the sequence counter 1029 * protection. This function is used in restricted contexts like the x86 MCE 1030 * handler and in KGDB. It's unprotected on 32-bit vs. concurrent half 1031 * completed modification and only to be used for such critical contexts. 1032 * 1033 * Returns: Racy snapshot of the CLOCK_REALTIME seconds value 1034 */ 1035 noinstr time64_t __ktime_get_real_seconds(void) 1036 { 1037 struct timekeeper *tk = &tk_core.timekeeper; 1038 1039 return tk->xtime_sec; 1040 } 1041 1042 /** 1043 * ktime_get_snapshot - snapshots the realtime/monotonic raw clocks with counter 1044 * @systime_snapshot: pointer to struct receiving the system time snapshot 1045 */ 1046 void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot) 1047 { 1048 struct timekeeper *tk = &tk_core.timekeeper; 1049 unsigned int seq; 1050 ktime_t base_raw; 1051 ktime_t base_real; 1052 ktime_t base_boot; 1053 u64 nsec_raw; 1054 u64 nsec_real; 1055 u64 now; 1056 1057 WARN_ON_ONCE(timekeeping_suspended); 1058 1059 do { 1060 seq = read_seqcount_begin(&tk_core.seq); 1061 now = tk_clock_read(&tk->tkr_mono); 1062 systime_snapshot->cs_id = tk->tkr_mono.clock->id; 1063 systime_snapshot->cs_was_changed_seq = tk->cs_was_changed_seq; 1064 systime_snapshot->clock_was_set_seq = tk->clock_was_set_seq; 1065 base_real = ktime_add(tk->tkr_mono.base, 1066 tk_core.timekeeper.offs_real); 1067 base_boot = ktime_add(tk->tkr_mono.base, 1068 tk_core.timekeeper.offs_boot); 1069 base_raw = tk->tkr_raw.base; 1070 nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, now); 1071 nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, now); 1072 } while (read_seqcount_retry(&tk_core.seq, seq)); 1073 1074 systime_snapshot->cycles = now; 1075 systime_snapshot->real = ktime_add_ns(base_real, nsec_real); 1076 systime_snapshot->boot = ktime_add_ns(base_boot, nsec_real); 1077 systime_snapshot->raw = ktime_add_ns(base_raw, nsec_raw); 1078 } 1079 EXPORT_SYMBOL_GPL(ktime_get_snapshot); 1080 1081 /* Scale base by mult/div checking for overflow */ 1082 static int scale64_check_overflow(u64 mult, u64 div, u64 *base) 1083 { 1084 u64 tmp, rem; 1085 1086 tmp = div64_u64_rem(*base, div, &rem); 1087 1088 if (((int)sizeof(u64)*8 - fls64(mult) < fls64(tmp)) || 1089 ((int)sizeof(u64)*8 - fls64(mult) < fls64(rem))) 1090 return -EOVERFLOW; 1091 tmp *= mult; 1092 1093 rem = div64_u64(rem * mult, div); 1094 *base = tmp + rem; 1095 return 0; 1096 } 1097 1098 /** 1099 * adjust_historical_crosststamp - adjust crosstimestamp previous to current interval 1100 * @history: Snapshot representing start of history 1101 * @partial_history_cycles: Cycle offset into history (fractional part) 1102 * @total_history_cycles: Total history length in cycles 1103 * @discontinuity: True indicates clock was set on history period 1104 * @ts: Cross timestamp that should be adjusted using 1105 * partial/total ratio 1106 * 1107 * Helper function used by get_device_system_crosststamp() to correct the 1108 * crosstimestamp corresponding to the start of the current interval to the 1109 * system counter value (timestamp point) provided by the driver. The 1110 * total_history_* quantities are the total history starting at the provided 1111 * reference point and ending at the start of the current interval. The cycle 1112 * count between the driver timestamp point and the start of the current 1113 * interval is partial_history_cycles. 1114 */ 1115 static int adjust_historical_crosststamp(struct system_time_snapshot *history, 1116 u64 partial_history_cycles, 1117 u64 total_history_cycles, 1118 bool discontinuity, 1119 struct system_device_crosststamp *ts) 1120 { 1121 struct timekeeper *tk = &tk_core.timekeeper; 1122 u64 corr_raw, corr_real; 1123 bool interp_forward; 1124 int ret; 1125 1126 if (total_history_cycles == 0 || partial_history_cycles == 0) 1127 return 0; 1128 1129 /* Interpolate shortest distance from beginning or end of history */ 1130 interp_forward = partial_history_cycles > total_history_cycles / 2; 1131 partial_history_cycles = interp_forward ? 1132 total_history_cycles - partial_history_cycles : 1133 partial_history_cycles; 1134 1135 /* 1136 * Scale the monotonic raw time delta by: 1137 * partial_history_cycles / total_history_cycles 1138 */ 1139 corr_raw = (u64)ktime_to_ns( 1140 ktime_sub(ts->sys_monoraw, history->raw)); 1141 ret = scale64_check_overflow(partial_history_cycles, 1142 total_history_cycles, &corr_raw); 1143 if (ret) 1144 return ret; 1145 1146 /* 1147 * If there is a discontinuity in the history, scale monotonic raw 1148 * correction by: 1149 * mult(real)/mult(raw) yielding the realtime correction 1150 * Otherwise, calculate the realtime correction similar to monotonic 1151 * raw calculation 1152 */ 1153 if (discontinuity) { 1154 corr_real = mul_u64_u32_div 1155 (corr_raw, tk->tkr_mono.mult, tk->tkr_raw.mult); 1156 } else { 1157 corr_real = (u64)ktime_to_ns( 1158 ktime_sub(ts->sys_realtime, history->real)); 1159 ret = scale64_check_overflow(partial_history_cycles, 1160 total_history_cycles, &corr_real); 1161 if (ret) 1162 return ret; 1163 } 1164 1165 /* Fixup monotonic raw and real time time values */ 1166 if (interp_forward) { 1167 ts->sys_monoraw = ktime_add_ns(history->raw, corr_raw); 1168 ts->sys_realtime = ktime_add_ns(history->real, corr_real); 1169 } else { 1170 ts->sys_monoraw = ktime_sub_ns(ts->sys_monoraw, corr_raw); 1171 ts->sys_realtime = ktime_sub_ns(ts->sys_realtime, corr_real); 1172 } 1173 1174 return 0; 1175 } 1176 1177 /* 1178 * timestamp_in_interval - true if ts is chronologically in [start, end] 1179 * 1180 * True if ts occurs chronologically at or after start, and before or at end. 1181 */ 1182 static bool timestamp_in_interval(u64 start, u64 end, u64 ts) 1183 { 1184 if (ts >= start && ts <= end) 1185 return true; 1186 if (start > end && (ts >= start || ts <= end)) 1187 return true; 1188 return false; 1189 } 1190 1191 static bool convert_clock(u64 *val, u32 numerator, u32 denominator) 1192 { 1193 u64 rem, res; 1194 1195 if (!numerator || !denominator) 1196 return false; 1197 1198 res = div64_u64_rem(*val, denominator, &rem) * numerator; 1199 *val = res + div_u64(rem * numerator, denominator); 1200 return true; 1201 } 1202 1203 static bool convert_base_to_cs(struct system_counterval_t *scv) 1204 { 1205 struct clocksource *cs = tk_core.timekeeper.tkr_mono.clock; 1206 struct clocksource_base *base; 1207 u32 num, den; 1208 1209 /* The timestamp was taken from the time keeper clock source */ 1210 if (cs->id == scv->cs_id) 1211 return true; 1212 1213 /* 1214 * Check whether cs_id matches the base clock. Prevent the compiler from 1215 * re-evaluating @base as the clocksource might change concurrently. 1216 */ 1217 base = READ_ONCE(cs->base); 1218 if (!base || base->id != scv->cs_id) 1219 return false; 1220 1221 num = scv->use_nsecs ? cs->freq_khz : base->numerator; 1222 den = scv->use_nsecs ? USEC_PER_SEC : base->denominator; 1223 1224 if (!convert_clock(&scv->cycles, num, den)) 1225 return false; 1226 1227 scv->cycles += base->offset; 1228 return true; 1229 } 1230 1231 static bool convert_cs_to_base(u64 *cycles, enum clocksource_ids base_id) 1232 { 1233 struct clocksource *cs = tk_core.timekeeper.tkr_mono.clock; 1234 struct clocksource_base *base; 1235 1236 /* 1237 * Check whether base_id matches the base clock. Prevent the compiler from 1238 * re-evaluating @base as the clocksource might change concurrently. 1239 */ 1240 base = READ_ONCE(cs->base); 1241 if (!base || base->id != base_id) 1242 return false; 1243 1244 *cycles -= base->offset; 1245 if (!convert_clock(cycles, base->denominator, base->numerator)) 1246 return false; 1247 return true; 1248 } 1249 1250 static bool convert_ns_to_cs(u64 *delta) 1251 { 1252 struct tk_read_base *tkr = &tk_core.timekeeper.tkr_mono; 1253 1254 if (BITS_TO_BYTES(fls64(*delta) + tkr->shift) >= sizeof(*delta)) 1255 return false; 1256 1257 *delta = div_u64((*delta << tkr->shift) - tkr->xtime_nsec, tkr->mult); 1258 return true; 1259 } 1260 1261 /** 1262 * ktime_real_to_base_clock() - Convert CLOCK_REALTIME timestamp to a base clock timestamp 1263 * @treal: CLOCK_REALTIME timestamp to convert 1264 * @base_id: base clocksource id 1265 * @cycles: pointer to store the converted base clock timestamp 1266 * 1267 * Converts a supplied, future realtime clock value to the corresponding base clock value. 1268 * 1269 * Return: true if the conversion is successful, false otherwise. 1270 */ 1271 bool ktime_real_to_base_clock(ktime_t treal, enum clocksource_ids base_id, u64 *cycles) 1272 { 1273 struct timekeeper *tk = &tk_core.timekeeper; 1274 unsigned int seq; 1275 u64 delta; 1276 1277 do { 1278 seq = read_seqcount_begin(&tk_core.seq); 1279 if ((u64)treal < tk->tkr_mono.base_real) 1280 return false; 1281 delta = (u64)treal - tk->tkr_mono.base_real; 1282 if (!convert_ns_to_cs(&delta)) 1283 return false; 1284 *cycles = tk->tkr_mono.cycle_last + delta; 1285 if (!convert_cs_to_base(cycles, base_id)) 1286 return false; 1287 } while (read_seqcount_retry(&tk_core.seq, seq)); 1288 1289 return true; 1290 } 1291 EXPORT_SYMBOL_GPL(ktime_real_to_base_clock); 1292 1293 /** 1294 * get_device_system_crosststamp - Synchronously capture system/device timestamp 1295 * @get_time_fn: Callback to get simultaneous device time and 1296 * system counter from the device driver 1297 * @ctx: Context passed to get_time_fn() 1298 * @history_begin: Historical reference point used to interpolate system 1299 * time when counter provided by the driver is before the current interval 1300 * @xtstamp: Receives simultaneously captured system and device time 1301 * 1302 * Reads a timestamp from a device and correlates it to system time 1303 */ 1304 int get_device_system_crosststamp(int (*get_time_fn) 1305 (ktime_t *device_time, 1306 struct system_counterval_t *sys_counterval, 1307 void *ctx), 1308 void *ctx, 1309 struct system_time_snapshot *history_begin, 1310 struct system_device_crosststamp *xtstamp) 1311 { 1312 struct system_counterval_t system_counterval = {}; 1313 struct timekeeper *tk = &tk_core.timekeeper; 1314 u64 cycles, now, interval_start; 1315 unsigned int clock_was_set_seq = 0; 1316 ktime_t base_real, base_raw; 1317 u64 nsec_real, nsec_raw; 1318 u8 cs_was_changed_seq; 1319 unsigned int seq; 1320 bool do_interp; 1321 int ret; 1322 1323 do { 1324 seq = read_seqcount_begin(&tk_core.seq); 1325 /* 1326 * Try to synchronously capture device time and a system 1327 * counter value calling back into the device driver 1328 */ 1329 ret = get_time_fn(&xtstamp->device, &system_counterval, ctx); 1330 if (ret) 1331 return ret; 1332 1333 /* 1334 * Verify that the clocksource ID associated with the captured 1335 * system counter value is the same as for the currently 1336 * installed timekeeper clocksource 1337 */ 1338 if (system_counterval.cs_id == CSID_GENERIC || 1339 !convert_base_to_cs(&system_counterval)) 1340 return -ENODEV; 1341 cycles = system_counterval.cycles; 1342 1343 /* 1344 * Check whether the system counter value provided by the 1345 * device driver is on the current timekeeping interval. 1346 */ 1347 now = tk_clock_read(&tk->tkr_mono); 1348 interval_start = tk->tkr_mono.cycle_last; 1349 if (!timestamp_in_interval(interval_start, now, cycles)) { 1350 clock_was_set_seq = tk->clock_was_set_seq; 1351 cs_was_changed_seq = tk->cs_was_changed_seq; 1352 cycles = interval_start; 1353 do_interp = true; 1354 } else { 1355 do_interp = false; 1356 } 1357 1358 base_real = ktime_add(tk->tkr_mono.base, 1359 tk_core.timekeeper.offs_real); 1360 base_raw = tk->tkr_raw.base; 1361 1362 nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, cycles); 1363 nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, cycles); 1364 } while (read_seqcount_retry(&tk_core.seq, seq)); 1365 1366 xtstamp->sys_realtime = ktime_add_ns(base_real, nsec_real); 1367 xtstamp->sys_monoraw = ktime_add_ns(base_raw, nsec_raw); 1368 1369 /* 1370 * Interpolate if necessary, adjusting back from the start of the 1371 * current interval 1372 */ 1373 if (do_interp) { 1374 u64 partial_history_cycles, total_history_cycles; 1375 bool discontinuity; 1376 1377 /* 1378 * Check that the counter value is not before the provided 1379 * history reference and that the history doesn't cross a 1380 * clocksource change 1381 */ 1382 if (!history_begin || 1383 !timestamp_in_interval(history_begin->cycles, 1384 cycles, system_counterval.cycles) || 1385 history_begin->cs_was_changed_seq != cs_was_changed_seq) 1386 return -EINVAL; 1387 partial_history_cycles = cycles - system_counterval.cycles; 1388 total_history_cycles = cycles - history_begin->cycles; 1389 discontinuity = 1390 history_begin->clock_was_set_seq != clock_was_set_seq; 1391 1392 ret = adjust_historical_crosststamp(history_begin, 1393 partial_history_cycles, 1394 total_history_cycles, 1395 discontinuity, xtstamp); 1396 if (ret) 1397 return ret; 1398 } 1399 1400 return 0; 1401 } 1402 EXPORT_SYMBOL_GPL(get_device_system_crosststamp); 1403 1404 /** 1405 * timekeeping_clocksource_has_base - Check whether the current clocksource 1406 * is based on given a base clock 1407 * @id: base clocksource ID 1408 * 1409 * Note: The return value is a snapshot which can become invalid right 1410 * after the function returns. 1411 * 1412 * Return: true if the timekeeper clocksource has a base clock with @id, 1413 * false otherwise 1414 */ 1415 bool timekeeping_clocksource_has_base(enum clocksource_ids id) 1416 { 1417 /* 1418 * This is a snapshot, so no point in using the sequence 1419 * count. Just prevent the compiler from re-evaluating @base as the 1420 * clocksource might change concurrently. 1421 */ 1422 struct clocksource_base *base = READ_ONCE(tk_core.timekeeper.tkr_mono.clock->base); 1423 1424 return base ? base->id == id : false; 1425 } 1426 EXPORT_SYMBOL_GPL(timekeeping_clocksource_has_base); 1427 1428 /** 1429 * do_settimeofday64 - Sets the time of day. 1430 * @ts: pointer to the timespec64 variable containing the new time 1431 * 1432 * Sets the time of day to the new time and update NTP and notify hrtimers 1433 */ 1434 int do_settimeofday64(const struct timespec64 *ts) 1435 { 1436 struct timespec64 ts_delta, xt; 1437 1438 if (!timespec64_valid_settod(ts)) 1439 return -EINVAL; 1440 1441 scoped_guard (raw_spinlock_irqsave, &tk_core.lock) { 1442 struct timekeeper *tks = &tk_core.shadow_timekeeper; 1443 1444 timekeeping_forward_now(tks); 1445 1446 xt = tk_xtime(tks); 1447 ts_delta = timespec64_sub(*ts, xt); 1448 1449 if (timespec64_compare(&tks->wall_to_monotonic, &ts_delta) > 0) { 1450 timekeeping_restore_shadow(&tk_core); 1451 return -EINVAL; 1452 } 1453 1454 tk_set_wall_to_mono(tks, timespec64_sub(tks->wall_to_monotonic, ts_delta)); 1455 tk_set_xtime(tks, ts); 1456 timekeeping_update_from_shadow(&tk_core, TK_UPDATE_ALL); 1457 } 1458 1459 /* Signal hrtimers about time change */ 1460 clock_was_set(CLOCK_SET_WALL); 1461 1462 audit_tk_injoffset(ts_delta); 1463 add_device_randomness(ts, sizeof(*ts)); 1464 return 0; 1465 } 1466 EXPORT_SYMBOL(do_settimeofday64); 1467 1468 static inline bool timekeeper_is_core_tk(struct timekeeper *tk) 1469 { 1470 return !IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS) || tk->id == TIMEKEEPER_CORE; 1471 } 1472 1473 /** 1474 * __timekeeping_inject_offset - Adds or subtracts from the current time. 1475 * @tkd: Pointer to the timekeeper to modify 1476 * @ts: Pointer to the timespec variable containing the offset 1477 * 1478 * Adds or subtracts an offset value from the current time. 1479 */ 1480 static int __timekeeping_inject_offset(struct tk_data *tkd, const struct timespec64 *ts) 1481 { 1482 struct timekeeper *tks = &tkd->shadow_timekeeper; 1483 struct timespec64 tmp; 1484 1485 if (ts->tv_nsec < 0 || ts->tv_nsec >= NSEC_PER_SEC) 1486 return -EINVAL; 1487 1488 timekeeping_forward_now(tks); 1489 1490 if (timekeeper_is_core_tk(tks)) { 1491 /* Make sure the proposed value is valid */ 1492 tmp = timespec64_add(tk_xtime(tks), *ts); 1493 if (timespec64_compare(&tks->wall_to_monotonic, ts) > 0 || 1494 !timespec64_valid_settod(&tmp)) { 1495 timekeeping_restore_shadow(tkd); 1496 return -EINVAL; 1497 } 1498 1499 tk_xtime_add(tks, ts); 1500 tk_set_wall_to_mono(tks, timespec64_sub(tks->wall_to_monotonic, *ts)); 1501 } else { 1502 struct tk_read_base *tkr_mono = &tks->tkr_mono; 1503 ktime_t now, offs; 1504 1505 /* Get the current time */ 1506 now = ktime_add_ns(tkr_mono->base, timekeeping_get_ns(tkr_mono)); 1507 /* Add the relative offset change */ 1508 offs = ktime_add(tks->offs_aux, timespec64_to_ktime(*ts)); 1509 1510 /* Prevent that the resulting time becomes negative */ 1511 if (ktime_add(now, offs) < 0) { 1512 timekeeping_restore_shadow(tkd); 1513 return -EINVAL; 1514 } 1515 tk_update_aux_offs(tks, offs); 1516 } 1517 1518 timekeeping_update_from_shadow(tkd, TK_UPDATE_ALL); 1519 return 0; 1520 } 1521 1522 static int timekeeping_inject_offset(const struct timespec64 *ts) 1523 { 1524 int ret; 1525 1526 scoped_guard (raw_spinlock_irqsave, &tk_core.lock) 1527 ret = __timekeeping_inject_offset(&tk_core, ts); 1528 1529 /* Signal hrtimers about time change */ 1530 if (!ret) 1531 clock_was_set(CLOCK_SET_WALL); 1532 return ret; 1533 } 1534 1535 /* 1536 * Indicates if there is an offset between the system clock and the hardware 1537 * clock/persistent clock/rtc. 1538 */ 1539 int persistent_clock_is_local; 1540 1541 /* 1542 * Adjust the time obtained from the CMOS to be UTC time instead of 1543 * local time. 1544 * 1545 * This is ugly, but preferable to the alternatives. Otherwise we 1546 * would either need to write a program to do it in /etc/rc (and risk 1547 * confusion if the program gets run more than once; it would also be 1548 * hard to make the program warp the clock precisely n hours) or 1549 * compile in the timezone information into the kernel. Bad, bad.... 1550 * 1551 * - TYT, 1992-01-01 1552 * 1553 * The best thing to do is to keep the CMOS clock in universal time (UTC) 1554 * as real UNIX machines always do it. This avoids all headaches about 1555 * daylight saving times and warping kernel clocks. 1556 */ 1557 void timekeeping_warp_clock(void) 1558 { 1559 if (sys_tz.tz_minuteswest != 0) { 1560 struct timespec64 adjust; 1561 1562 persistent_clock_is_local = 1; 1563 adjust.tv_sec = sys_tz.tz_minuteswest * 60; 1564 adjust.tv_nsec = 0; 1565 timekeeping_inject_offset(&adjust); 1566 } 1567 } 1568 1569 /* 1570 * __timekeeping_set_tai_offset - Sets the TAI offset from UTC and monotonic 1571 */ 1572 static void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset) 1573 { 1574 tk->tai_offset = tai_offset; 1575 tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0)); 1576 } 1577 1578 /* 1579 * change_clocksource - Swaps clocksources if a new one is available 1580 * 1581 * Accumulates current time interval and initializes new clocksource 1582 */ 1583 static int change_clocksource(void *data) 1584 { 1585 struct clocksource *new = data, *old = NULL; 1586 1587 /* 1588 * If the clocksource is in a module, get a module reference. 1589 * Succeeds for built-in code (owner == NULL) as well. Abort if the 1590 * reference can't be acquired. 1591 */ 1592 if (!try_module_get(new->owner)) 1593 return 0; 1594 1595 /* Abort if the device can't be enabled */ 1596 if (new->enable && new->enable(new) != 0) { 1597 module_put(new->owner); 1598 return 0; 1599 } 1600 1601 scoped_guard (raw_spinlock_irqsave, &tk_core.lock) { 1602 struct timekeeper *tks = &tk_core.shadow_timekeeper; 1603 1604 timekeeping_forward_now(tks); 1605 old = tks->tkr_mono.clock; 1606 tk_setup_internals(tks, new); 1607 timekeeping_update_from_shadow(&tk_core, TK_UPDATE_ALL); 1608 } 1609 1610 tk_aux_update_clocksource(); 1611 1612 if (old) { 1613 if (old->disable) 1614 old->disable(old); 1615 module_put(old->owner); 1616 } 1617 1618 return 0; 1619 } 1620 1621 /** 1622 * timekeeping_notify - Install a new clock source 1623 * @clock: pointer to the clock source 1624 * 1625 * This function is called from clocksource.c after a new, better clock 1626 * source has been registered. The caller holds the clocksource_mutex. 1627 */ 1628 int timekeeping_notify(struct clocksource *clock) 1629 { 1630 struct timekeeper *tk = &tk_core.timekeeper; 1631 1632 if (tk->tkr_mono.clock == clock) 1633 return 0; 1634 stop_machine(change_clocksource, clock, NULL); 1635 tick_clock_notify(); 1636 return tk->tkr_mono.clock == clock ? 0 : -1; 1637 } 1638 1639 /** 1640 * ktime_get_raw_ts64 - Returns the raw monotonic time in a timespec 1641 * @ts: pointer to the timespec64 to be set 1642 * 1643 * Returns the raw monotonic time (completely un-modified by ntp) 1644 */ 1645 void ktime_get_raw_ts64(struct timespec64 *ts) 1646 { 1647 struct timekeeper *tk = &tk_core.timekeeper; 1648 unsigned int seq; 1649 u64 nsecs; 1650 1651 do { 1652 seq = read_seqcount_begin(&tk_core.seq); 1653 ts->tv_sec = tk->raw_sec; 1654 nsecs = timekeeping_get_ns(&tk->tkr_raw); 1655 1656 } while (read_seqcount_retry(&tk_core.seq, seq)); 1657 1658 ts->tv_nsec = 0; 1659 timespec64_add_ns(ts, nsecs); 1660 } 1661 EXPORT_SYMBOL(ktime_get_raw_ts64); 1662 1663 /** 1664 * ktime_get_clock_ts64 - Returns time of a clock in a timespec 1665 * @id: POSIX clock ID of the clock to read 1666 * @ts: Pointer to the timespec64 to be set 1667 * 1668 * The timestamp is invalidated (@ts->sec is set to -1) if the 1669 * clock @id is not available. 1670 */ 1671 void ktime_get_clock_ts64(clockid_t id, struct timespec64 *ts) 1672 { 1673 /* Invalidate time stamp */ 1674 ts->tv_sec = -1; 1675 ts->tv_nsec = 0; 1676 1677 switch (id) { 1678 case CLOCK_REALTIME: 1679 ktime_get_real_ts64(ts); 1680 return; 1681 case CLOCK_MONOTONIC: 1682 ktime_get_ts64(ts); 1683 return; 1684 case CLOCK_MONOTONIC_RAW: 1685 ktime_get_raw_ts64(ts); 1686 return; 1687 case CLOCK_AUX ... CLOCK_AUX_LAST: 1688 if (IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS)) 1689 ktime_get_aux_ts64(id, ts); 1690 return; 1691 default: 1692 WARN_ON_ONCE(1); 1693 } 1694 } 1695 EXPORT_SYMBOL_GPL(ktime_get_clock_ts64); 1696 1697 /** 1698 * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres 1699 */ 1700 int timekeeping_valid_for_hres(void) 1701 { 1702 struct timekeeper *tk = &tk_core.timekeeper; 1703 unsigned int seq; 1704 int ret; 1705 1706 do { 1707 seq = read_seqcount_begin(&tk_core.seq); 1708 1709 ret = tk->tkr_mono.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES; 1710 1711 } while (read_seqcount_retry(&tk_core.seq, seq)); 1712 1713 return ret; 1714 } 1715 1716 /** 1717 * timekeeping_max_deferment - Returns max time the clocksource can be deferred 1718 */ 1719 u64 timekeeping_max_deferment(void) 1720 { 1721 struct timekeeper *tk = &tk_core.timekeeper; 1722 unsigned int seq; 1723 u64 ret; 1724 1725 do { 1726 seq = read_seqcount_begin(&tk_core.seq); 1727 1728 ret = tk->tkr_mono.clock->max_idle_ns; 1729 1730 } while (read_seqcount_retry(&tk_core.seq, seq)); 1731 1732 return ret; 1733 } 1734 1735 /** 1736 * read_persistent_clock64 - Return time from the persistent clock. 1737 * @ts: Pointer to the storage for the readout value 1738 * 1739 * Weak dummy function for arches that do not yet support it. 1740 * Reads the time from the battery backed persistent clock. 1741 * Returns a timespec with tv_sec=0 and tv_nsec=0 if unsupported. 1742 * 1743 * XXX - Do be sure to remove it once all arches implement it. 1744 */ 1745 void __weak read_persistent_clock64(struct timespec64 *ts) 1746 { 1747 ts->tv_sec = 0; 1748 ts->tv_nsec = 0; 1749 } 1750 1751 /** 1752 * read_persistent_wall_and_boot_offset - Read persistent clock, and also offset 1753 * from the boot. 1754 * @wall_time: current time as returned by persistent clock 1755 * @boot_offset: offset that is defined as wall_time - boot_time 1756 * 1757 * Weak dummy function for arches that do not yet support it. 1758 * 1759 * The default function calculates offset based on the current value of 1760 * local_clock(). This way architectures that support sched_clock() but don't 1761 * support dedicated boot time clock will provide the best estimate of the 1762 * boot time. 1763 */ 1764 void __weak __init 1765 read_persistent_wall_and_boot_offset(struct timespec64 *wall_time, 1766 struct timespec64 *boot_offset) 1767 { 1768 read_persistent_clock64(wall_time); 1769 *boot_offset = ns_to_timespec64(local_clock()); 1770 } 1771 1772 static __init void tkd_basic_setup(struct tk_data *tkd, enum timekeeper_ids tk_id, bool valid) 1773 { 1774 raw_spin_lock_init(&tkd->lock); 1775 seqcount_raw_spinlock_init(&tkd->seq, &tkd->lock); 1776 tkd->timekeeper.id = tkd->shadow_timekeeper.id = tk_id; 1777 tkd->timekeeper.clock_valid = tkd->shadow_timekeeper.clock_valid = valid; 1778 } 1779 1780 /* 1781 * Flag reflecting whether timekeeping_resume() has injected sleeptime. 1782 * 1783 * The flag starts of false and is only set when a suspend reaches 1784 * timekeeping_suspend(), timekeeping_resume() sets it to false when the 1785 * timekeeper clocksource is not stopping across suspend and has been 1786 * used to update sleep time. If the timekeeper clocksource has stopped 1787 * then the flag stays true and is used by the RTC resume code to decide 1788 * whether sleeptime must be injected and if so the flag gets false then. 1789 * 1790 * If a suspend fails before reaching timekeeping_resume() then the flag 1791 * stays false and prevents erroneous sleeptime injection. 1792 */ 1793 static bool suspend_timing_needed; 1794 1795 /* Flag for if there is a persistent clock on this platform */ 1796 static bool persistent_clock_exists; 1797 1798 /* 1799 * timekeeping_init - Initializes the clocksource and common timekeeping values 1800 */ 1801 void __init timekeeping_init(void) 1802 { 1803 struct timespec64 wall_time, boot_offset, wall_to_mono; 1804 struct timekeeper *tks = &tk_core.shadow_timekeeper; 1805 struct clocksource *clock; 1806 1807 tkd_basic_setup(&tk_core, TIMEKEEPER_CORE, true); 1808 tk_aux_setup(); 1809 1810 read_persistent_wall_and_boot_offset(&wall_time, &boot_offset); 1811 if (timespec64_valid_settod(&wall_time) && 1812 timespec64_to_ns(&wall_time) > 0) { 1813 persistent_clock_exists = true; 1814 } else if (timespec64_to_ns(&wall_time) != 0) { 1815 pr_warn("Persistent clock returned invalid value"); 1816 wall_time = (struct timespec64){0}; 1817 } 1818 1819 if (timespec64_compare(&wall_time, &boot_offset) < 0) 1820 boot_offset = (struct timespec64){0}; 1821 1822 /* 1823 * We want set wall_to_mono, so the following is true: 1824 * wall time + wall_to_mono = boot time 1825 */ 1826 wall_to_mono = timespec64_sub(boot_offset, wall_time); 1827 1828 guard(raw_spinlock_irqsave)(&tk_core.lock); 1829 1830 ntp_init(); 1831 1832 clock = clocksource_default_clock(); 1833 if (clock->enable) 1834 clock->enable(clock); 1835 tk_setup_internals(tks, clock); 1836 1837 tk_set_xtime(tks, &wall_time); 1838 tks->raw_sec = 0; 1839 1840 tk_set_wall_to_mono(tks, wall_to_mono); 1841 1842 timekeeping_update_from_shadow(&tk_core, TK_CLOCK_WAS_SET); 1843 } 1844 1845 /* time in seconds when suspend began for persistent clock */ 1846 static struct timespec64 timekeeping_suspend_time; 1847 1848 /** 1849 * __timekeeping_inject_sleeptime - Internal function to add sleep interval 1850 * @tk: Pointer to the timekeeper to be updated 1851 * @delta: Pointer to the delta value in timespec64 format 1852 * 1853 * Takes a timespec offset measuring a suspend interval and properly 1854 * adds the sleep offset to the timekeeping variables. 1855 */ 1856 static void __timekeeping_inject_sleeptime(struct timekeeper *tk, 1857 const struct timespec64 *delta) 1858 { 1859 if (!timespec64_valid_strict(delta)) { 1860 printk_deferred(KERN_WARNING 1861 "__timekeeping_inject_sleeptime: Invalid " 1862 "sleep delta value!\n"); 1863 return; 1864 } 1865 tk_xtime_add(tk, delta); 1866 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *delta)); 1867 tk_update_sleep_time(tk, timespec64_to_ktime(*delta)); 1868 tk_debug_account_sleep_time(delta); 1869 } 1870 1871 #if defined(CONFIG_PM_SLEEP) && defined(CONFIG_RTC_HCTOSYS_DEVICE) 1872 /* 1873 * We have three kinds of time sources to use for sleep time 1874 * injection, the preference order is: 1875 * 1) non-stop clocksource 1876 * 2) persistent clock (ie: RTC accessible when irqs are off) 1877 * 3) RTC 1878 * 1879 * 1) and 2) are used by timekeeping, 3) by RTC subsystem. 1880 * If system has neither 1) nor 2), 3) will be used finally. 1881 * 1882 * 1883 * If timekeeping has injected sleeptime via either 1) or 2), 1884 * 3) becomes needless, so in this case we don't need to call 1885 * rtc_resume(), and this is what timekeeping_rtc_skipresume() 1886 * means. 1887 */ 1888 bool timekeeping_rtc_skipresume(void) 1889 { 1890 return !suspend_timing_needed; 1891 } 1892 1893 /* 1894 * 1) can be determined whether to use or not only when doing 1895 * timekeeping_resume() which is invoked after rtc_suspend(), 1896 * so we can't skip rtc_suspend() surely if system has 1). 1897 * 1898 * But if system has 2), 2) will definitely be used, so in this 1899 * case we don't need to call rtc_suspend(), and this is what 1900 * timekeeping_rtc_skipsuspend() means. 1901 */ 1902 bool timekeeping_rtc_skipsuspend(void) 1903 { 1904 return persistent_clock_exists; 1905 } 1906 1907 /** 1908 * timekeeping_inject_sleeptime64 - Adds suspend interval to timeekeeping values 1909 * @delta: pointer to a timespec64 delta value 1910 * 1911 * This hook is for architectures that cannot support read_persistent_clock64 1912 * because their RTC/persistent clock is only accessible when irqs are enabled. 1913 * and also don't have an effective nonstop clocksource. 1914 * 1915 * This function should only be called by rtc_resume(), and allows 1916 * a suspend offset to be injected into the timekeeping values. 1917 */ 1918 void timekeeping_inject_sleeptime64(const struct timespec64 *delta) 1919 { 1920 scoped_guard(raw_spinlock_irqsave, &tk_core.lock) { 1921 struct timekeeper *tks = &tk_core.shadow_timekeeper; 1922 1923 suspend_timing_needed = false; 1924 timekeeping_forward_now(tks); 1925 __timekeeping_inject_sleeptime(tks, delta); 1926 timekeeping_update_from_shadow(&tk_core, TK_UPDATE_ALL); 1927 } 1928 1929 /* Signal hrtimers about time change */ 1930 clock_was_set(CLOCK_SET_WALL | CLOCK_SET_BOOT); 1931 } 1932 #endif 1933 1934 /** 1935 * timekeeping_resume - Resumes the generic timekeeping subsystem. 1936 */ 1937 void timekeeping_resume(void) 1938 { 1939 struct timekeeper *tks = &tk_core.shadow_timekeeper; 1940 struct clocksource *clock = tks->tkr_mono.clock; 1941 struct timespec64 ts_new, ts_delta; 1942 bool inject_sleeptime = false; 1943 u64 cycle_now, nsec; 1944 unsigned long flags; 1945 1946 read_persistent_clock64(&ts_new); 1947 1948 clockevents_resume(); 1949 clocksource_resume(); 1950 1951 raw_spin_lock_irqsave(&tk_core.lock, flags); 1952 1953 /* 1954 * After system resumes, we need to calculate the suspended time and 1955 * compensate it for the OS time. There are 3 sources that could be 1956 * used: Nonstop clocksource during suspend, persistent clock and rtc 1957 * device. 1958 * 1959 * One specific platform may have 1 or 2 or all of them, and the 1960 * preference will be: 1961 * suspend-nonstop clocksource -> persistent clock -> rtc 1962 * The less preferred source will only be tried if there is no better 1963 * usable source. The rtc part is handled separately in rtc core code. 1964 */ 1965 cycle_now = tk_clock_read(&tks->tkr_mono); 1966 nsec = clocksource_stop_suspend_timing(clock, cycle_now); 1967 if (nsec > 0) { 1968 ts_delta = ns_to_timespec64(nsec); 1969 inject_sleeptime = true; 1970 } else if (timespec64_compare(&ts_new, &timekeeping_suspend_time) > 0) { 1971 ts_delta = timespec64_sub(ts_new, timekeeping_suspend_time); 1972 inject_sleeptime = true; 1973 } 1974 1975 if (inject_sleeptime) { 1976 suspend_timing_needed = false; 1977 __timekeeping_inject_sleeptime(tks, &ts_delta); 1978 } 1979 1980 /* Re-base the last cycle value */ 1981 tks->tkr_mono.cycle_last = cycle_now; 1982 tks->tkr_raw.cycle_last = cycle_now; 1983 1984 tks->ntp_error = 0; 1985 timekeeping_suspended = 0; 1986 timekeeping_update_from_shadow(&tk_core, TK_CLOCK_WAS_SET); 1987 raw_spin_unlock_irqrestore(&tk_core.lock, flags); 1988 1989 touch_softlockup_watchdog(); 1990 1991 /* Resume the clockevent device(s) and hrtimers */ 1992 tick_resume(); 1993 /* Notify timerfd as resume is equivalent to clock_was_set() */ 1994 timerfd_resume(); 1995 } 1996 1997 int timekeeping_suspend(void) 1998 { 1999 struct timekeeper *tks = &tk_core.shadow_timekeeper; 2000 struct timespec64 delta, delta_delta; 2001 static struct timespec64 old_delta; 2002 struct clocksource *curr_clock; 2003 unsigned long flags; 2004 u64 cycle_now; 2005 2006 read_persistent_clock64(&timekeeping_suspend_time); 2007 2008 /* 2009 * On some systems the persistent_clock can not be detected at 2010 * timekeeping_init by its return value, so if we see a valid 2011 * value returned, update the persistent_clock_exists flag. 2012 */ 2013 if (timekeeping_suspend_time.tv_sec || timekeeping_suspend_time.tv_nsec) 2014 persistent_clock_exists = true; 2015 2016 suspend_timing_needed = true; 2017 2018 raw_spin_lock_irqsave(&tk_core.lock, flags); 2019 timekeeping_forward_now(tks); 2020 timekeeping_suspended = 1; 2021 2022 /* 2023 * Since we've called forward_now, cycle_last stores the value 2024 * just read from the current clocksource. Save this to potentially 2025 * use in suspend timing. 2026 */ 2027 curr_clock = tks->tkr_mono.clock; 2028 cycle_now = tks->tkr_mono.cycle_last; 2029 clocksource_start_suspend_timing(curr_clock, cycle_now); 2030 2031 if (persistent_clock_exists) { 2032 /* 2033 * To avoid drift caused by repeated suspend/resumes, 2034 * which each can add ~1 second drift error, 2035 * try to compensate so the difference in system time 2036 * and persistent_clock time stays close to constant. 2037 */ 2038 delta = timespec64_sub(tk_xtime(tks), timekeeping_suspend_time); 2039 delta_delta = timespec64_sub(delta, old_delta); 2040 if (abs(delta_delta.tv_sec) >= 2) { 2041 /* 2042 * if delta_delta is too large, assume time correction 2043 * has occurred and set old_delta to the current delta. 2044 */ 2045 old_delta = delta; 2046 } else { 2047 /* Otherwise try to adjust old_system to compensate */ 2048 timekeeping_suspend_time = 2049 timespec64_add(timekeeping_suspend_time, delta_delta); 2050 } 2051 } 2052 2053 timekeeping_update_from_shadow(&tk_core, 0); 2054 halt_fast_timekeeper(tks); 2055 raw_spin_unlock_irqrestore(&tk_core.lock, flags); 2056 2057 tick_suspend(); 2058 clocksource_suspend(); 2059 clockevents_suspend(); 2060 2061 return 0; 2062 } 2063 2064 /* sysfs resume/suspend bits for timekeeping */ 2065 static struct syscore_ops timekeeping_syscore_ops = { 2066 .resume = timekeeping_resume, 2067 .suspend = timekeeping_suspend, 2068 }; 2069 2070 static int __init timekeeping_init_ops(void) 2071 { 2072 register_syscore_ops(&timekeeping_syscore_ops); 2073 return 0; 2074 } 2075 device_initcall(timekeeping_init_ops); 2076 2077 /* 2078 * Apply a multiplier adjustment to the timekeeper 2079 */ 2080 static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk, 2081 s64 offset, 2082 s32 mult_adj) 2083 { 2084 s64 interval = tk->cycle_interval; 2085 2086 if (mult_adj == 0) { 2087 return; 2088 } else if (mult_adj == -1) { 2089 interval = -interval; 2090 offset = -offset; 2091 } else if (mult_adj != 1) { 2092 interval *= mult_adj; 2093 offset *= mult_adj; 2094 } 2095 2096 /* 2097 * So the following can be confusing. 2098 * 2099 * To keep things simple, lets assume mult_adj == 1 for now. 2100 * 2101 * When mult_adj != 1, remember that the interval and offset values 2102 * have been appropriately scaled so the math is the same. 2103 * 2104 * The basic idea here is that we're increasing the multiplier 2105 * by one, this causes the xtime_interval to be incremented by 2106 * one cycle_interval. This is because: 2107 * xtime_interval = cycle_interval * mult 2108 * So if mult is being incremented by one: 2109 * xtime_interval = cycle_interval * (mult + 1) 2110 * Its the same as: 2111 * xtime_interval = (cycle_interval * mult) + cycle_interval 2112 * Which can be shortened to: 2113 * xtime_interval += cycle_interval 2114 * 2115 * So offset stores the non-accumulated cycles. Thus the current 2116 * time (in shifted nanoseconds) is: 2117 * now = (offset * adj) + xtime_nsec 2118 * Now, even though we're adjusting the clock frequency, we have 2119 * to keep time consistent. In other words, we can't jump back 2120 * in time, and we also want to avoid jumping forward in time. 2121 * 2122 * So given the same offset value, we need the time to be the same 2123 * both before and after the freq adjustment. 2124 * now = (offset * adj_1) + xtime_nsec_1 2125 * now = (offset * adj_2) + xtime_nsec_2 2126 * So: 2127 * (offset * adj_1) + xtime_nsec_1 = 2128 * (offset * adj_2) + xtime_nsec_2 2129 * And we know: 2130 * adj_2 = adj_1 + 1 2131 * So: 2132 * (offset * adj_1) + xtime_nsec_1 = 2133 * (offset * (adj_1+1)) + xtime_nsec_2 2134 * (offset * adj_1) + xtime_nsec_1 = 2135 * (offset * adj_1) + offset + xtime_nsec_2 2136 * Canceling the sides: 2137 * xtime_nsec_1 = offset + xtime_nsec_2 2138 * Which gives us: 2139 * xtime_nsec_2 = xtime_nsec_1 - offset 2140 * Which simplifies to: 2141 * xtime_nsec -= offset 2142 */ 2143 if ((mult_adj > 0) && (tk->tkr_mono.mult + mult_adj < mult_adj)) { 2144 /* NTP adjustment caused clocksource mult overflow */ 2145 WARN_ON_ONCE(1); 2146 return; 2147 } 2148 2149 tk->tkr_mono.mult += mult_adj; 2150 tk->xtime_interval += interval; 2151 tk->tkr_mono.xtime_nsec -= offset; 2152 } 2153 2154 /* 2155 * Adjust the timekeeper's multiplier to the correct frequency 2156 * and also to reduce the accumulated error value. 2157 */ 2158 static void timekeeping_adjust(struct timekeeper *tk, s64 offset) 2159 { 2160 u64 ntp_tl = ntp_tick_length(tk->id); 2161 u32 mult; 2162 2163 /* 2164 * Determine the multiplier from the current NTP tick length. 2165 * Avoid expensive division when the tick length doesn't change. 2166 */ 2167 if (likely(tk->ntp_tick == ntp_tl)) { 2168 mult = tk->tkr_mono.mult - tk->ntp_err_mult; 2169 } else { 2170 tk->ntp_tick = ntp_tl; 2171 mult = div64_u64((tk->ntp_tick >> tk->ntp_error_shift) - 2172 tk->xtime_remainder, tk->cycle_interval); 2173 } 2174 2175 /* 2176 * If the clock is behind the NTP time, increase the multiplier by 1 2177 * to catch up with it. If it's ahead and there was a remainder in the 2178 * tick division, the clock will slow down. Otherwise it will stay 2179 * ahead until the tick length changes to a non-divisible value. 2180 */ 2181 tk->ntp_err_mult = tk->ntp_error > 0 ? 1 : 0; 2182 mult += tk->ntp_err_mult; 2183 2184 timekeeping_apply_adjustment(tk, offset, mult - tk->tkr_mono.mult); 2185 2186 if (unlikely(tk->tkr_mono.clock->maxadj && 2187 (abs(tk->tkr_mono.mult - tk->tkr_mono.clock->mult) 2188 > tk->tkr_mono.clock->maxadj))) { 2189 printk_once(KERN_WARNING 2190 "Adjusting %s more than 11%% (%ld vs %ld)\n", 2191 tk->tkr_mono.clock->name, (long)tk->tkr_mono.mult, 2192 (long)tk->tkr_mono.clock->mult + tk->tkr_mono.clock->maxadj); 2193 } 2194 2195 /* 2196 * It may be possible that when we entered this function, xtime_nsec 2197 * was very small. Further, if we're slightly speeding the clocksource 2198 * in the code above, its possible the required corrective factor to 2199 * xtime_nsec could cause it to underflow. 2200 * 2201 * Now, since we have already accumulated the second and the NTP 2202 * subsystem has been notified via second_overflow(), we need to skip 2203 * the next update. 2204 */ 2205 if (unlikely((s64)tk->tkr_mono.xtime_nsec < 0)) { 2206 tk->tkr_mono.xtime_nsec += (u64)NSEC_PER_SEC << 2207 tk->tkr_mono.shift; 2208 tk->xtime_sec--; 2209 tk->skip_second_overflow = 1; 2210 } 2211 } 2212 2213 /* 2214 * accumulate_nsecs_to_secs - Accumulates nsecs into secs 2215 * 2216 * Helper function that accumulates the nsecs greater than a second 2217 * from the xtime_nsec field to the xtime_secs field. 2218 * It also calls into the NTP code to handle leapsecond processing. 2219 */ 2220 static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk) 2221 { 2222 u64 nsecps = (u64)NSEC_PER_SEC << tk->tkr_mono.shift; 2223 unsigned int clock_set = 0; 2224 2225 while (tk->tkr_mono.xtime_nsec >= nsecps) { 2226 int leap; 2227 2228 tk->tkr_mono.xtime_nsec -= nsecps; 2229 tk->xtime_sec++; 2230 2231 /* 2232 * Skip NTP update if this second was accumulated before, 2233 * i.e. xtime_nsec underflowed in timekeeping_adjust() 2234 */ 2235 if (unlikely(tk->skip_second_overflow)) { 2236 tk->skip_second_overflow = 0; 2237 continue; 2238 } 2239 2240 /* Figure out if its a leap sec and apply if needed */ 2241 leap = second_overflow(tk->id, tk->xtime_sec); 2242 if (unlikely(leap)) { 2243 struct timespec64 ts; 2244 2245 tk->xtime_sec += leap; 2246 2247 ts.tv_sec = leap; 2248 ts.tv_nsec = 0; 2249 tk_set_wall_to_mono(tk, 2250 timespec64_sub(tk->wall_to_monotonic, ts)); 2251 2252 __timekeeping_set_tai_offset(tk, tk->tai_offset - leap); 2253 2254 clock_set = TK_CLOCK_WAS_SET; 2255 } 2256 } 2257 return clock_set; 2258 } 2259 2260 /* 2261 * logarithmic_accumulation - shifted accumulation of cycles 2262 * 2263 * This functions accumulates a shifted interval of cycles into 2264 * a shifted interval nanoseconds. Allows for O(log) accumulation 2265 * loop. 2266 * 2267 * Returns the unconsumed cycles. 2268 */ 2269 static u64 logarithmic_accumulation(struct timekeeper *tk, u64 offset, 2270 u32 shift, unsigned int *clock_set) 2271 { 2272 u64 interval = tk->cycle_interval << shift; 2273 u64 snsec_per_sec; 2274 2275 /* If the offset is smaller than a shifted interval, do nothing */ 2276 if (offset < interval) 2277 return offset; 2278 2279 /* Accumulate one shifted interval */ 2280 offset -= interval; 2281 tk->tkr_mono.cycle_last += interval; 2282 tk->tkr_raw.cycle_last += interval; 2283 2284 tk->tkr_mono.xtime_nsec += tk->xtime_interval << shift; 2285 *clock_set |= accumulate_nsecs_to_secs(tk); 2286 2287 /* Accumulate raw time */ 2288 tk->tkr_raw.xtime_nsec += tk->raw_interval << shift; 2289 snsec_per_sec = (u64)NSEC_PER_SEC << tk->tkr_raw.shift; 2290 while (tk->tkr_raw.xtime_nsec >= snsec_per_sec) { 2291 tk->tkr_raw.xtime_nsec -= snsec_per_sec; 2292 tk->raw_sec++; 2293 } 2294 2295 /* Accumulate error between NTP and clock interval */ 2296 tk->ntp_error += tk->ntp_tick << shift; 2297 tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) << 2298 (tk->ntp_error_shift + shift); 2299 2300 return offset; 2301 } 2302 2303 /* 2304 * timekeeping_advance - Updates the timekeeper to the current time and 2305 * current NTP tick length 2306 */ 2307 static bool __timekeeping_advance(struct tk_data *tkd, enum timekeeping_adv_mode mode) 2308 { 2309 struct timekeeper *tk = &tkd->shadow_timekeeper; 2310 struct timekeeper *real_tk = &tkd->timekeeper; 2311 unsigned int clock_set = 0; 2312 int shift = 0, maxshift; 2313 u64 offset, orig_offset; 2314 2315 /* Make sure we're fully resumed: */ 2316 if (unlikely(timekeeping_suspended)) 2317 return false; 2318 2319 offset = clocksource_delta(tk_clock_read(&tk->tkr_mono), 2320 tk->tkr_mono.cycle_last, tk->tkr_mono.mask, 2321 tk->tkr_mono.clock->max_raw_delta); 2322 orig_offset = offset; 2323 /* Check if there's really nothing to do */ 2324 if (offset < real_tk->cycle_interval && mode == TK_ADV_TICK) 2325 return false; 2326 2327 /* 2328 * With NO_HZ we may have to accumulate many cycle_intervals 2329 * (think "ticks") worth of time at once. To do this efficiently, 2330 * we calculate the largest doubling multiple of cycle_intervals 2331 * that is smaller than the offset. We then accumulate that 2332 * chunk in one go, and then try to consume the next smaller 2333 * doubled multiple. 2334 */ 2335 shift = ilog2(offset) - ilog2(tk->cycle_interval); 2336 shift = max(0, shift); 2337 /* Bound shift to one less than what overflows tick_length */ 2338 maxshift = (64 - (ilog2(ntp_tick_length(tk->id)) + 1)) - 1; 2339 shift = min(shift, maxshift); 2340 while (offset >= tk->cycle_interval) { 2341 offset = logarithmic_accumulation(tk, offset, shift, &clock_set); 2342 if (offset < tk->cycle_interval<<shift) 2343 shift--; 2344 } 2345 2346 /* Adjust the multiplier to correct NTP error */ 2347 timekeeping_adjust(tk, offset); 2348 2349 /* 2350 * Finally, make sure that after the rounding 2351 * xtime_nsec isn't larger than NSEC_PER_SEC 2352 */ 2353 clock_set |= accumulate_nsecs_to_secs(tk); 2354 2355 /* 2356 * To avoid inconsistencies caused adjtimex TK_ADV_FREQ calls 2357 * making small negative adjustments to the base xtime_nsec 2358 * value, only update the coarse clocks if we accumulated time 2359 */ 2360 if (orig_offset != offset) 2361 tk_update_coarse_nsecs(tk); 2362 2363 timekeeping_update_from_shadow(tkd, clock_set); 2364 2365 return !!clock_set; 2366 } 2367 2368 static bool timekeeping_advance(enum timekeeping_adv_mode mode) 2369 { 2370 guard(raw_spinlock_irqsave)(&tk_core.lock); 2371 return __timekeeping_advance(&tk_core, mode); 2372 } 2373 2374 /** 2375 * update_wall_time - Uses the current clocksource to increment the wall time 2376 * 2377 * It also updates the enabled auxiliary clock timekeepers 2378 */ 2379 void update_wall_time(void) 2380 { 2381 if (timekeeping_advance(TK_ADV_TICK)) 2382 clock_was_set_delayed(); 2383 tk_aux_advance(); 2384 } 2385 2386 /** 2387 * getboottime64 - Return the real time of system boot. 2388 * @ts: pointer to the timespec64 to be set 2389 * 2390 * Returns the wall-time of boot in a timespec64. 2391 * 2392 * This is based on the wall_to_monotonic offset and the total suspend 2393 * time. Calls to settimeofday will affect the value returned (which 2394 * basically means that however wrong your real time clock is at boot time, 2395 * you get the right time here). 2396 */ 2397 void getboottime64(struct timespec64 *ts) 2398 { 2399 struct timekeeper *tk = &tk_core.timekeeper; 2400 ktime_t t = ktime_sub(tk->offs_real, tk->offs_boot); 2401 2402 *ts = ktime_to_timespec64(t); 2403 } 2404 EXPORT_SYMBOL_GPL(getboottime64); 2405 2406 void ktime_get_coarse_real_ts64(struct timespec64 *ts) 2407 { 2408 struct timekeeper *tk = &tk_core.timekeeper; 2409 unsigned int seq; 2410 2411 do { 2412 seq = read_seqcount_begin(&tk_core.seq); 2413 2414 *ts = tk_xtime_coarse(tk); 2415 } while (read_seqcount_retry(&tk_core.seq, seq)); 2416 } 2417 EXPORT_SYMBOL(ktime_get_coarse_real_ts64); 2418 2419 /** 2420 * ktime_get_coarse_real_ts64_mg - return latter of coarse grained time or floor 2421 * @ts: timespec64 to be filled 2422 * 2423 * Fetch the global mg_floor value, convert it to realtime and compare it 2424 * to the current coarse-grained time. Fill @ts with whichever is 2425 * latest. Note that this is a filesystem-specific interface and should be 2426 * avoided outside of that context. 2427 */ 2428 void ktime_get_coarse_real_ts64_mg(struct timespec64 *ts) 2429 { 2430 struct timekeeper *tk = &tk_core.timekeeper; 2431 u64 floor = atomic64_read(&mg_floor); 2432 ktime_t f_real, offset, coarse; 2433 unsigned int seq; 2434 2435 do { 2436 seq = read_seqcount_begin(&tk_core.seq); 2437 *ts = tk_xtime_coarse(tk); 2438 offset = tk_core.timekeeper.offs_real; 2439 } while (read_seqcount_retry(&tk_core.seq, seq)); 2440 2441 coarse = timespec64_to_ktime(*ts); 2442 f_real = ktime_add(floor, offset); 2443 if (ktime_after(f_real, coarse)) 2444 *ts = ktime_to_timespec64(f_real); 2445 } 2446 2447 /** 2448 * ktime_get_real_ts64_mg - attempt to update floor value and return result 2449 * @ts: pointer to the timespec to be set 2450 * 2451 * Get a monotonic fine-grained time value and attempt to swap it into 2452 * mg_floor. If that succeeds then accept the new floor value. If it fails 2453 * then another task raced in during the interim time and updated the 2454 * floor. Since any update to the floor must be later than the previous 2455 * floor, either outcome is acceptable. 2456 * 2457 * Typically this will be called after calling ktime_get_coarse_real_ts64_mg(), 2458 * and determining that the resulting coarse-grained timestamp did not effect 2459 * a change in ctime. Any more recent floor value would effect a change to 2460 * ctime, so there is no need to retry the atomic64_try_cmpxchg() on failure. 2461 * 2462 * @ts will be filled with the latest floor value, regardless of the outcome of 2463 * the cmpxchg. Note that this is a filesystem specific interface and should be 2464 * avoided outside of that context. 2465 */ 2466 void ktime_get_real_ts64_mg(struct timespec64 *ts) 2467 { 2468 struct timekeeper *tk = &tk_core.timekeeper; 2469 ktime_t old = atomic64_read(&mg_floor); 2470 ktime_t offset, mono; 2471 unsigned int seq; 2472 u64 nsecs; 2473 2474 do { 2475 seq = read_seqcount_begin(&tk_core.seq); 2476 2477 ts->tv_sec = tk->xtime_sec; 2478 mono = tk->tkr_mono.base; 2479 nsecs = timekeeping_get_ns(&tk->tkr_mono); 2480 offset = tk_core.timekeeper.offs_real; 2481 } while (read_seqcount_retry(&tk_core.seq, seq)); 2482 2483 mono = ktime_add_ns(mono, nsecs); 2484 2485 /* 2486 * Attempt to update the floor with the new time value. As any 2487 * update must be later then the existing floor, and would effect 2488 * a change to ctime from the perspective of the current task, 2489 * accept the resulting floor value regardless of the outcome of 2490 * the swap. 2491 */ 2492 if (atomic64_try_cmpxchg(&mg_floor, &old, mono)) { 2493 ts->tv_nsec = 0; 2494 timespec64_add_ns(ts, nsecs); 2495 timekeeping_inc_mg_floor_swaps(); 2496 } else { 2497 /* 2498 * Another task changed mg_floor since "old" was fetched. 2499 * "old" has been updated with the latest value of "mg_floor". 2500 * That value is newer than the previous floor value, which 2501 * is enough to effect a change to ctime. Accept it. 2502 */ 2503 *ts = ktime_to_timespec64(ktime_add(old, offset)); 2504 } 2505 } 2506 2507 void ktime_get_coarse_ts64(struct timespec64 *ts) 2508 { 2509 struct timekeeper *tk = &tk_core.timekeeper; 2510 struct timespec64 now, mono; 2511 unsigned int seq; 2512 2513 do { 2514 seq = read_seqcount_begin(&tk_core.seq); 2515 2516 now = tk_xtime_coarse(tk); 2517 mono = tk->wall_to_monotonic; 2518 } while (read_seqcount_retry(&tk_core.seq, seq)); 2519 2520 set_normalized_timespec64(ts, now.tv_sec + mono.tv_sec, 2521 now.tv_nsec + mono.tv_nsec); 2522 } 2523 EXPORT_SYMBOL(ktime_get_coarse_ts64); 2524 2525 /* 2526 * Must hold jiffies_lock 2527 */ 2528 void do_timer(unsigned long ticks) 2529 { 2530 jiffies_64 += ticks; 2531 calc_global_load(); 2532 } 2533 2534 /** 2535 * ktime_get_update_offsets_now - hrtimer helper 2536 * @cwsseq: pointer to check and store the clock was set sequence number 2537 * @offs_real: pointer to storage for monotonic -> realtime offset 2538 * @offs_boot: pointer to storage for monotonic -> boottime offset 2539 * @offs_tai: pointer to storage for monotonic -> clock tai offset 2540 * 2541 * Returns current monotonic time and updates the offsets if the 2542 * sequence number in @cwsseq and timekeeper.clock_was_set_seq are 2543 * different. 2544 * 2545 * Called from hrtimer_interrupt() or retrigger_next_event() 2546 */ 2547 ktime_t ktime_get_update_offsets_now(unsigned int *cwsseq, ktime_t *offs_real, 2548 ktime_t *offs_boot, ktime_t *offs_tai) 2549 { 2550 struct timekeeper *tk = &tk_core.timekeeper; 2551 unsigned int seq; 2552 ktime_t base; 2553 u64 nsecs; 2554 2555 do { 2556 seq = read_seqcount_begin(&tk_core.seq); 2557 2558 base = tk->tkr_mono.base; 2559 nsecs = timekeeping_get_ns(&tk->tkr_mono); 2560 base = ktime_add_ns(base, nsecs); 2561 2562 if (*cwsseq != tk->clock_was_set_seq) { 2563 *cwsseq = tk->clock_was_set_seq; 2564 *offs_real = tk->offs_real; 2565 *offs_boot = tk->offs_boot; 2566 *offs_tai = tk->offs_tai; 2567 } 2568 2569 /* Handle leapsecond insertion adjustments */ 2570 if (unlikely(base >= tk->next_leap_ktime)) 2571 *offs_real = ktime_sub(tk->offs_real, ktime_set(1, 0)); 2572 2573 } while (read_seqcount_retry(&tk_core.seq, seq)); 2574 2575 return base; 2576 } 2577 2578 /* 2579 * timekeeping_validate_timex - Ensures the timex is ok for use in do_adjtimex 2580 */ 2581 static int timekeeping_validate_timex(const struct __kernel_timex *txc, bool aux_clock) 2582 { 2583 if (txc->modes & ADJ_ADJTIME) { 2584 /* singleshot must not be used with any other mode bits */ 2585 if (!(txc->modes & ADJ_OFFSET_SINGLESHOT)) 2586 return -EINVAL; 2587 if (!(txc->modes & ADJ_OFFSET_READONLY) && 2588 !capable(CAP_SYS_TIME)) 2589 return -EPERM; 2590 } else { 2591 /* In order to modify anything, you gotta be super-user! */ 2592 if (txc->modes && !capable(CAP_SYS_TIME)) 2593 return -EPERM; 2594 /* 2595 * if the quartz is off by more than 10% then 2596 * something is VERY wrong! 2597 */ 2598 if (txc->modes & ADJ_TICK && 2599 (txc->tick < 900000/USER_HZ || 2600 txc->tick > 1100000/USER_HZ)) 2601 return -EINVAL; 2602 } 2603 2604 if (txc->modes & ADJ_SETOFFSET) { 2605 /* In order to inject time, you gotta be super-user! */ 2606 if (!capable(CAP_SYS_TIME)) 2607 return -EPERM; 2608 2609 /* 2610 * Validate if a timespec/timeval used to inject a time 2611 * offset is valid. Offsets can be positive or negative, so 2612 * we don't check tv_sec. The value of the timeval/timespec 2613 * is the sum of its fields,but *NOTE*: 2614 * The field tv_usec/tv_nsec must always be non-negative and 2615 * we can't have more nanoseconds/microseconds than a second. 2616 */ 2617 if (txc->time.tv_usec < 0) 2618 return -EINVAL; 2619 2620 if (txc->modes & ADJ_NANO) { 2621 if (txc->time.tv_usec >= NSEC_PER_SEC) 2622 return -EINVAL; 2623 } else { 2624 if (txc->time.tv_usec >= USEC_PER_SEC) 2625 return -EINVAL; 2626 } 2627 } 2628 2629 /* 2630 * Check for potential multiplication overflows that can 2631 * only happen on 64-bit systems: 2632 */ 2633 if ((txc->modes & ADJ_FREQUENCY) && (BITS_PER_LONG == 64)) { 2634 if (LLONG_MIN / PPM_SCALE > txc->freq) 2635 return -EINVAL; 2636 if (LLONG_MAX / PPM_SCALE < txc->freq) 2637 return -EINVAL; 2638 } 2639 2640 if (aux_clock) { 2641 /* Auxiliary clocks are similar to TAI and do not have leap seconds */ 2642 if (txc->modes & ADJ_STATUS && 2643 txc->status & (STA_INS | STA_DEL)) 2644 return -EINVAL; 2645 2646 /* No TAI offset setting */ 2647 if (txc->modes & ADJ_TAI) 2648 return -EINVAL; 2649 2650 /* No PPS support either */ 2651 if (txc->modes & ADJ_STATUS && 2652 txc->status & (STA_PPSFREQ | STA_PPSTIME)) 2653 return -EINVAL; 2654 } 2655 2656 return 0; 2657 } 2658 2659 /** 2660 * random_get_entropy_fallback - Returns the raw clock source value, 2661 * used by random.c for platforms with no valid random_get_entropy(). 2662 */ 2663 unsigned long random_get_entropy_fallback(void) 2664 { 2665 struct tk_read_base *tkr = &tk_core.timekeeper.tkr_mono; 2666 struct clocksource *clock = READ_ONCE(tkr->clock); 2667 2668 if (unlikely(timekeeping_suspended || !clock)) 2669 return 0; 2670 return clock->read(clock); 2671 } 2672 EXPORT_SYMBOL_GPL(random_get_entropy_fallback); 2673 2674 struct adjtimex_result { 2675 struct audit_ntp_data ad; 2676 struct timespec64 delta; 2677 bool clock_set; 2678 }; 2679 2680 static int __do_adjtimex(struct tk_data *tkd, struct __kernel_timex *txc, 2681 struct adjtimex_result *result) 2682 { 2683 struct timekeeper *tks = &tkd->shadow_timekeeper; 2684 bool aux_clock = !timekeeper_is_core_tk(tks); 2685 struct timespec64 ts; 2686 s32 orig_tai, tai; 2687 int ret; 2688 2689 /* Validate the data before disabling interrupts */ 2690 ret = timekeeping_validate_timex(txc, aux_clock); 2691 if (ret) 2692 return ret; 2693 add_device_randomness(txc, sizeof(*txc)); 2694 2695 if (!aux_clock) 2696 ktime_get_real_ts64(&ts); 2697 else 2698 tk_get_aux_ts64(tkd->timekeeper.id, &ts); 2699 2700 add_device_randomness(&ts, sizeof(ts)); 2701 2702 guard(raw_spinlock_irqsave)(&tkd->lock); 2703 2704 if (!tks->clock_valid) 2705 return -ENODEV; 2706 2707 if (txc->modes & ADJ_SETOFFSET) { 2708 result->delta.tv_sec = txc->time.tv_sec; 2709 result->delta.tv_nsec = txc->time.tv_usec; 2710 if (!(txc->modes & ADJ_NANO)) 2711 result->delta.tv_nsec *= 1000; 2712 ret = __timekeeping_inject_offset(tkd, &result->delta); 2713 if (ret) 2714 return ret; 2715 result->clock_set = true; 2716 } 2717 2718 orig_tai = tai = tks->tai_offset; 2719 ret = ntp_adjtimex(tks->id, txc, &ts, &tai, &result->ad); 2720 2721 if (tai != orig_tai) { 2722 __timekeeping_set_tai_offset(tks, tai); 2723 timekeeping_update_from_shadow(tkd, TK_CLOCK_WAS_SET); 2724 result->clock_set = true; 2725 } else { 2726 tk_update_leap_state_all(tkd); 2727 } 2728 2729 /* Update the multiplier immediately if frequency was set directly */ 2730 if (txc->modes & (ADJ_FREQUENCY | ADJ_TICK)) 2731 result->clock_set |= __timekeeping_advance(tkd, TK_ADV_FREQ); 2732 2733 return ret; 2734 } 2735 2736 /** 2737 * do_adjtimex() - Accessor function to NTP __do_adjtimex function 2738 * @txc: Pointer to kernel_timex structure containing NTP parameters 2739 */ 2740 int do_adjtimex(struct __kernel_timex *txc) 2741 { 2742 struct adjtimex_result result = { }; 2743 int ret; 2744 2745 ret = __do_adjtimex(&tk_core, txc, &result); 2746 if (ret < 0) 2747 return ret; 2748 2749 if (txc->modes & ADJ_SETOFFSET) 2750 audit_tk_injoffset(result.delta); 2751 2752 audit_ntp_log(&result.ad); 2753 2754 if (result.clock_set) 2755 clock_was_set(CLOCK_SET_WALL); 2756 2757 ntp_notify_cmos_timer(result.delta.tv_sec != 0); 2758 2759 return ret; 2760 } 2761 2762 /* 2763 * Invoked from NTP with the time keeper lock held, so lockless access is 2764 * fine. 2765 */ 2766 long ktime_get_ntp_seconds(unsigned int id) 2767 { 2768 return timekeeper_data[id].timekeeper.xtime_sec; 2769 } 2770 2771 #ifdef CONFIG_NTP_PPS 2772 /** 2773 * hardpps() - Accessor function to NTP __hardpps function 2774 * @phase_ts: Pointer to timespec64 structure representing phase timestamp 2775 * @raw_ts: Pointer to timespec64 structure representing raw timestamp 2776 */ 2777 void hardpps(const struct timespec64 *phase_ts, const struct timespec64 *raw_ts) 2778 { 2779 guard(raw_spinlock_irqsave)(&tk_core.lock); 2780 __hardpps(phase_ts, raw_ts); 2781 } 2782 EXPORT_SYMBOL(hardpps); 2783 #endif /* CONFIG_NTP_PPS */ 2784 2785 #ifdef CONFIG_POSIX_AUX_CLOCKS 2786 #include "posix-timers.h" 2787 2788 /* 2789 * Bitmap for the activated auxiliary timekeepers to allow lockless quick 2790 * checks in the hot paths without touching extra cache lines. If set, then 2791 * the state of the corresponding timekeeper has to be re-checked under 2792 * timekeeper::lock. 2793 */ 2794 static unsigned long aux_timekeepers; 2795 2796 static inline unsigned int clockid_to_tkid(unsigned int id) 2797 { 2798 return TIMEKEEPER_AUX_FIRST + id - CLOCK_AUX; 2799 } 2800 2801 static inline struct tk_data *aux_get_tk_data(clockid_t id) 2802 { 2803 if (!clockid_aux_valid(id)) 2804 return NULL; 2805 return &timekeeper_data[clockid_to_tkid(id)]; 2806 } 2807 2808 /* Invoked from timekeeping after a clocksource change */ 2809 static void tk_aux_update_clocksource(void) 2810 { 2811 unsigned long active = READ_ONCE(aux_timekeepers); 2812 unsigned int id; 2813 2814 for_each_set_bit(id, &active, BITS_PER_LONG) { 2815 struct tk_data *tkd = &timekeeper_data[id + TIMEKEEPER_AUX_FIRST]; 2816 struct timekeeper *tks = &tkd->shadow_timekeeper; 2817 2818 guard(raw_spinlock_irqsave)(&tkd->lock); 2819 if (!tks->clock_valid) 2820 continue; 2821 2822 timekeeping_forward_now(tks); 2823 tk_setup_internals(tks, tk_core.timekeeper.tkr_mono.clock); 2824 timekeeping_update_from_shadow(tkd, TK_UPDATE_ALL); 2825 } 2826 } 2827 2828 static void tk_aux_advance(void) 2829 { 2830 unsigned long active = READ_ONCE(aux_timekeepers); 2831 unsigned int id; 2832 2833 /* Lockless quick check to avoid extra cache lines */ 2834 for_each_set_bit(id, &active, BITS_PER_LONG) { 2835 struct tk_data *aux_tkd = &timekeeper_data[id + TIMEKEEPER_AUX_FIRST]; 2836 2837 guard(raw_spinlock)(&aux_tkd->lock); 2838 if (aux_tkd->shadow_timekeeper.clock_valid) 2839 __timekeeping_advance(aux_tkd, TK_ADV_TICK); 2840 } 2841 } 2842 2843 /** 2844 * ktime_get_aux - Get time for a AUX clock 2845 * @id: ID of the clock to read (CLOCK_AUX...) 2846 * @kt: Pointer to ktime_t to store the time stamp 2847 * 2848 * Returns: True if the timestamp is valid, false otherwise 2849 */ 2850 bool ktime_get_aux(clockid_t id, ktime_t *kt) 2851 { 2852 struct tk_data *aux_tkd = aux_get_tk_data(id); 2853 struct timekeeper *aux_tk; 2854 unsigned int seq; 2855 ktime_t base; 2856 u64 nsecs; 2857 2858 WARN_ON(timekeeping_suspended); 2859 2860 if (!aux_tkd) 2861 return false; 2862 2863 aux_tk = &aux_tkd->timekeeper; 2864 do { 2865 seq = read_seqcount_begin(&aux_tkd->seq); 2866 if (!aux_tk->clock_valid) 2867 return false; 2868 2869 base = ktime_add(aux_tk->tkr_mono.base, aux_tk->offs_aux); 2870 nsecs = timekeeping_get_ns(&aux_tk->tkr_mono); 2871 } while (read_seqcount_retry(&aux_tkd->seq, seq)); 2872 2873 *kt = ktime_add_ns(base, nsecs); 2874 return true; 2875 } 2876 EXPORT_SYMBOL_GPL(ktime_get_aux); 2877 2878 /** 2879 * ktime_get_aux_ts64 - Get time for a AUX clock 2880 * @id: ID of the clock to read (CLOCK_AUX...) 2881 * @ts: Pointer to timespec64 to store the time stamp 2882 * 2883 * Returns: True if the timestamp is valid, false otherwise 2884 */ 2885 bool ktime_get_aux_ts64(clockid_t id, struct timespec64 *ts) 2886 { 2887 ktime_t now; 2888 2889 if (!ktime_get_aux(id, &now)) 2890 return false; 2891 *ts = ktime_to_timespec64(now); 2892 return true; 2893 } 2894 EXPORT_SYMBOL_GPL(ktime_get_aux_ts64); 2895 2896 static int aux_get_res(clockid_t id, struct timespec64 *tp) 2897 { 2898 if (!clockid_aux_valid(id)) 2899 return -ENODEV; 2900 2901 tp->tv_sec = aux_clock_resolution_ns() / NSEC_PER_SEC; 2902 tp->tv_nsec = aux_clock_resolution_ns() % NSEC_PER_SEC; 2903 return 0; 2904 } 2905 2906 static int aux_get_timespec(clockid_t id, struct timespec64 *tp) 2907 { 2908 return ktime_get_aux_ts64(id, tp) ? 0 : -ENODEV; 2909 } 2910 2911 static int aux_clock_set(const clockid_t id, const struct timespec64 *tnew) 2912 { 2913 struct tk_data *aux_tkd = aux_get_tk_data(id); 2914 struct timekeeper *aux_tks; 2915 ktime_t tnow, nsecs; 2916 2917 if (!timespec64_valid_settod(tnew)) 2918 return -EINVAL; 2919 if (!aux_tkd) 2920 return -ENODEV; 2921 2922 aux_tks = &aux_tkd->shadow_timekeeper; 2923 2924 guard(raw_spinlock_irq)(&aux_tkd->lock); 2925 if (!aux_tks->clock_valid) 2926 return -ENODEV; 2927 2928 /* Forward the timekeeper base time */ 2929 timekeeping_forward_now(aux_tks); 2930 /* 2931 * Get the updated base time. tkr_mono.base has not been 2932 * updated yet, so do that first. That makes the update 2933 * in timekeeping_update_from_shadow() redundant, but 2934 * that's harmless. After that @tnow can be calculated 2935 * by using tkr_mono::cycle_last, which has been set 2936 * by timekeeping_forward_now(). 2937 */ 2938 tk_update_ktime_data(aux_tks); 2939 nsecs = timekeeping_cycles_to_ns(&aux_tks->tkr_mono, aux_tks->tkr_mono.cycle_last); 2940 tnow = ktime_add(aux_tks->tkr_mono.base, nsecs); 2941 2942 /* 2943 * Calculate the new AUX offset as delta to @tnow ("monotonic"). 2944 * That avoids all the tk::xtime back and forth conversions as 2945 * xtime ("realtime") is not applicable for auxiliary clocks and 2946 * kept in sync with "monotonic". 2947 */ 2948 tk_update_aux_offs(aux_tks, ktime_sub(timespec64_to_ktime(*tnew), tnow)); 2949 2950 timekeeping_update_from_shadow(aux_tkd, TK_UPDATE_ALL); 2951 return 0; 2952 } 2953 2954 static int aux_clock_adj(const clockid_t id, struct __kernel_timex *txc) 2955 { 2956 struct tk_data *aux_tkd = aux_get_tk_data(id); 2957 struct adjtimex_result result = { }; 2958 2959 if (!aux_tkd) 2960 return -ENODEV; 2961 2962 /* 2963 * @result is ignored for now as there are neither hrtimers nor a 2964 * RTC related to auxiliary clocks for now. 2965 */ 2966 return __do_adjtimex(aux_tkd, txc, &result); 2967 } 2968 2969 const struct k_clock clock_aux = { 2970 .clock_getres = aux_get_res, 2971 .clock_get_timespec = aux_get_timespec, 2972 .clock_set = aux_clock_set, 2973 .clock_adj = aux_clock_adj, 2974 }; 2975 2976 static void aux_clock_enable(clockid_t id) 2977 { 2978 struct tk_read_base *tkr_raw = &tk_core.timekeeper.tkr_raw; 2979 struct tk_data *aux_tkd = aux_get_tk_data(id); 2980 struct timekeeper *aux_tks = &aux_tkd->shadow_timekeeper; 2981 2982 /* Prevent the core timekeeper from changing. */ 2983 guard(raw_spinlock_irq)(&tk_core.lock); 2984 2985 /* 2986 * Setup the auxiliary clock assuming that the raw core timekeeper 2987 * clock frequency conversion is close enough. Userspace has to 2988 * adjust for the deviation via clock_adjtime(2). 2989 */ 2990 guard(raw_spinlock_nested)(&aux_tkd->lock); 2991 2992 /* Remove leftovers of a previous registration */ 2993 memset(aux_tks, 0, sizeof(*aux_tks)); 2994 /* Restore the timekeeper id */ 2995 aux_tks->id = aux_tkd->timekeeper.id; 2996 /* Setup the timekeeper based on the current system clocksource */ 2997 tk_setup_internals(aux_tks, tkr_raw->clock); 2998 2999 /* Mark it valid and set it live */ 3000 aux_tks->clock_valid = true; 3001 timekeeping_update_from_shadow(aux_tkd, TK_UPDATE_ALL); 3002 } 3003 3004 static void aux_clock_disable(clockid_t id) 3005 { 3006 struct tk_data *aux_tkd = aux_get_tk_data(id); 3007 3008 guard(raw_spinlock_irq)(&aux_tkd->lock); 3009 aux_tkd->shadow_timekeeper.clock_valid = false; 3010 timekeeping_update_from_shadow(aux_tkd, TK_UPDATE_ALL); 3011 } 3012 3013 static DEFINE_MUTEX(aux_clock_mutex); 3014 3015 static ssize_t aux_clock_enable_store(struct kobject *kobj, struct kobj_attribute *attr, 3016 const char *buf, size_t count) 3017 { 3018 /* Lazy atoi() as name is "0..7" */ 3019 int id = kobj->name[0] & 0x7; 3020 bool enable; 3021 3022 if (!capable(CAP_SYS_TIME)) 3023 return -EPERM; 3024 3025 if (kstrtobool(buf, &enable) < 0) 3026 return -EINVAL; 3027 3028 guard(mutex)(&aux_clock_mutex); 3029 if (enable == test_bit(id, &aux_timekeepers)) 3030 return count; 3031 3032 if (enable) { 3033 aux_clock_enable(CLOCK_AUX + id); 3034 set_bit(id, &aux_timekeepers); 3035 } else { 3036 aux_clock_disable(CLOCK_AUX + id); 3037 clear_bit(id, &aux_timekeepers); 3038 } 3039 return count; 3040 } 3041 3042 static ssize_t aux_clock_enable_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) 3043 { 3044 unsigned long active = READ_ONCE(aux_timekeepers); 3045 /* Lazy atoi() as name is "0..7" */ 3046 int id = kobj->name[0] & 0x7; 3047 3048 return sysfs_emit(buf, "%d\n", test_bit(id, &active)); 3049 } 3050 3051 static struct kobj_attribute aux_clock_enable_attr = __ATTR_RW(aux_clock_enable); 3052 3053 static struct attribute *aux_clock_enable_attrs[] = { 3054 &aux_clock_enable_attr.attr, 3055 NULL 3056 }; 3057 3058 static const struct attribute_group aux_clock_enable_attr_group = { 3059 .attrs = aux_clock_enable_attrs, 3060 }; 3061 3062 static int __init tk_aux_sysfs_init(void) 3063 { 3064 struct kobject *auxo, *tko = kobject_create_and_add("time", kernel_kobj); 3065 int ret = -ENOMEM; 3066 3067 if (!tko) 3068 return ret; 3069 3070 auxo = kobject_create_and_add("aux_clocks", tko); 3071 if (!auxo) 3072 goto err_clean; 3073 3074 for (int i = 0; i < MAX_AUX_CLOCKS; i++) { 3075 char id[2] = { [0] = '0' + i, }; 3076 struct kobject *clk = kobject_create_and_add(id, auxo); 3077 3078 if (!clk) { 3079 ret = -ENOMEM; 3080 goto err_clean; 3081 } 3082 3083 ret = sysfs_create_group(clk, &aux_clock_enable_attr_group); 3084 if (ret) 3085 goto err_clean; 3086 } 3087 return 0; 3088 3089 err_clean: 3090 kobject_put(auxo); 3091 kobject_put(tko); 3092 return ret; 3093 } 3094 late_initcall(tk_aux_sysfs_init); 3095 3096 static __init void tk_aux_setup(void) 3097 { 3098 for (int i = TIMEKEEPER_AUX_FIRST; i <= TIMEKEEPER_AUX_LAST; i++) 3099 tkd_basic_setup(&timekeeper_data[i], i, false); 3100 } 3101 #endif /* CONFIG_POSIX_AUX_CLOCKS */