개념 설명 전체 · v6.18.37 / arch/arm64/kvm/arm.c
1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 - Virtual Open Systems and Columbia University 4 * Author: Christoffer Dall <c.dall@virtualopensystems.com> 5 */ 6 7 #include <linux/bug.h> 8 #include <linux/cpu_pm.h> 9 #include <linux/errno.h> 10 #include <linux/err.h> 11 #include <linux/kvm_host.h> 12 #include <linux/list.h> 13 #include <linux/module.h> 14 #include <linux/vmalloc.h> 15 #include <linux/fs.h> 16 #include <linux/mman.h> 17 #include <linux/sched.h> 18 #include <linux/kvm.h> 19 #include <linux/kvm_irqfd.h> 20 #include <linux/irqbypass.h> 21 #include <linux/sched/stat.h> 22 #include <linux/psci.h> 23 #include <trace/events/kvm.h> 24 25 #define CREATE_TRACE_POINTS 26 #include "trace_arm.h" 27 28 #include <linux/uaccess.h> 29 #include <asm/ptrace.h> 30 #include <asm/mman.h> 31 #include <asm/tlbflush.h> 32 #include <asm/cacheflush.h> 33 #include <asm/cpufeature.h> 34 #include <asm/virt.h> 35 #include <asm/kvm_arm.h> 36 #include <asm/kvm_asm.h> 37 #include <asm/kvm_emulate.h> 38 #include <asm/kvm_mmu.h> 39 #include <asm/kvm_nested.h> 40 #include <asm/kvm_pkvm.h> 41 #include <asm/kvm_ptrauth.h> 42 #include <asm/sections.h> 43 44 #include <kvm/arm_hypercalls.h> 45 #include <kvm/arm_pmu.h> 46 #include <kvm/arm_psci.h> 47 48 #include "sys_regs.h" 49 50 static enum kvm_mode kvm_mode = KVM_MODE_DEFAULT; 51 52 enum kvm_wfx_trap_policy { 53 KVM_WFX_NOTRAP_SINGLE_TASK, /* Default option */ 54 KVM_WFX_NOTRAP, 55 KVM_WFX_TRAP, 56 }; 57 58 static enum kvm_wfx_trap_policy kvm_wfi_trap_policy __read_mostly = KVM_WFX_NOTRAP_SINGLE_TASK; 59 static enum kvm_wfx_trap_policy kvm_wfe_trap_policy __read_mostly = KVM_WFX_NOTRAP_SINGLE_TASK; 60 61 DECLARE_KVM_HYP_PER_CPU(unsigned long, kvm_hyp_vector); 62 63 DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_base); 64 DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params); 65 66 DECLARE_KVM_NVHE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt); 67 68 static bool vgic_present, kvm_arm_initialised; 69 70 static DEFINE_PER_CPU(unsigned char, kvm_hyp_initialized); 71 72 bool is_kvm_arm_initialised(void) 73 { 74 return kvm_arm_initialised; 75 } 76 77 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) 78 { 79 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE; 80 } 81 82 int kvm_vm_ioctl_enable_cap(struct kvm *kvm, 83 struct kvm_enable_cap *cap) 84 { 85 int r = -EINVAL; 86 87 if (cap->flags) 88 return -EINVAL; 89 90 if (kvm_vm_is_protected(kvm) && !kvm_pvm_ext_allowed(cap->cap)) 91 return -EINVAL; 92 93 switch (cap->cap) { 94 case KVM_CAP_ARM_NISV_TO_USER: 95 r = 0; 96 set_bit(KVM_ARCH_FLAG_RETURN_NISV_IO_ABORT_TO_USER, 97 &kvm->arch.flags); 98 break; 99 case KVM_CAP_ARM_MTE: 100 mutex_lock(&kvm->lock); 101 if (system_supports_mte() && !kvm->created_vcpus) { 102 r = 0; 103 set_bit(KVM_ARCH_FLAG_MTE_ENABLED, &kvm->arch.flags); 104 } 105 mutex_unlock(&kvm->lock); 106 break; 107 case KVM_CAP_ARM_SYSTEM_SUSPEND: 108 r = 0; 109 set_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags); 110 break; 111 case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE: 112 mutex_lock(&kvm->slots_lock); 113 /* 114 * To keep things simple, allow changing the chunk 115 * size only when no memory slots have been created. 116 */ 117 if (kvm_are_all_memslots_empty(kvm)) { 118 u64 new_cap = cap->args[0]; 119 120 if (!new_cap || kvm_is_block_size_supported(new_cap)) { 121 r = 0; 122 kvm->arch.mmu.split_page_chunk_size = new_cap; 123 } 124 } 125 mutex_unlock(&kvm->slots_lock); 126 break; 127 case KVM_CAP_ARM_WRITABLE_IMP_ID_REGS: 128 mutex_lock(&kvm->lock); 129 if (!kvm->created_vcpus) { 130 r = 0; 131 set_bit(KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS, &kvm->arch.flags); 132 } 133 mutex_unlock(&kvm->lock); 134 break; 135 default: 136 break; 137 } 138 139 return r; 140 } 141 142 static int kvm_arm_default_max_vcpus(void) 143 { 144 return vgic_present ? kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS; 145 } 146 147 /** 148 * kvm_arch_init_vm - initializes a VM data structure 149 * @kvm: pointer to the KVM struct 150 * @type: kvm device type 151 */ 152 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) 153 { 154 int ret; 155 156 mutex_init(&kvm->arch.config_lock); 157 158 #ifdef CONFIG_LOCKDEP 159 /* Clue in lockdep that the config_lock must be taken inside kvm->lock */ 160 mutex_lock(&kvm->lock); 161 mutex_lock(&kvm->arch.config_lock); 162 mutex_unlock(&kvm->arch.config_lock); 163 mutex_unlock(&kvm->lock); 164 #endif 165 166 kvm_init_nested(kvm); 167 168 ret = kvm_share_hyp(kvm, kvm + 1); 169 if (ret) 170 return ret; 171 172 if (!zalloc_cpumask_var(&kvm->arch.supported_cpus, GFP_KERNEL_ACCOUNT)) { 173 ret = -ENOMEM; 174 goto err_unshare_kvm; 175 } 176 cpumask_copy(kvm->arch.supported_cpus, cpu_possible_mask); 177 178 ret = kvm_init_stage2_mmu(kvm, &kvm->arch.mmu, type); 179 if (ret) 180 goto err_free_cpumask; 181 182 if (is_protected_kvm_enabled()) { 183 /* 184 * If any failures occur after this is successful, make sure to 185 * call __pkvm_unreserve_vm to unreserve the VM in hyp. 186 */ 187 ret = pkvm_init_host_vm(kvm); 188 if (ret) 189 goto err_free_cpumask; 190 } 191 192 kvm_vgic_early_init(kvm); 193 194 kvm_timer_init_vm(kvm); 195 196 /* The maximum number of VCPUs is limited by the host's GIC model */ 197 kvm->max_vcpus = kvm_arm_default_max_vcpus(); 198 199 kvm_arm_init_hypercalls(kvm); 200 201 bitmap_zero(kvm->arch.vcpu_features, KVM_VCPU_MAX_FEATURES); 202 203 return 0; 204 205 err_free_cpumask: 206 free_cpumask_var(kvm->arch.supported_cpus); 207 err_unshare_kvm: 208 kvm_unshare_hyp(kvm, kvm + 1); 209 return ret; 210 } 211 212 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) 213 { 214 return VM_FAULT_SIGBUS; 215 } 216 217 void kvm_arch_create_vm_debugfs(struct kvm *kvm) 218 { 219 kvm_sys_regs_create_debugfs(kvm); 220 kvm_s2_ptdump_create_debugfs(kvm); 221 } 222 223 static void kvm_destroy_mpidr_data(struct kvm *kvm) 224 { 225 struct kvm_mpidr_data *data; 226 227 mutex_lock(&kvm->arch.config_lock); 228 229 data = rcu_dereference_protected(kvm->arch.mpidr_data, 230 lockdep_is_held(&kvm->arch.config_lock)); 231 if (data) { 232 rcu_assign_pointer(kvm->arch.mpidr_data, NULL); 233 synchronize_rcu(); 234 kfree(data); 235 } 236 237 mutex_unlock(&kvm->arch.config_lock); 238 } 239 240 /** 241 * kvm_arch_destroy_vm - destroy the VM data structure 242 * @kvm: pointer to the KVM struct 243 */ 244 void kvm_arch_destroy_vm(struct kvm *kvm) 245 { 246 bitmap_free(kvm->arch.pmu_filter); 247 free_cpumask_var(kvm->arch.supported_cpus); 248 249 kvm_vgic_destroy(kvm); 250 251 if (is_protected_kvm_enabled()) 252 pkvm_destroy_hyp_vm(kvm); 253 254 kvm_destroy_mpidr_data(kvm); 255 256 kfree(kvm->arch.sysreg_masks); 257 kvm_destroy_vcpus(kvm); 258 259 kvm_unshare_hyp(kvm, kvm + 1); 260 261 kvm_arm_teardown_hypercalls(kvm); 262 } 263 264 static bool kvm_has_full_ptr_auth(void) 265 { 266 bool apa, gpa, api, gpi, apa3, gpa3; 267 u64 isar1, isar2, val; 268 269 /* 270 * Check that: 271 * 272 * - both Address and Generic auth are implemented for a given 273 * algorithm (Q5, IMPDEF or Q3) 274 * - only a single algorithm is implemented. 275 */ 276 if (!system_has_full_ptr_auth()) 277 return false; 278 279 isar1 = read_sanitised_ftr_reg(SYS_ID_AA64ISAR1_EL1); 280 isar2 = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1); 281 282 apa = !!FIELD_GET(ID_AA64ISAR1_EL1_APA_MASK, isar1); 283 val = FIELD_GET(ID_AA64ISAR1_EL1_GPA_MASK, isar1); 284 gpa = (val == ID_AA64ISAR1_EL1_GPA_IMP); 285 286 api = !!FIELD_GET(ID_AA64ISAR1_EL1_API_MASK, isar1); 287 val = FIELD_GET(ID_AA64ISAR1_EL1_GPI_MASK, isar1); 288 gpi = (val == ID_AA64ISAR1_EL1_GPI_IMP); 289 290 apa3 = !!FIELD_GET(ID_AA64ISAR2_EL1_APA3_MASK, isar2); 291 val = FIELD_GET(ID_AA64ISAR2_EL1_GPA3_MASK, isar2); 292 gpa3 = (val == ID_AA64ISAR2_EL1_GPA3_IMP); 293 294 return (apa == gpa && api == gpi && apa3 == gpa3 && 295 (apa + api + apa3) == 1); 296 } 297 298 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) 299 { 300 int r; 301 302 if (kvm && kvm_vm_is_protected(kvm) && !kvm_pvm_ext_allowed(ext)) 303 return 0; 304 305 switch (ext) { 306 case KVM_CAP_IRQCHIP: 307 r = vgic_present; 308 break; 309 case KVM_CAP_IOEVENTFD: 310 case KVM_CAP_USER_MEMORY: 311 case KVM_CAP_SYNC_MMU: 312 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: 313 case KVM_CAP_ONE_REG: 314 case KVM_CAP_ARM_PSCI: 315 case KVM_CAP_ARM_PSCI_0_2: 316 case KVM_CAP_READONLY_MEM: 317 case KVM_CAP_MP_STATE: 318 case KVM_CAP_IMMEDIATE_EXIT: 319 case KVM_CAP_VCPU_EVENTS: 320 case KVM_CAP_ARM_IRQ_LINE_LAYOUT_2: 321 case KVM_CAP_ARM_NISV_TO_USER: 322 case KVM_CAP_ARM_INJECT_EXT_DABT: 323 case KVM_CAP_SET_GUEST_DEBUG: 324 case KVM_CAP_VCPU_ATTRIBUTES: 325 case KVM_CAP_PTP_KVM: 326 case KVM_CAP_ARM_SYSTEM_SUSPEND: 327 case KVM_CAP_IRQFD_RESAMPLE: 328 case KVM_CAP_COUNTER_OFFSET: 329 case KVM_CAP_ARM_WRITABLE_IMP_ID_REGS: 330 r = 1; 331 break; 332 case KVM_CAP_SET_GUEST_DEBUG2: 333 return KVM_GUESTDBG_VALID_MASK; 334 case KVM_CAP_ARM_SET_DEVICE_ADDR: 335 r = 1; 336 break; 337 case KVM_CAP_NR_VCPUS: 338 /* 339 * ARM64 treats KVM_CAP_NR_CPUS differently from all other 340 * architectures, as it does not always bound it to 341 * KVM_CAP_MAX_VCPUS. It should not matter much because 342 * this is just an advisory value. 343 */ 344 r = min_t(unsigned int, num_online_cpus(), 345 kvm_arm_default_max_vcpus()); 346 break; 347 case KVM_CAP_MAX_VCPUS: 348 case KVM_CAP_MAX_VCPU_ID: 349 if (kvm) 350 r = kvm->max_vcpus; 351 else 352 r = kvm_arm_default_max_vcpus(); 353 break; 354 case KVM_CAP_MSI_DEVID: 355 if (!kvm) 356 r = -EINVAL; 357 else 358 r = kvm->arch.vgic.msis_require_devid; 359 break; 360 case KVM_CAP_ARM_USER_IRQ: 361 /* 362 * 1: EL1_VTIMER, EL1_PTIMER, and PMU. 363 * (bump this number if adding more devices) 364 */ 365 r = 1; 366 break; 367 case KVM_CAP_ARM_MTE: 368 r = system_supports_mte(); 369 break; 370 case KVM_CAP_STEAL_TIME: 371 r = kvm_arm_pvtime_supported(); 372 break; 373 case KVM_CAP_ARM_EL1_32BIT: 374 r = cpus_have_final_cap(ARM64_HAS_32BIT_EL1); 375 break; 376 case KVM_CAP_ARM_EL2: 377 r = cpus_have_final_cap(ARM64_HAS_NESTED_VIRT); 378 break; 379 case KVM_CAP_ARM_EL2_E2H0: 380 r = cpus_have_final_cap(ARM64_HAS_HCR_NV1); 381 break; 382 case KVM_CAP_GUEST_DEBUG_HW_BPS: 383 r = get_num_brps(); 384 break; 385 case KVM_CAP_GUEST_DEBUG_HW_WPS: 386 r = get_num_wrps(); 387 break; 388 case KVM_CAP_ARM_PMU_V3: 389 r = kvm_supports_guest_pmuv3(); 390 break; 391 case KVM_CAP_ARM_INJECT_SERROR_ESR: 392 r = cpus_have_final_cap(ARM64_HAS_RAS_EXTN); 393 break; 394 case KVM_CAP_ARM_VM_IPA_SIZE: 395 r = get_kvm_ipa_limit(); 396 break; 397 case KVM_CAP_ARM_SVE: 398 r = system_supports_sve(); 399 break; 400 case KVM_CAP_ARM_PTRAUTH_ADDRESS: 401 case KVM_CAP_ARM_PTRAUTH_GENERIC: 402 r = kvm_has_full_ptr_auth(); 403 break; 404 case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE: 405 if (kvm) 406 r = kvm->arch.mmu.split_page_chunk_size; 407 else 408 r = KVM_ARM_EAGER_SPLIT_CHUNK_SIZE_DEFAULT; 409 break; 410 case KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES: 411 r = kvm_supported_block_sizes(); 412 break; 413 case KVM_CAP_ARM_SUPPORTED_REG_MASK_RANGES: 414 r = BIT(0); 415 break; 416 case KVM_CAP_ARM_CACHEABLE_PFNMAP_SUPPORTED: 417 if (!kvm) 418 r = -EINVAL; 419 else 420 r = kvm_supports_cacheable_pfnmap(); 421 break; 422 423 default: 424 r = 0; 425 } 426 427 return r; 428 } 429 430 long kvm_arch_dev_ioctl(struct file *filp, 431 unsigned int ioctl, unsigned long arg) 432 { 433 return -EINVAL; 434 } 435 436 struct kvm *kvm_arch_alloc_vm(void) 437 { 438 size_t sz = sizeof(struct kvm); 439 440 if (!has_vhe()) 441 return kzalloc(sz, GFP_KERNEL_ACCOUNT); 442 443 return __vmalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_HIGHMEM | __GFP_ZERO); 444 } 445 446 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) 447 { 448 if (irqchip_in_kernel(kvm) && vgic_initialized(kvm)) 449 return -EBUSY; 450 451 if (id >= kvm->max_vcpus) 452 return -EINVAL; 453 454 return 0; 455 } 456 457 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) 458 { 459 int err; 460 461 spin_lock_init(&vcpu->arch.mp_state_lock); 462 463 #ifdef CONFIG_LOCKDEP 464 /* Inform lockdep that the config_lock is acquired after vcpu->mutex */ 465 mutex_lock(&vcpu->mutex); 466 mutex_lock(&vcpu->kvm->arch.config_lock); 467 mutex_unlock(&vcpu->kvm->arch.config_lock); 468 mutex_unlock(&vcpu->mutex); 469 #endif 470 471 /* Force users to call KVM_ARM_VCPU_INIT */ 472 vcpu_clear_flag(vcpu, VCPU_INITIALIZED); 473 474 vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO; 475 476 /* Set up the timer */ 477 kvm_timer_vcpu_init(vcpu); 478 479 kvm_pmu_vcpu_init(vcpu); 480 481 kvm_arm_pvtime_vcpu_init(&vcpu->arch); 482 483 vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu; 484 485 /* 486 * This vCPU may have been created after mpidr_data was initialized. 487 * Throw out the pre-computed mappings if that is the case which forces 488 * KVM to fall back to iteratively searching the vCPUs. 489 */ 490 kvm_destroy_mpidr_data(vcpu->kvm); 491 492 err = kvm_vgic_vcpu_init(vcpu); 493 if (err) { 494 kvm_vgic_vcpu_destroy(vcpu); 495 return err; 496 } 497 498 err = kvm_share_hyp(vcpu, vcpu + 1); 499 if (err) 500 kvm_vgic_vcpu_destroy(vcpu); 501 502 return err; 503 } 504 505 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) 506 { 507 } 508 509 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) 510 { 511 if (!is_protected_kvm_enabled()) 512 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); 513 else 514 free_hyp_memcache(&vcpu->arch.pkvm_memcache); 515 kvm_timer_vcpu_terminate(vcpu); 516 kvm_pmu_vcpu_destroy(vcpu); 517 kvm_vgic_vcpu_destroy(vcpu); 518 kvm_arm_vcpu_destroy(vcpu); 519 } 520 521 void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) 522 { 523 524 } 525 526 void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) 527 { 528 529 } 530 531 static void vcpu_set_pauth_traps(struct kvm_vcpu *vcpu) 532 { 533 if (vcpu_has_ptrauth(vcpu) && !is_protected_kvm_enabled()) { 534 /* 535 * Either we're running an L2 guest, and the API/APK bits come 536 * from L1's HCR_EL2, or API/APK are both set. 537 */ 538 if (unlikely(is_nested_ctxt(vcpu))) { 539 u64 val; 540 541 val = __vcpu_sys_reg(vcpu, HCR_EL2); 542 val &= (HCR_API | HCR_APK); 543 vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK); 544 vcpu->arch.hcr_el2 |= val; 545 } else { 546 vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK); 547 } 548 549 /* 550 * Save the host keys if there is any chance for the guest 551 * to use pauth, as the entry code will reload the guest 552 * keys in that case. 553 */ 554 if (vcpu->arch.hcr_el2 & (HCR_API | HCR_APK)) { 555 struct kvm_cpu_context *ctxt; 556 557 ctxt = this_cpu_ptr_hyp_sym(kvm_hyp_ctxt); 558 ptrauth_save_keys(ctxt); 559 } 560 } 561 } 562 563 static bool kvm_vcpu_should_clear_twi(struct kvm_vcpu *vcpu) 564 { 565 if (unlikely(kvm_wfi_trap_policy != KVM_WFX_NOTRAP_SINGLE_TASK)) 566 return kvm_wfi_trap_policy == KVM_WFX_NOTRAP; 567 568 return single_task_running() && 569 (atomic_read(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe.vlpi_count) || 570 vcpu->kvm->arch.vgic.nassgireq); 571 } 572 573 static bool kvm_vcpu_should_clear_twe(struct kvm_vcpu *vcpu) 574 { 575 if (unlikely(kvm_wfe_trap_policy != KVM_WFX_NOTRAP_SINGLE_TASK)) 576 return kvm_wfe_trap_policy == KVM_WFX_NOTRAP; 577 578 return single_task_running(); 579 } 580 581 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 582 { 583 struct kvm_s2_mmu *mmu; 584 int *last_ran; 585 586 if (is_protected_kvm_enabled()) 587 goto nommu; 588 589 if (vcpu_has_nv(vcpu)) 590 kvm_vcpu_load_hw_mmu(vcpu); 591 592 mmu = vcpu->arch.hw_mmu; 593 last_ran = this_cpu_ptr(mmu->last_vcpu_ran); 594 595 /* 596 * Ensure a VMID is allocated for the MMU before programming VTTBR_EL2, 597 * which happens eagerly in VHE. 598 * 599 * Also, the VMID allocator only preserves VMIDs that are active at the 600 * time of rollover, so KVM might need to grab a new VMID for the MMU if 601 * this is called from kvm_sched_in(). 602 */ 603 kvm_arm_vmid_update(&mmu->vmid); 604 605 /* 606 * We guarantee that both TLBs and I-cache are private to each 607 * vcpu. If detecting that a vcpu from the same VM has 608 * previously run on the same physical CPU, call into the 609 * hypervisor code to nuke the relevant contexts. 610 * 611 * We might get preempted before the vCPU actually runs, but 612 * over-invalidation doesn't affect correctness. 613 */ 614 if (*last_ran != vcpu->vcpu_idx) { 615 kvm_call_hyp(__kvm_flush_cpu_context, mmu); 616 *last_ran = vcpu->vcpu_idx; 617 } 618 619 nommu: 620 vcpu->cpu = cpu; 621 622 /* 623 * The timer must be loaded before the vgic to correctly set up physical 624 * interrupt deactivation in nested state (e.g. timer interrupt). 625 */ 626 kvm_timer_vcpu_load(vcpu); 627 kvm_vgic_load(vcpu); 628 kvm_vcpu_load_debug(vcpu); 629 kvm_vcpu_load_fgt(vcpu); 630 if (has_vhe()) 631 kvm_vcpu_load_vhe(vcpu); 632 kvm_arch_vcpu_load_fp(vcpu); 633 kvm_vcpu_pmu_restore_guest(vcpu); 634 if (kvm_arm_is_pvtime_enabled(&vcpu->arch)) 635 kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu); 636 637 if (kvm_vcpu_should_clear_twe(vcpu)) 638 vcpu->arch.hcr_el2 &= ~HCR_TWE; 639 else 640 vcpu->arch.hcr_el2 |= HCR_TWE; 641 642 if (kvm_vcpu_should_clear_twi(vcpu)) 643 vcpu->arch.hcr_el2 &= ~HCR_TWI; 644 else 645 vcpu->arch.hcr_el2 |= HCR_TWI; 646 647 vcpu_set_pauth_traps(vcpu); 648 649 if (is_protected_kvm_enabled()) { 650 kvm_call_hyp_nvhe(__pkvm_vcpu_load, 651 vcpu->kvm->arch.pkvm.handle, 652 vcpu->vcpu_idx, vcpu->arch.hcr_el2); 653 kvm_call_hyp(__vgic_v3_restore_vmcr_aprs, 654 &vcpu->arch.vgic_cpu.vgic_v3); 655 } 656 657 if (!cpumask_test_cpu(cpu, vcpu->kvm->arch.supported_cpus)) 658 vcpu_set_on_unsupported_cpu(vcpu); 659 } 660 661 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 662 { 663 if (is_protected_kvm_enabled()) { 664 kvm_call_hyp(__vgic_v3_save_vmcr_aprs, 665 &vcpu->arch.vgic_cpu.vgic_v3); 666 kvm_call_hyp_nvhe(__pkvm_vcpu_put); 667 } 668 669 kvm_vcpu_put_debug(vcpu); 670 kvm_arch_vcpu_put_fp(vcpu); 671 if (has_vhe()) 672 kvm_vcpu_put_vhe(vcpu); 673 kvm_timer_vcpu_put(vcpu); 674 kvm_vgic_put(vcpu); 675 kvm_vcpu_pmu_restore_host(vcpu); 676 if (vcpu_has_nv(vcpu)) 677 kvm_vcpu_put_hw_mmu(vcpu); 678 kvm_arm_vmid_clear_active(); 679 680 vcpu_clear_on_unsupported_cpu(vcpu); 681 vcpu->cpu = -1; 682 } 683 684 static void __kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu) 685 { 686 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED); 687 kvm_make_request(KVM_REQ_SLEEP, vcpu); 688 kvm_vcpu_kick(vcpu); 689 } 690 691 void kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu) 692 { 693 spin_lock(&vcpu->arch.mp_state_lock); 694 __kvm_arm_vcpu_power_off(vcpu); 695 spin_unlock(&vcpu->arch.mp_state_lock); 696 } 697 698 bool kvm_arm_vcpu_stopped(struct kvm_vcpu *vcpu) 699 { 700 return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_STOPPED; 701 } 702 703 static void kvm_arm_vcpu_suspend(struct kvm_vcpu *vcpu) 704 { 705 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_SUSPENDED); 706 kvm_make_request(KVM_REQ_SUSPEND, vcpu); 707 kvm_vcpu_kick(vcpu); 708 } 709 710 static bool kvm_arm_vcpu_suspended(struct kvm_vcpu *vcpu) 711 { 712 return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_SUSPENDED; 713 } 714 715 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 716 struct kvm_mp_state *mp_state) 717 { 718 *mp_state = READ_ONCE(vcpu->arch.mp_state); 719 720 return 0; 721 } 722 723 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 724 struct kvm_mp_state *mp_state) 725 { 726 int ret = 0; 727 728 spin_lock(&vcpu->arch.mp_state_lock); 729 730 switch (mp_state->mp_state) { 731 case KVM_MP_STATE_RUNNABLE: 732 WRITE_ONCE(vcpu->arch.mp_state, *mp_state); 733 break; 734 case KVM_MP_STATE_STOPPED: 735 __kvm_arm_vcpu_power_off(vcpu); 736 break; 737 case KVM_MP_STATE_SUSPENDED: 738 kvm_arm_vcpu_suspend(vcpu); 739 break; 740 default: 741 ret = -EINVAL; 742 } 743 744 spin_unlock(&vcpu->arch.mp_state_lock); 745 746 return ret; 747 } 748 749 /** 750 * kvm_arch_vcpu_runnable - determine if the vcpu can be scheduled 751 * @v: The VCPU pointer 752 * 753 * If the guest CPU is not waiting for interrupts or an interrupt line is 754 * asserted, the CPU is by definition runnable. 755 */ 756 int kvm_arch_vcpu_runnable(struct kvm_vcpu *v) 757 { 758 bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF | HCR_VSE); 759 760 irq_lines |= (!irqchip_in_kernel(v->kvm) && 761 (kvm_timer_should_notify_user(v) || 762 kvm_pmu_should_notify_user(v))); 763 764 return ((irq_lines || kvm_vgic_vcpu_pending_irq(v)) 765 && !kvm_arm_vcpu_stopped(v) && !v->arch.pause); 766 } 767 768 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) 769 { 770 return vcpu_mode_priv(vcpu); 771 } 772 773 #ifdef CONFIG_GUEST_PERF_EVENTS 774 unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu) 775 { 776 return *vcpu_pc(vcpu); 777 } 778 #endif 779 780 static void kvm_init_mpidr_data(struct kvm *kvm) 781 { 782 struct kvm_mpidr_data *data = NULL; 783 unsigned long c, mask, nr_entries; 784 u64 aff_set = 0, aff_clr = ~0UL; 785 struct kvm_vcpu *vcpu; 786 787 mutex_lock(&kvm->arch.config_lock); 788 789 if (rcu_access_pointer(kvm->arch.mpidr_data) || 790 atomic_read(&kvm->online_vcpus) == 1) 791 goto out; 792 793 kvm_for_each_vcpu(c, vcpu, kvm) { 794 u64 aff = kvm_vcpu_get_mpidr_aff(vcpu); 795 aff_set |= aff; 796 aff_clr &= aff; 797 } 798 799 /* 800 * A significant bit can be either 0 or 1, and will only appear in 801 * aff_set. Use aff_clr to weed out the useless stuff. 802 */ 803 mask = aff_set ^ aff_clr; 804 nr_entries = BIT_ULL(hweight_long(mask)); 805 806 /* 807 * Don't let userspace fool us. If we need more than a single page 808 * to describe the compressed MPIDR array, just fall back to the 809 * iterative method. Single vcpu VMs do not need this either. 810 */ 811 if (struct_size(data, cmpidr_to_idx, nr_entries) <= PAGE_SIZE) 812 data = kzalloc(struct_size(data, cmpidr_to_idx, nr_entries), 813 GFP_KERNEL_ACCOUNT); 814 815 if (!data) 816 goto out; 817 818 data->mpidr_mask = mask; 819 820 kvm_for_each_vcpu(c, vcpu, kvm) { 821 u64 aff = kvm_vcpu_get_mpidr_aff(vcpu); 822 u16 index = kvm_mpidr_index(data, aff); 823 824 data->cmpidr_to_idx[index] = c; 825 } 826 827 rcu_assign_pointer(kvm->arch.mpidr_data, data); 828 out: 829 mutex_unlock(&kvm->arch.config_lock); 830 } 831 832 /* 833 * Handle both the initialisation that is being done when the vcpu is 834 * run for the first time, as well as the updates that must be 835 * performed each time we get a new thread dealing with this vcpu. 836 */ 837 int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) 838 { 839 struct kvm *kvm = vcpu->kvm; 840 int ret; 841 842 if (!kvm_vcpu_initialized(vcpu)) 843 return -ENOEXEC; 844 845 if (!kvm_arm_vcpu_is_finalized(vcpu)) 846 return -EPERM; 847 848 if (likely(vcpu_has_run_once(vcpu))) 849 return 0; 850 851 kvm_init_mpidr_data(kvm); 852 853 if (likely(irqchip_in_kernel(kvm))) { 854 /* 855 * Map the VGIC hardware resources before running a vcpu the 856 * first time on this VM. 857 */ 858 ret = kvm_vgic_map_resources(kvm); 859 if (ret) 860 return ret; 861 } 862 863 ret = kvm_finalize_sys_regs(vcpu); 864 if (ret) 865 return ret; 866 867 if (vcpu_has_nv(vcpu)) { 868 ret = kvm_vcpu_allocate_vncr_tlb(vcpu); 869 if (ret) 870 return ret; 871 872 ret = kvm_vgic_vcpu_nv_init(vcpu); 873 if (ret) 874 return ret; 875 } 876 877 /* 878 * This needs to happen after any restriction has been applied 879 * to the feature set. 880 */ 881 kvm_calculate_traps(vcpu); 882 883 ret = kvm_timer_enable(vcpu); 884 if (ret) 885 return ret; 886 887 if (kvm_vcpu_has_pmu(vcpu)) { 888 ret = kvm_arm_pmu_v3_enable(vcpu); 889 if (ret) 890 return ret; 891 } 892 893 if (is_protected_kvm_enabled()) { 894 ret = pkvm_create_hyp_vm(kvm); 895 if (ret) 896 return ret; 897 898 ret = pkvm_create_hyp_vcpu(vcpu); 899 if (ret) 900 return ret; 901 } 902 903 mutex_lock(&kvm->arch.config_lock); 904 set_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &kvm->arch.flags); 905 mutex_unlock(&kvm->arch.config_lock); 906 907 return ret; 908 } 909 910 bool kvm_arch_intc_initialized(struct kvm *kvm) 911 { 912 return vgic_initialized(kvm); 913 } 914 915 void kvm_arm_halt_guest(struct kvm *kvm) 916 { 917 unsigned long i; 918 struct kvm_vcpu *vcpu; 919 920 kvm_for_each_vcpu(i, vcpu, kvm) 921 vcpu->arch.pause = true; 922 kvm_make_all_cpus_request(kvm, KVM_REQ_SLEEP); 923 } 924 925 void kvm_arm_resume_guest(struct kvm *kvm) 926 { 927 unsigned long i; 928 struct kvm_vcpu *vcpu; 929 930 kvm_for_each_vcpu(i, vcpu, kvm) { 931 vcpu->arch.pause = false; 932 __kvm_vcpu_wake_up(vcpu); 933 } 934 } 935 936 static void kvm_vcpu_sleep(struct kvm_vcpu *vcpu) 937 { 938 struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu); 939 940 rcuwait_wait_event(wait, 941 (!kvm_arm_vcpu_stopped(vcpu)) && (!vcpu->arch.pause), 942 TASK_INTERRUPTIBLE); 943 944 if (kvm_arm_vcpu_stopped(vcpu) || vcpu->arch.pause) { 945 /* Awaken to handle a signal, request we sleep again later. */ 946 kvm_make_request(KVM_REQ_SLEEP, vcpu); 947 } 948 949 /* 950 * Make sure we will observe a potential reset request if we've 951 * observed a change to the power state. Pairs with the smp_wmb() in 952 * kvm_psci_vcpu_on(). 953 */ 954 smp_rmb(); 955 } 956 957 /** 958 * kvm_vcpu_wfi - emulate Wait-For-Interrupt behavior 959 * @vcpu: The VCPU pointer 960 * 961 * Suspend execution of a vCPU until a valid wake event is detected, i.e. until 962 * the vCPU is runnable. The vCPU may or may not be scheduled out, depending 963 * on when a wake event arrives, e.g. there may already be a pending wake event. 964 */ 965 void kvm_vcpu_wfi(struct kvm_vcpu *vcpu) 966 { 967 /* 968 * Sync back the state of the GIC CPU interface so that we have 969 * the latest PMR and group enables. This ensures that 970 * kvm_arch_vcpu_runnable has up-to-date data to decide whether 971 * we have pending interrupts, e.g. when determining if the 972 * vCPU should block. 973 * 974 * For the same reason, we want to tell GICv4 that we need 975 * doorbells to be signalled, should an interrupt become pending. 976 */ 977 preempt_disable(); 978 vcpu_set_flag(vcpu, IN_WFI); 979 kvm_vgic_put(vcpu); 980 preempt_enable(); 981 982 kvm_vcpu_halt(vcpu); 983 vcpu_clear_flag(vcpu, IN_WFIT); 984 985 preempt_disable(); 986 vcpu_clear_flag(vcpu, IN_WFI); 987 kvm_vgic_load(vcpu); 988 preempt_enable(); 989 } 990 991 static int kvm_vcpu_suspend(struct kvm_vcpu *vcpu) 992 { 993 if (!kvm_arm_vcpu_suspended(vcpu)) 994 return 1; 995 996 kvm_vcpu_wfi(vcpu); 997 998 /* 999 * The suspend state is sticky; we do not leave it until userspace 1000 * explicitly marks the vCPU as runnable. Request that we suspend again 1001 * later. 1002 */ 1003 kvm_make_request(KVM_REQ_SUSPEND, vcpu); 1004 1005 /* 1006 * Check to make sure the vCPU is actually runnable. If so, exit to 1007 * userspace informing it of the wakeup condition. 1008 */ 1009 if (kvm_arch_vcpu_runnable(vcpu)) { 1010 memset(&vcpu->run->system_event, 0, sizeof(vcpu->run->system_event)); 1011 vcpu->run->system_event.type = KVM_SYSTEM_EVENT_WAKEUP; 1012 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; 1013 return 0; 1014 } 1015 1016 /* 1017 * Otherwise, we were unblocked to process a different event, such as a 1018 * pending signal. Return 1 and allow kvm_arch_vcpu_ioctl_run() to 1019 * process the event. 1020 */ 1021 return 1; 1022 } 1023 1024 /** 1025 * check_vcpu_requests - check and handle pending vCPU requests 1026 * @vcpu: the VCPU pointer 1027 * 1028 * Return: 1 if we should enter the guest 1029 * 0 if we should exit to userspace 1030 * < 0 if we should exit to userspace, where the return value indicates 1031 * an error 1032 */ 1033 static int check_vcpu_requests(struct kvm_vcpu *vcpu) 1034 { 1035 if (kvm_request_pending(vcpu)) { 1036 if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) 1037 return -EIO; 1038 1039 if (kvm_check_request(KVM_REQ_SLEEP, vcpu)) 1040 kvm_vcpu_sleep(vcpu); 1041 1042 if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu)) 1043 kvm_reset_vcpu(vcpu); 1044 1045 /* 1046 * Clear IRQ_PENDING requests that were made to guarantee 1047 * that a VCPU sees new virtual interrupts. 1048 */ 1049 kvm_check_request(KVM_REQ_IRQ_PENDING, vcpu); 1050 1051 if (kvm_check_request(KVM_REQ_RECORD_STEAL, vcpu)) 1052 kvm_update_stolen_time(vcpu); 1053 1054 if (kvm_check_request(KVM_REQ_RELOAD_GICv4, vcpu)) { 1055 /* The distributor enable bits were changed */ 1056 preempt_disable(); 1057 vgic_v4_put(vcpu); 1058 vgic_v4_load(vcpu); 1059 preempt_enable(); 1060 } 1061 1062 if (kvm_check_request(KVM_REQ_RELOAD_PMU, vcpu)) 1063 kvm_vcpu_reload_pmu(vcpu); 1064 1065 if (kvm_check_request(KVM_REQ_RESYNC_PMU_EL0, vcpu)) 1066 kvm_vcpu_pmu_restore_guest(vcpu); 1067 1068 if (kvm_check_request(KVM_REQ_SUSPEND, vcpu)) 1069 return kvm_vcpu_suspend(vcpu); 1070 1071 if (kvm_dirty_ring_check_request(vcpu)) 1072 return 0; 1073 1074 check_nested_vcpu_requests(vcpu); 1075 } 1076 1077 return 1; 1078 } 1079 1080 static bool vcpu_mode_is_bad_32bit(struct kvm_vcpu *vcpu) 1081 { 1082 if (likely(!vcpu_mode_is_32bit(vcpu))) 1083 return false; 1084 1085 if (vcpu_has_nv(vcpu)) 1086 return true; 1087 1088 return !kvm_supports_32bit_el0(); 1089 } 1090 1091 /** 1092 * kvm_vcpu_exit_request - returns true if the VCPU should *not* enter the guest 1093 * @vcpu: The VCPU pointer 1094 * @ret: Pointer to write optional return code 1095 * 1096 * Returns: true if the VCPU needs to return to a preemptible + interruptible 1097 * and skip guest entry. 1098 * 1099 * This function disambiguates between two different types of exits: exits to a 1100 * preemptible + interruptible kernel context and exits to userspace. For an 1101 * exit to userspace, this function will write the return code to ret and return 1102 * true. For an exit to preemptible + interruptible kernel context (i.e. check 1103 * for pending work and re-enter), return true without writing to ret. 1104 */ 1105 static bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu, int *ret) 1106 { 1107 struct kvm_run *run = vcpu->run; 1108 1109 /* 1110 * If we're using a userspace irqchip, then check if we need 1111 * to tell a userspace irqchip about timer or PMU level 1112 * changes and if so, exit to userspace (the actual level 1113 * state gets updated in kvm_timer_update_run and 1114 * kvm_pmu_update_run below). 1115 */ 1116 if (unlikely(!irqchip_in_kernel(vcpu->kvm))) { 1117 if (kvm_timer_should_notify_user(vcpu) || 1118 kvm_pmu_should_notify_user(vcpu)) { 1119 *ret = -EINTR; 1120 run->exit_reason = KVM_EXIT_INTR; 1121 return true; 1122 } 1123 } 1124 1125 if (unlikely(vcpu_on_unsupported_cpu(vcpu))) { 1126 run->exit_reason = KVM_EXIT_FAIL_ENTRY; 1127 run->fail_entry.hardware_entry_failure_reason = KVM_EXIT_FAIL_ENTRY_CPU_UNSUPPORTED; 1128 run->fail_entry.cpu = smp_processor_id(); 1129 *ret = 0; 1130 return true; 1131 } 1132 1133 return kvm_request_pending(vcpu) || 1134 xfer_to_guest_mode_work_pending(); 1135 } 1136 1137 /* 1138 * Actually run the vCPU, entering an RCU extended quiescent state (EQS) while 1139 * the vCPU is running. 1140 * 1141 * This must be noinstr as instrumentation may make use of RCU, and this is not 1142 * safe during the EQS. 1143 */ 1144 static int noinstr kvm_arm_vcpu_enter_exit(struct kvm_vcpu *vcpu) 1145 { 1146 int ret; 1147 1148 guest_state_enter_irqoff(); 1149 ret = kvm_call_hyp_ret(__kvm_vcpu_run, vcpu); 1150 guest_state_exit_irqoff(); 1151 1152 return ret; 1153 } 1154 1155 /** 1156 * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code 1157 * @vcpu: The VCPU pointer 1158 * 1159 * This function is called through the VCPU_RUN ioctl called from user space. It 1160 * will execute VM code in a loop until the time slice for the process is used 1161 * or some emulation is needed from user space in which case the function will 1162 * return with return value 0 and with the kvm_run structure filled in with the 1163 * required data for the requested emulation. 1164 */ 1165 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) 1166 { 1167 struct kvm_run *run = vcpu->run; 1168 int ret; 1169 1170 if (run->exit_reason == KVM_EXIT_MMIO) { 1171 ret = kvm_handle_mmio_return(vcpu); 1172 if (ret <= 0) 1173 return ret; 1174 } 1175 1176 vcpu_load(vcpu); 1177 1178 if (!vcpu->wants_to_run) { 1179 ret = -EINTR; 1180 goto out; 1181 } 1182 1183 kvm_sigset_activate(vcpu); 1184 1185 ret = 1; 1186 run->exit_reason = KVM_EXIT_UNKNOWN; 1187 run->flags = 0; 1188 while (ret > 0) { 1189 /* 1190 * Check conditions before entering the guest 1191 */ 1192 ret = kvm_xfer_to_guest_mode_handle_work(vcpu); 1193 if (!ret) 1194 ret = 1; 1195 1196 if (ret > 0) 1197 ret = check_vcpu_requests(vcpu); 1198 1199 /* 1200 * Preparing the interrupts to be injected also 1201 * involves poking the GIC, which must be done in a 1202 * non-preemptible context. 1203 */ 1204 preempt_disable(); 1205 1206 kvm_nested_flush_hwstate(vcpu); 1207 1208 if (kvm_vcpu_has_pmu(vcpu)) 1209 kvm_pmu_flush_hwstate(vcpu); 1210 1211 local_irq_disable(); 1212 1213 kvm_vgic_flush_hwstate(vcpu); 1214 1215 kvm_pmu_update_vcpu_events(vcpu); 1216 1217 /* 1218 * Ensure we set mode to IN_GUEST_MODE after we disable 1219 * interrupts and before the final VCPU requests check. 1220 * See the comment in kvm_vcpu_exiting_guest_mode() and 1221 * Documentation/virt/kvm/vcpu-requests.rst 1222 */ 1223 smp_store_mb(vcpu->mode, IN_GUEST_MODE); 1224 1225 if (ret <= 0 || kvm_vcpu_exit_request(vcpu, &ret)) { 1226 vcpu->mode = OUTSIDE_GUEST_MODE; 1227 isb(); /* Ensure work in x_flush_hwstate is committed */ 1228 if (kvm_vcpu_has_pmu(vcpu)) 1229 kvm_pmu_sync_hwstate(vcpu); 1230 if (unlikely(!irqchip_in_kernel(vcpu->kvm))) 1231 kvm_timer_sync_user(vcpu); 1232 kvm_vgic_sync_hwstate(vcpu); 1233 local_irq_enable(); 1234 preempt_enable(); 1235 continue; 1236 } 1237 1238 kvm_arch_vcpu_ctxflush_fp(vcpu); 1239 1240 /************************************************************** 1241 * Enter the guest 1242 */ 1243 trace_kvm_entry(*vcpu_pc(vcpu)); 1244 guest_timing_enter_irqoff(); 1245 1246 ret = kvm_arm_vcpu_enter_exit(vcpu); 1247 1248 vcpu->mode = OUTSIDE_GUEST_MODE; 1249 vcpu->stat.exits++; 1250 /* 1251 * Back from guest 1252 *************************************************************/ 1253 1254 /* 1255 * We must sync the PMU state before the vgic state so 1256 * that the vgic can properly sample the updated state of the 1257 * interrupt line. 1258 */ 1259 if (kvm_vcpu_has_pmu(vcpu)) 1260 kvm_pmu_sync_hwstate(vcpu); 1261 1262 /* 1263 * Sync the vgic state before syncing the timer state because 1264 * the timer code needs to know if the virtual timer 1265 * interrupts are active. 1266 */ 1267 kvm_vgic_sync_hwstate(vcpu); 1268 1269 /* 1270 * Sync the timer hardware state before enabling interrupts as 1271 * we don't want vtimer interrupts to race with syncing the 1272 * timer virtual interrupt state. 1273 */ 1274 if (unlikely(!irqchip_in_kernel(vcpu->kvm))) 1275 kvm_timer_sync_user(vcpu); 1276 1277 if (is_hyp_ctxt(vcpu)) 1278 kvm_timer_sync_nested(vcpu); 1279 1280 kvm_arch_vcpu_ctxsync_fp(vcpu); 1281 1282 /* 1283 * We must ensure that any pending interrupts are taken before 1284 * we exit guest timing so that timer ticks are accounted as 1285 * guest time. Transiently unmask interrupts so that any 1286 * pending interrupts are taken. 1287 * 1288 * Per ARM DDI 0487G.b section D1.13.4, an ISB (or other 1289 * context synchronization event) is necessary to ensure that 1290 * pending interrupts are taken. 1291 */ 1292 if (ARM_EXCEPTION_CODE(ret) == ARM_EXCEPTION_IRQ) { 1293 local_irq_enable(); 1294 isb(); 1295 local_irq_disable(); 1296 } 1297 1298 guest_timing_exit_irqoff(); 1299 1300 local_irq_enable(); 1301 1302 trace_kvm_exit(ret, kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu)); 1303 1304 /* Exit types that need handling before we can be preempted */ 1305 handle_exit_early(vcpu, ret); 1306 1307 kvm_nested_sync_hwstate(vcpu); 1308 1309 preempt_enable(); 1310 1311 /* 1312 * The ARMv8 architecture doesn't give the hypervisor 1313 * a mechanism to prevent a guest from dropping to AArch32 EL0 1314 * if implemented by the CPU. If we spot the guest in such 1315 * state and that we decided it wasn't supposed to do so (like 1316 * with the asymmetric AArch32 case), return to userspace with 1317 * a fatal error. 1318 */ 1319 if (vcpu_mode_is_bad_32bit(vcpu)) { 1320 /* 1321 * As we have caught the guest red-handed, decide that 1322 * it isn't fit for purpose anymore by making the vcpu 1323 * invalid. The VMM can try and fix it by issuing a 1324 * KVM_ARM_VCPU_INIT if it really wants to. 1325 */ 1326 vcpu_clear_flag(vcpu, VCPU_INITIALIZED); 1327 ret = ARM_EXCEPTION_IL; 1328 } 1329 1330 ret = handle_exit(vcpu, ret); 1331 } 1332 1333 /* Tell userspace about in-kernel device output levels */ 1334 if (unlikely(!irqchip_in_kernel(vcpu->kvm))) { 1335 kvm_timer_update_run(vcpu); 1336 kvm_pmu_update_run(vcpu); 1337 } 1338 1339 kvm_sigset_deactivate(vcpu); 1340 1341 out: 1342 /* 1343 * In the unlikely event that we are returning to userspace 1344 * with pending exceptions or PC adjustment, commit these 1345 * adjustments in order to give userspace a consistent view of 1346 * the vcpu state. Note that this relies on __kvm_adjust_pc() 1347 * being preempt-safe on VHE. 1348 */ 1349 if (unlikely(vcpu_get_flag(vcpu, PENDING_EXCEPTION) || 1350 vcpu_get_flag(vcpu, INCREMENT_PC))) 1351 kvm_call_hyp(__kvm_adjust_pc, vcpu); 1352 1353 vcpu_put(vcpu); 1354 return ret; 1355 } 1356 1357 static int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level) 1358 { 1359 int bit_index; 1360 bool set; 1361 unsigned long *hcr; 1362 1363 if (number == KVM_ARM_IRQ_CPU_IRQ) 1364 bit_index = __ffs(HCR_VI); 1365 else /* KVM_ARM_IRQ_CPU_FIQ */ 1366 bit_index = __ffs(HCR_VF); 1367 1368 hcr = vcpu_hcr(vcpu); 1369 if (level) 1370 set = test_and_set_bit(bit_index, hcr); 1371 else 1372 set = test_and_clear_bit(bit_index, hcr); 1373 1374 /* 1375 * If we didn't change anything, no need to wake up or kick other CPUs 1376 */ 1377 if (set == level) 1378 return 0; 1379 1380 /* 1381 * The vcpu irq_lines field was updated, wake up sleeping VCPUs and 1382 * trigger a world-switch round on the running physical CPU to set the 1383 * virtual IRQ/FIQ fields in the HCR appropriately. 1384 */ 1385 kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu); 1386 kvm_vcpu_kick(vcpu); 1387 1388 return 0; 1389 } 1390 1391 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, 1392 bool line_status) 1393 { 1394 u32 irq = irq_level->irq; 1395 unsigned int irq_type, vcpu_id, irq_num; 1396 struct kvm_vcpu *vcpu = NULL; 1397 bool level = irq_level->level; 1398 1399 irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK; 1400 vcpu_id = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK; 1401 vcpu_id += ((irq >> KVM_ARM_IRQ_VCPU2_SHIFT) & KVM_ARM_IRQ_VCPU2_MASK) * (KVM_ARM_IRQ_VCPU_MASK + 1); 1402 irq_num = (irq >> KVM_ARM_IRQ_NUM_SHIFT) & KVM_ARM_IRQ_NUM_MASK; 1403 1404 trace_kvm_irq_line(irq_type, vcpu_id, irq_num, irq_level->level); 1405 1406 switch (irq_type) { 1407 case KVM_ARM_IRQ_TYPE_CPU: 1408 if (irqchip_in_kernel(kvm)) 1409 return -ENXIO; 1410 1411 vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id); 1412 if (!vcpu) 1413 return -EINVAL; 1414 1415 if (irq_num > KVM_ARM_IRQ_CPU_FIQ) 1416 return -EINVAL; 1417 1418 return vcpu_interrupt_line(vcpu, irq_num, level); 1419 case KVM_ARM_IRQ_TYPE_PPI: 1420 if (!irqchip_in_kernel(kvm)) 1421 return -ENXIO; 1422 1423 vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id); 1424 if (!vcpu) 1425 return -EINVAL; 1426 1427 if (irq_num < VGIC_NR_SGIS || irq_num >= VGIC_NR_PRIVATE_IRQS) 1428 return -EINVAL; 1429 1430 return kvm_vgic_inject_irq(kvm, vcpu, irq_num, level, NULL); 1431 case KVM_ARM_IRQ_TYPE_SPI: 1432 if (!irqchip_in_kernel(kvm)) 1433 return -ENXIO; 1434 1435 if (irq_num < VGIC_NR_PRIVATE_IRQS) 1436 return -EINVAL; 1437 1438 return kvm_vgic_inject_irq(kvm, NULL, irq_num, level, NULL); 1439 } 1440 1441 return -EINVAL; 1442 } 1443 1444 static unsigned long system_supported_vcpu_features(void) 1445 { 1446 unsigned long features = KVM_VCPU_VALID_FEATURES; 1447 1448 if (!cpus_have_final_cap(ARM64_HAS_32BIT_EL1)) 1449 clear_bit(KVM_ARM_VCPU_EL1_32BIT, &features); 1450 1451 if (!kvm_supports_guest_pmuv3()) 1452 clear_bit(KVM_ARM_VCPU_PMU_V3, &features); 1453 1454 if (!system_supports_sve()) 1455 clear_bit(KVM_ARM_VCPU_SVE, &features); 1456 1457 if (!kvm_has_full_ptr_auth()) { 1458 clear_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, &features); 1459 clear_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, &features); 1460 } 1461 1462 if (!cpus_have_final_cap(ARM64_HAS_NESTED_VIRT)) 1463 clear_bit(KVM_ARM_VCPU_HAS_EL2, &features); 1464 1465 return features; 1466 } 1467 1468 static int kvm_vcpu_init_check_features(struct kvm_vcpu *vcpu, 1469 const struct kvm_vcpu_init *init) 1470 { 1471 unsigned long features = init->features[0]; 1472 int i; 1473 1474 if (features & ~KVM_VCPU_VALID_FEATURES) 1475 return -ENOENT; 1476 1477 for (i = 1; i < ARRAY_SIZE(init->features); i++) { 1478 if (init->features[i]) 1479 return -ENOENT; 1480 } 1481 1482 if (features & ~system_supported_vcpu_features()) 1483 return -EINVAL; 1484 1485 /* 1486 * For now make sure that both address/generic pointer authentication 1487 * features are requested by the userspace together. 1488 */ 1489 if (test_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, &features) != 1490 test_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, &features)) 1491 return -EINVAL; 1492 1493 if (!test_bit(KVM_ARM_VCPU_EL1_32BIT, &features)) 1494 return 0; 1495 1496 /* MTE is incompatible with AArch32 */ 1497 if (kvm_has_mte(vcpu->kvm)) 1498 return -EINVAL; 1499 1500 /* NV is incompatible with AArch32 */ 1501 if (test_bit(KVM_ARM_VCPU_HAS_EL2, &features)) 1502 return -EINVAL; 1503 1504 return 0; 1505 } 1506 1507 static bool kvm_vcpu_init_changed(struct kvm_vcpu *vcpu, 1508 const struct kvm_vcpu_init *init) 1509 { 1510 unsigned long features = init->features[0]; 1511 1512 return !bitmap_equal(vcpu->kvm->arch.vcpu_features, &features, 1513 KVM_VCPU_MAX_FEATURES); 1514 } 1515 1516 static int kvm_setup_vcpu(struct kvm_vcpu *vcpu) 1517 { 1518 struct kvm *kvm = vcpu->kvm; 1519 int ret = 0; 1520 1521 /* 1522 * When the vCPU has a PMU, but no PMU is set for the guest 1523 * yet, set the default one. 1524 */ 1525 if (kvm_vcpu_has_pmu(vcpu) && !kvm->arch.arm_pmu) 1526 ret = kvm_arm_set_default_pmu(kvm); 1527 1528 /* Prepare for nested if required */ 1529 if (!ret && vcpu_has_nv(vcpu)) 1530 ret = kvm_vcpu_init_nested(vcpu); 1531 1532 return ret; 1533 } 1534 1535 static int __kvm_vcpu_set_target(struct kvm_vcpu *vcpu, 1536 const struct kvm_vcpu_init *init) 1537 { 1538 unsigned long features = init->features[0]; 1539 struct kvm *kvm = vcpu->kvm; 1540 int ret = -EINVAL; 1541 1542 mutex_lock(&kvm->arch.config_lock); 1543 1544 if (test_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags) && 1545 kvm_vcpu_init_changed(vcpu, init)) 1546 goto out_unlock; 1547 1548 bitmap_copy(kvm->arch.vcpu_features, &features, KVM_VCPU_MAX_FEATURES); 1549 1550 ret = kvm_setup_vcpu(vcpu); 1551 if (ret) 1552 goto out_unlock; 1553 1554 /* Now we know what it is, we can reset it. */ 1555 kvm_reset_vcpu(vcpu); 1556 1557 set_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags); 1558 vcpu_set_flag(vcpu, VCPU_INITIALIZED); 1559 ret = 0; 1560 out_unlock: 1561 mutex_unlock(&kvm->arch.config_lock); 1562 return ret; 1563 } 1564 1565 static int kvm_vcpu_set_target(struct kvm_vcpu *vcpu, 1566 const struct kvm_vcpu_init *init) 1567 { 1568 int ret; 1569 1570 if (init->target != KVM_ARM_TARGET_GENERIC_V8 && 1571 init->target != kvm_target_cpu()) 1572 return -EINVAL; 1573 1574 ret = kvm_vcpu_init_check_features(vcpu, init); 1575 if (ret) 1576 return ret; 1577 1578 if (!kvm_vcpu_initialized(vcpu)) 1579 return __kvm_vcpu_set_target(vcpu, init); 1580 1581 if (kvm_vcpu_init_changed(vcpu, init)) 1582 return -EINVAL; 1583 1584 kvm_reset_vcpu(vcpu); 1585 return 0; 1586 } 1587 1588 static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu, 1589 struct kvm_vcpu_init *init) 1590 { 1591 bool power_off = false; 1592 int ret; 1593 1594 /* 1595 * Treat the power-off vCPU feature as ephemeral. Clear the bit to avoid 1596 * reflecting it in the finalized feature set, thus limiting its scope 1597 * to a single KVM_ARM_VCPU_INIT call. 1598 */ 1599 if (init->features[0] & BIT(KVM_ARM_VCPU_POWER_OFF)) { 1600 init->features[0] &= ~BIT(KVM_ARM_VCPU_POWER_OFF); 1601 power_off = true; 1602 } 1603 1604 ret = kvm_vcpu_set_target(vcpu, init); 1605 if (ret) 1606 return ret; 1607 1608 /* 1609 * Ensure a rebooted VM will fault in RAM pages and detect if the 1610 * guest MMU is turned off and flush the caches as needed. 1611 * 1612 * S2FWB enforces all memory accesses to RAM being cacheable, 1613 * ensuring that the data side is always coherent. We still 1614 * need to invalidate the I-cache though, as FWB does *not* 1615 * imply CTR_EL0.DIC. 1616 */ 1617 if (vcpu_has_run_once(vcpu)) { 1618 if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) 1619 stage2_unmap_vm(vcpu->kvm); 1620 else 1621 icache_inval_all_pou(); 1622 } 1623 1624 vcpu_reset_hcr(vcpu); 1625 1626 /* 1627 * Handle the "start in power-off" case. 1628 */ 1629 spin_lock(&vcpu->arch.mp_state_lock); 1630 1631 if (power_off) 1632 __kvm_arm_vcpu_power_off(vcpu); 1633 else 1634 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE); 1635 1636 spin_unlock(&vcpu->arch.mp_state_lock); 1637 1638 return 0; 1639 } 1640 1641 static int kvm_arm_vcpu_set_attr(struct kvm_vcpu *vcpu, 1642 struct kvm_device_attr *attr) 1643 { 1644 int ret = -ENXIO; 1645 1646 switch (attr->group) { 1647 default: 1648 ret = kvm_arm_vcpu_arch_set_attr(vcpu, attr); 1649 break; 1650 } 1651 1652 return ret; 1653 } 1654 1655 static int kvm_arm_vcpu_get_attr(struct kvm_vcpu *vcpu, 1656 struct kvm_device_attr *attr) 1657 { 1658 int ret = -ENXIO; 1659 1660 switch (attr->group) { 1661 default: 1662 ret = kvm_arm_vcpu_arch_get_attr(vcpu, attr); 1663 break; 1664 } 1665 1666 return ret; 1667 } 1668 1669 static int kvm_arm_vcpu_has_attr(struct kvm_vcpu *vcpu, 1670 struct kvm_device_attr *attr) 1671 { 1672 int ret = -ENXIO; 1673 1674 switch (attr->group) { 1675 default: 1676 ret = kvm_arm_vcpu_arch_has_attr(vcpu, attr); 1677 break; 1678 } 1679 1680 return ret; 1681 } 1682 1683 static int kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu, 1684 struct kvm_vcpu_events *events) 1685 { 1686 memset(events, 0, sizeof(*events)); 1687 1688 return __kvm_arm_vcpu_get_events(vcpu, events); 1689 } 1690 1691 static int kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu, 1692 struct kvm_vcpu_events *events) 1693 { 1694 int i; 1695 1696 /* check whether the reserved field is zero */ 1697 for (i = 0; i < ARRAY_SIZE(events->reserved); i++) 1698 if (events->reserved[i]) 1699 return -EINVAL; 1700 1701 /* check whether the pad field is zero */ 1702 for (i = 0; i < ARRAY_SIZE(events->exception.pad); i++) 1703 if (events->exception.pad[i]) 1704 return -EINVAL; 1705 1706 return __kvm_arm_vcpu_set_events(vcpu, events); 1707 } 1708 1709 long kvm_arch_vcpu_ioctl(struct file *filp, 1710 unsigned int ioctl, unsigned long arg) 1711 { 1712 struct kvm_vcpu *vcpu = filp->private_data; 1713 void __user *argp = (void __user *)arg; 1714 struct kvm_device_attr attr; 1715 long r; 1716 1717 switch (ioctl) { 1718 case KVM_ARM_VCPU_INIT: { 1719 struct kvm_vcpu_init init; 1720 1721 r = -EFAULT; 1722 if (copy_from_user(&init, argp, sizeof(init))) 1723 break; 1724 1725 r = kvm_arch_vcpu_ioctl_vcpu_init(vcpu, &init); 1726 break; 1727 } 1728 case KVM_SET_ONE_REG: 1729 case KVM_GET_ONE_REG: { 1730 struct kvm_one_reg reg; 1731 1732 r = -ENOEXEC; 1733 if (unlikely(!kvm_vcpu_initialized(vcpu))) 1734 break; 1735 1736 r = -EFAULT; 1737 if (copy_from_user(®, argp, sizeof(reg))) 1738 break; 1739 1740 /* 1741 * We could owe a reset due to PSCI. Handle the pending reset 1742 * here to ensure userspace register accesses are ordered after 1743 * the reset. 1744 */ 1745 if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu)) 1746 kvm_reset_vcpu(vcpu); 1747 1748 if (ioctl == KVM_SET_ONE_REG) 1749 r = kvm_arm_set_reg(vcpu, ®); 1750 else 1751 r = kvm_arm_get_reg(vcpu, ®); 1752 break; 1753 } 1754 case KVM_GET_REG_LIST: { 1755 struct kvm_reg_list __user *user_list = argp; 1756 struct kvm_reg_list reg_list; 1757 unsigned n; 1758 1759 r = -ENOEXEC; 1760 if (unlikely(!kvm_vcpu_initialized(vcpu))) 1761 break; 1762 1763 r = -EPERM; 1764 if (!kvm_arm_vcpu_is_finalized(vcpu)) 1765 break; 1766 1767 r = -EFAULT; 1768 if (copy_from_user(®_list, user_list, sizeof(reg_list))) 1769 break; 1770 n = reg_list.n; 1771 reg_list.n = kvm_arm_num_regs(vcpu); 1772 if (copy_to_user(user_list, ®_list, sizeof(reg_list))) 1773 break; 1774 r = -E2BIG; 1775 if (n < reg_list.n) 1776 break; 1777 r = kvm_arm_copy_reg_indices(vcpu, user_list->reg); 1778 break; 1779 } 1780 case KVM_SET_DEVICE_ATTR: { 1781 r = -EFAULT; 1782 if (copy_from_user(&attr, argp, sizeof(attr))) 1783 break; 1784 r = kvm_arm_vcpu_set_attr(vcpu, &attr); 1785 break; 1786 } 1787 case KVM_GET_DEVICE_ATTR: { 1788 r = -EFAULT; 1789 if (copy_from_user(&attr, argp, sizeof(attr))) 1790 break; 1791 r = kvm_arm_vcpu_get_attr(vcpu, &attr); 1792 break; 1793 } 1794 case KVM_HAS_DEVICE_ATTR: { 1795 r = -EFAULT; 1796 if (copy_from_user(&attr, argp, sizeof(attr))) 1797 break; 1798 r = kvm_arm_vcpu_has_attr(vcpu, &attr); 1799 break; 1800 } 1801 case KVM_GET_VCPU_EVENTS: { 1802 struct kvm_vcpu_events events; 1803 1804 if (!kvm_vcpu_initialized(vcpu)) 1805 return -ENOEXEC; 1806 1807 if (kvm_arm_vcpu_get_events(vcpu, &events)) 1808 return -EINVAL; 1809 1810 if (copy_to_user(argp, &events, sizeof(events))) 1811 return -EFAULT; 1812 1813 return 0; 1814 } 1815 case KVM_SET_VCPU_EVENTS: { 1816 struct kvm_vcpu_events events; 1817 1818 if (!kvm_vcpu_initialized(vcpu)) 1819 return -ENOEXEC; 1820 1821 if (copy_from_user(&events, argp, sizeof(events))) 1822 return -EFAULT; 1823 1824 return kvm_arm_vcpu_set_events(vcpu, &events); 1825 } 1826 case KVM_ARM_VCPU_FINALIZE: { 1827 int what; 1828 1829 if (!kvm_vcpu_initialized(vcpu)) 1830 return -ENOEXEC; 1831 1832 if (get_user(what, (const int __user *)argp)) 1833 return -EFAULT; 1834 1835 return kvm_arm_vcpu_finalize(vcpu, what); 1836 } 1837 default: 1838 r = -EINVAL; 1839 } 1840 1841 return r; 1842 } 1843 1844 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) 1845 { 1846 1847 } 1848 1849 static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm, 1850 struct kvm_arm_device_addr *dev_addr) 1851 { 1852 switch (FIELD_GET(KVM_ARM_DEVICE_ID_MASK, dev_addr->id)) { 1853 case KVM_ARM_DEVICE_VGIC_V2: 1854 if (!vgic_present) 1855 return -ENXIO; 1856 return kvm_set_legacy_vgic_v2_addr(kvm, dev_addr); 1857 default: 1858 return -ENODEV; 1859 } 1860 } 1861 1862 static int kvm_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr) 1863 { 1864 switch (attr->group) { 1865 case KVM_ARM_VM_SMCCC_CTRL: 1866 return kvm_vm_smccc_has_attr(kvm, attr); 1867 default: 1868 return -ENXIO; 1869 } 1870 } 1871 1872 static int kvm_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr) 1873 { 1874 switch (attr->group) { 1875 case KVM_ARM_VM_SMCCC_CTRL: 1876 return kvm_vm_smccc_set_attr(kvm, attr); 1877 default: 1878 return -ENXIO; 1879 } 1880 } 1881 1882 int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) 1883 { 1884 struct kvm *kvm = filp->private_data; 1885 void __user *argp = (void __user *)arg; 1886 struct kvm_device_attr attr; 1887 1888 switch (ioctl) { 1889 case KVM_CREATE_IRQCHIP: { 1890 int ret; 1891 if (!vgic_present) 1892 return -ENXIO; 1893 mutex_lock(&kvm->lock); 1894 ret = kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2); 1895 mutex_unlock(&kvm->lock); 1896 return ret; 1897 } 1898 case KVM_ARM_SET_DEVICE_ADDR: { 1899 struct kvm_arm_device_addr dev_addr; 1900 1901 if (copy_from_user(&dev_addr, argp, sizeof(dev_addr))) 1902 return -EFAULT; 1903 return kvm_vm_ioctl_set_device_addr(kvm, &dev_addr); 1904 } 1905 case KVM_ARM_PREFERRED_TARGET: { 1906 struct kvm_vcpu_init init = { 1907 .target = KVM_ARM_TARGET_GENERIC_V8, 1908 }; 1909 1910 if (copy_to_user(argp, &init, sizeof(init))) 1911 return -EFAULT; 1912 1913 return 0; 1914 } 1915 case KVM_ARM_MTE_COPY_TAGS: { 1916 struct kvm_arm_copy_mte_tags copy_tags; 1917 1918 if (copy_from_user(©_tags, argp, sizeof(copy_tags))) 1919 return -EFAULT; 1920 return kvm_vm_ioctl_mte_copy_tags(kvm, ©_tags); 1921 } 1922 case KVM_ARM_SET_COUNTER_OFFSET: { 1923 struct kvm_arm_counter_offset offset; 1924 1925 if (copy_from_user(&offset, argp, sizeof(offset))) 1926 return -EFAULT; 1927 return kvm_vm_ioctl_set_counter_offset(kvm, &offset); 1928 } 1929 case KVM_HAS_DEVICE_ATTR: { 1930 if (copy_from_user(&attr, argp, sizeof(attr))) 1931 return -EFAULT; 1932 1933 return kvm_vm_has_attr(kvm, &attr); 1934 } 1935 case KVM_SET_DEVICE_ATTR: { 1936 if (copy_from_user(&attr, argp, sizeof(attr))) 1937 return -EFAULT; 1938 1939 return kvm_vm_set_attr(kvm, &attr); 1940 } 1941 case KVM_ARM_GET_REG_WRITABLE_MASKS: { 1942 struct reg_mask_range range; 1943 1944 if (copy_from_user(&range, argp, sizeof(range))) 1945 return -EFAULT; 1946 return kvm_vm_ioctl_get_reg_writable_masks(kvm, &range); 1947 } 1948 default: 1949 return -EINVAL; 1950 } 1951 } 1952 1953 static unsigned long nvhe_percpu_size(void) 1954 { 1955 return (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_end) - 1956 (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_start); 1957 } 1958 1959 static unsigned long nvhe_percpu_order(void) 1960 { 1961 unsigned long size = nvhe_percpu_size(); 1962 1963 return size ? get_order(size) : 0; 1964 } 1965 1966 static size_t pkvm_host_sve_state_order(void) 1967 { 1968 return get_order(pkvm_host_sve_state_size()); 1969 } 1970 1971 /* A lookup table holding the hypervisor VA for each vector slot */ 1972 static void *hyp_spectre_vector_selector[BP_HARDEN_EL2_SLOTS]; 1973 1974 static void kvm_init_vector_slot(void *base, enum arm64_hyp_spectre_vector slot) 1975 { 1976 hyp_spectre_vector_selector[slot] = __kvm_vector_slot2addr(base, slot); 1977 } 1978 1979 static int kvm_init_vector_slots(void) 1980 { 1981 int err; 1982 void *base; 1983 1984 base = kern_hyp_va(kvm_ksym_ref(__kvm_hyp_vector)); 1985 kvm_init_vector_slot(base, HYP_VECTOR_DIRECT); 1986 1987 base = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs)); 1988 kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_DIRECT); 1989 1990 if (kvm_system_needs_idmapped_vectors() && 1991 !is_protected_kvm_enabled()) { 1992 err = create_hyp_exec_mappings(__pa_symbol(__bp_harden_hyp_vecs), 1993 __BP_HARDEN_HYP_VECS_SZ, &base); 1994 if (err) 1995 return err; 1996 } 1997 1998 kvm_init_vector_slot(base, HYP_VECTOR_INDIRECT); 1999 kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_INDIRECT); 2000 return 0; 2001 } 2002 2003 static void __init cpu_prepare_hyp_mode(int cpu, u32 hyp_va_bits) 2004 { 2005 struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu); 2006 unsigned long tcr; 2007 2008 /* 2009 * Calculate the raw per-cpu offset without a translation from the 2010 * kernel's mapping to the linear mapping, and store it in tpidr_el2 2011 * so that we can use adr_l to access per-cpu variables in EL2. 2012 * Also drop the KASAN tag which gets in the way... 2013 */ 2014 params->tpidr_el2 = (unsigned long)kasan_reset_tag(per_cpu_ptr_nvhe_sym(__per_cpu_start, cpu)) - 2015 (unsigned long)kvm_ksym_ref(CHOOSE_NVHE_SYM(__per_cpu_start)); 2016 2017 params->mair_el2 = read_sysreg(mair_el1); 2018 2019 tcr = read_sysreg(tcr_el1); 2020 if (cpus_have_final_cap(ARM64_KVM_HVHE)) { 2021 tcr &= ~(TCR_HD | TCR_HA | TCR_A1 | TCR_T0SZ_MASK); 2022 tcr |= TCR_EPD1_MASK; 2023 } else { 2024 unsigned long ips = FIELD_GET(TCR_IPS_MASK, tcr); 2025 2026 tcr &= TCR_EL2_MASK; 2027 tcr |= TCR_EL2_RES1 | FIELD_PREP(TCR_EL2_PS_MASK, ips); 2028 if (lpa2_is_enabled()) 2029 tcr |= TCR_EL2_DS; 2030 } 2031 tcr |= TCR_T0SZ(hyp_va_bits); 2032 params->tcr_el2 = tcr; 2033 2034 params->pgd_pa = kvm_mmu_get_httbr(); 2035 if (is_protected_kvm_enabled()) 2036 params->hcr_el2 = HCR_HOST_NVHE_PROTECTED_FLAGS; 2037 else 2038 params->hcr_el2 = HCR_HOST_NVHE_FLAGS; 2039 if (cpus_have_final_cap(ARM64_KVM_HVHE)) 2040 params->hcr_el2 |= HCR_E2H; 2041 params->vttbr = params->vtcr = 0; 2042 2043 /* 2044 * Flush the init params from the data cache because the struct will 2045 * be read while the MMU is off. 2046 */ 2047 kvm_flush_dcache_to_poc(params, sizeof(*params)); 2048 } 2049 2050 static void hyp_install_host_vector(void) 2051 { 2052 struct kvm_nvhe_init_params *params; 2053 struct arm_smccc_res res; 2054 2055 /* Switch from the HYP stub to our own HYP init vector */ 2056 __hyp_set_vectors(kvm_get_idmap_vector()); 2057 2058 /* 2059 * Call initialization code, and switch to the full blown HYP code. 2060 * If the cpucaps haven't been finalized yet, something has gone very 2061 * wrong, and hyp will crash and burn when it uses any 2062 * cpus_have_*_cap() wrapper. 2063 */ 2064 BUG_ON(!system_capabilities_finalized()); 2065 params = this_cpu_ptr_nvhe_sym(kvm_init_params); 2066 arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__kvm_hyp_init), virt_to_phys(params), &res); 2067 WARN_ON(res.a0 != SMCCC_RET_SUCCESS); 2068 } 2069 2070 static void cpu_init_hyp_mode(void) 2071 { 2072 hyp_install_host_vector(); 2073 2074 /* 2075 * Disabling SSBD on a non-VHE system requires us to enable SSBS 2076 * at EL2. 2077 */ 2078 if (this_cpu_has_cap(ARM64_SSBS) && 2079 arm64_get_spectre_v4_state() == SPECTRE_VULNERABLE) { 2080 kvm_call_hyp_nvhe(__kvm_enable_ssbs); 2081 } 2082 } 2083 2084 static void cpu_hyp_reset(void) 2085 { 2086 if (!is_kernel_in_hyp_mode()) 2087 __hyp_reset_vectors(); 2088 } 2089 2090 /* 2091 * EL2 vectors can be mapped and rerouted in a number of ways, 2092 * depending on the kernel configuration and CPU present: 2093 * 2094 * - If the CPU is affected by Spectre-v2, the hardening sequence is 2095 * placed in one of the vector slots, which is executed before jumping 2096 * to the real vectors. 2097 * 2098 * - If the CPU also has the ARM64_SPECTRE_V3A cap, the slot 2099 * containing the hardening sequence is mapped next to the idmap page, 2100 * and executed before jumping to the real vectors. 2101 * 2102 * - If the CPU only has the ARM64_SPECTRE_V3A cap, then an 2103 * empty slot is selected, mapped next to the idmap page, and 2104 * executed before jumping to the real vectors. 2105 * 2106 * Note that ARM64_SPECTRE_V3A is somewhat incompatible with 2107 * VHE, as we don't have hypervisor-specific mappings. If the system 2108 * is VHE and yet selects this capability, it will be ignored. 2109 */ 2110 static void cpu_set_hyp_vector(void) 2111 { 2112 struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data); 2113 void *vector = hyp_spectre_vector_selector[data->slot]; 2114 2115 if (!is_protected_kvm_enabled()) 2116 *this_cpu_ptr_hyp_sym(kvm_hyp_vector) = (unsigned long)vector; 2117 else 2118 kvm_call_hyp_nvhe(__pkvm_cpu_set_vector, data->slot); 2119 } 2120 2121 static void cpu_hyp_init_context(void) 2122 { 2123 kvm_init_host_cpu_context(host_data_ptr(host_ctxt)); 2124 kvm_init_host_debug_data(); 2125 2126 if (!is_kernel_in_hyp_mode()) 2127 cpu_init_hyp_mode(); 2128 } 2129 2130 static void cpu_hyp_init_features(void) 2131 { 2132 cpu_set_hyp_vector(); 2133 2134 if (is_kernel_in_hyp_mode()) { 2135 kvm_timer_init_vhe(); 2136 kvm_debug_init_vhe(); 2137 } 2138 2139 if (vgic_present) 2140 kvm_vgic_init_cpu_hardware(); 2141 } 2142 2143 static void cpu_hyp_reinit(void) 2144 { 2145 cpu_hyp_reset(); 2146 cpu_hyp_init_context(); 2147 cpu_hyp_init_features(); 2148 } 2149 2150 static void cpu_hyp_init(void *discard) 2151 { 2152 if (!__this_cpu_read(kvm_hyp_initialized)) { 2153 cpu_hyp_reinit(); 2154 __this_cpu_write(kvm_hyp_initialized, 1); 2155 } 2156 } 2157 2158 static void cpu_hyp_uninit(void *discard) 2159 { 2160 if (!is_protected_kvm_enabled() && __this_cpu_read(kvm_hyp_initialized)) { 2161 cpu_hyp_reset(); 2162 __this_cpu_write(kvm_hyp_initialized, 0); 2163 } 2164 } 2165 2166 int kvm_arch_enable_virtualization_cpu(void) 2167 { 2168 /* 2169 * Most calls to this function are made with migration 2170 * disabled, but not with preemption disabled. The former is 2171 * enough to ensure correctness, but most of the helpers 2172 * expect the later and will throw a tantrum otherwise. 2173 */ 2174 preempt_disable(); 2175 2176 cpu_hyp_init(NULL); 2177 2178 kvm_vgic_cpu_up(); 2179 kvm_timer_cpu_up(); 2180 2181 preempt_enable(); 2182 2183 return 0; 2184 } 2185 2186 void kvm_arch_disable_virtualization_cpu(void) 2187 { 2188 kvm_timer_cpu_down(); 2189 kvm_vgic_cpu_down(); 2190 2191 if (!is_protected_kvm_enabled()) 2192 cpu_hyp_uninit(NULL); 2193 } 2194 2195 #ifdef CONFIG_CPU_PM 2196 static int hyp_init_cpu_pm_notifier(struct notifier_block *self, 2197 unsigned long cmd, 2198 void *v) 2199 { 2200 /* 2201 * kvm_hyp_initialized is left with its old value over 2202 * PM_ENTER->PM_EXIT. It is used to indicate PM_EXIT should 2203 * re-enable hyp. 2204 */ 2205 switch (cmd) { 2206 case CPU_PM_ENTER: 2207 if (__this_cpu_read(kvm_hyp_initialized)) 2208 /* 2209 * don't update kvm_hyp_initialized here 2210 * so that the hyp will be re-enabled 2211 * when we resume. See below. 2212 */ 2213 cpu_hyp_reset(); 2214 2215 return NOTIFY_OK; 2216 case CPU_PM_ENTER_FAILED: 2217 case CPU_PM_EXIT: 2218 if (__this_cpu_read(kvm_hyp_initialized)) 2219 /* The hyp was enabled before suspend. */ 2220 cpu_hyp_reinit(); 2221 2222 return NOTIFY_OK; 2223 2224 default: 2225 return NOTIFY_DONE; 2226 } 2227 } 2228 2229 static struct notifier_block hyp_init_cpu_pm_nb = { 2230 .notifier_call = hyp_init_cpu_pm_notifier, 2231 }; 2232 2233 static void __init hyp_cpu_pm_init(void) 2234 { 2235 if (!is_protected_kvm_enabled()) 2236 cpu_pm_register_notifier(&hyp_init_cpu_pm_nb); 2237 } 2238 static void __init hyp_cpu_pm_exit(void) 2239 { 2240 if (!is_protected_kvm_enabled()) 2241 cpu_pm_unregister_notifier(&hyp_init_cpu_pm_nb); 2242 } 2243 #else 2244 static inline void __init hyp_cpu_pm_init(void) 2245 { 2246 } 2247 static inline void __init hyp_cpu_pm_exit(void) 2248 { 2249 } 2250 #endif 2251 2252 static void __init init_cpu_logical_map(void) 2253 { 2254 unsigned int cpu; 2255 2256 /* 2257 * Copy the MPIDR <-> logical CPU ID mapping to hyp. 2258 * Only copy the set of online CPUs whose features have been checked 2259 * against the finalized system capabilities. The hypervisor will not 2260 * allow any other CPUs from the `possible` set to boot. 2261 */ 2262 for_each_online_cpu(cpu) 2263 hyp_cpu_logical_map[cpu] = cpu_logical_map(cpu); 2264 } 2265 2266 #define init_psci_0_1_impl_state(config, what) \ 2267 config.psci_0_1_ ## what ## _implemented = psci_ops.what 2268 2269 static bool __init init_psci_relay(void) 2270 { 2271 /* 2272 * If PSCI has not been initialized, protected KVM cannot install 2273 * itself on newly booted CPUs. 2274 */ 2275 if (!psci_ops.get_version) { 2276 kvm_err("Cannot initialize protected mode without PSCI\n"); 2277 return false; 2278 } 2279 2280 kvm_host_psci_config.version = psci_ops.get_version(); 2281 kvm_host_psci_config.smccc_version = arm_smccc_get_version(); 2282 2283 if (kvm_host_psci_config.version == PSCI_VERSION(0, 1)) { 2284 kvm_host_psci_config.function_ids_0_1 = get_psci_0_1_function_ids(); 2285 init_psci_0_1_impl_state(kvm_host_psci_config, cpu_suspend); 2286 init_psci_0_1_impl_state(kvm_host_psci_config, cpu_on); 2287 init_psci_0_1_impl_state(kvm_host_psci_config, cpu_off); 2288 init_psci_0_1_impl_state(kvm_host_psci_config, migrate); 2289 } 2290 return true; 2291 } 2292 2293 static int __init init_subsystems(void) 2294 { 2295 int err = 0; 2296 2297 /* 2298 * Enable hardware so that subsystem initialisation can access EL2. 2299 */ 2300 on_each_cpu(cpu_hyp_init, NULL, 1); 2301 2302 /* 2303 * Register CPU lower-power notifier 2304 */ 2305 hyp_cpu_pm_init(); 2306 2307 /* 2308 * Init HYP view of VGIC 2309 */ 2310 err = kvm_vgic_hyp_init(); 2311 switch (err) { 2312 case 0: 2313 vgic_present = true; 2314 break; 2315 case -ENODEV: 2316 case -ENXIO: 2317 /* 2318 * No VGIC? No pKVM for you. 2319 * 2320 * Protected mode assumes that VGICv3 is present, so no point 2321 * in trying to hobble along if vgic initialization fails. 2322 */ 2323 if (is_protected_kvm_enabled()) 2324 goto out; 2325 2326 /* 2327 * Otherwise, userspace could choose to implement a GIC for its 2328 * guest on non-cooperative hardware. 2329 */ 2330 vgic_present = false; 2331 err = 0; 2332 break; 2333 default: 2334 goto out; 2335 } 2336 2337 if (kvm_mode == KVM_MODE_NV && 2338 !(vgic_present && (kvm_vgic_global_state.type == VGIC_V3 || 2339 kvm_vgic_global_state.has_gcie_v3_compat))) { 2340 kvm_err("NV support requires GICv3 or GICv5 with legacy support, giving up\n"); 2341 err = -EINVAL; 2342 goto out; 2343 } 2344 2345 /* 2346 * Init HYP architected timer support 2347 */ 2348 err = kvm_timer_hyp_init(vgic_present); 2349 if (err) 2350 goto out; 2351 2352 kvm_register_perf_callbacks(NULL); 2353 2354 out: 2355 if (err) 2356 hyp_cpu_pm_exit(); 2357 2358 if (err || !is_protected_kvm_enabled()) 2359 on_each_cpu(cpu_hyp_uninit, NULL, 1); 2360 2361 return err; 2362 } 2363 2364 static void __init teardown_subsystems(void) 2365 { 2366 kvm_unregister_perf_callbacks(); 2367 hyp_cpu_pm_exit(); 2368 } 2369 2370 static void __init teardown_hyp_mode(void) 2371 { 2372 bool free_sve = system_supports_sve() && is_protected_kvm_enabled(); 2373 int cpu; 2374 2375 free_hyp_pgds(); 2376 for_each_possible_cpu(cpu) { 2377 if (per_cpu(kvm_hyp_initialized, cpu)) 2378 continue; 2379 2380 free_pages(per_cpu(kvm_arm_hyp_stack_base, cpu), NVHE_STACK_SHIFT - PAGE_SHIFT); 2381 2382 if (!kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu]) 2383 continue; 2384 2385 if (free_sve) { 2386 struct cpu_sve_state *sve_state; 2387 2388 sve_state = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state; 2389 free_pages((unsigned long) sve_state, pkvm_host_sve_state_order()); 2390 } 2391 2392 free_pages(kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu], nvhe_percpu_order()); 2393 2394 } 2395 } 2396 2397 static int __init do_pkvm_init(u32 hyp_va_bits) 2398 { 2399 void *per_cpu_base = kvm_ksym_ref(kvm_nvhe_sym(kvm_arm_hyp_percpu_base)); 2400 int ret; 2401 2402 preempt_disable(); 2403 cpu_hyp_init_context(); 2404 ret = kvm_call_hyp_nvhe(__pkvm_init, hyp_mem_base, hyp_mem_size, 2405 num_possible_cpus(), kern_hyp_va(per_cpu_base), 2406 hyp_va_bits); 2407 cpu_hyp_init_features(); 2408 2409 /* 2410 * The stub hypercalls are now disabled, so set our local flag to 2411 * prevent a later re-init attempt in kvm_arch_enable_virtualization_cpu(). 2412 */ 2413 __this_cpu_write(kvm_hyp_initialized, 1); 2414 preempt_enable(); 2415 2416 return ret; 2417 } 2418 2419 static u64 get_hyp_id_aa64pfr0_el1(void) 2420 { 2421 /* 2422 * Track whether the system isn't affected by spectre/meltdown in the 2423 * hypervisor's view of id_aa64pfr0_el1, used for protected VMs. 2424 * Although this is per-CPU, we make it global for simplicity, e.g., not 2425 * to have to worry about vcpu migration. 2426 * 2427 * Unlike for non-protected VMs, userspace cannot override this for 2428 * protected VMs. 2429 */ 2430 u64 val = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); 2431 2432 val &= ~(ID_AA64PFR0_EL1_CSV2 | 2433 ID_AA64PFR0_EL1_CSV3); 2434 2435 val |= FIELD_PREP(ID_AA64PFR0_EL1_CSV2, 2436 arm64_get_spectre_v2_state() == SPECTRE_UNAFFECTED); 2437 val |= FIELD_PREP(ID_AA64PFR0_EL1_CSV3, 2438 arm64_get_meltdown_state() == SPECTRE_UNAFFECTED); 2439 2440 return val; 2441 } 2442 2443 static void kvm_hyp_init_symbols(void) 2444 { 2445 kvm_nvhe_sym(id_aa64pfr0_el1_sys_val) = get_hyp_id_aa64pfr0_el1(); 2446 kvm_nvhe_sym(id_aa64pfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1); 2447 kvm_nvhe_sym(id_aa64isar0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR0_EL1); 2448 kvm_nvhe_sym(id_aa64isar1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR1_EL1); 2449 kvm_nvhe_sym(id_aa64isar2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1); 2450 kvm_nvhe_sym(id_aa64mmfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); 2451 kvm_nvhe_sym(id_aa64mmfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1); 2452 kvm_nvhe_sym(id_aa64mmfr2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR2_EL1); 2453 kvm_nvhe_sym(id_aa64smfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64SMFR0_EL1); 2454 kvm_nvhe_sym(__icache_flags) = __icache_flags; 2455 kvm_nvhe_sym(kvm_arm_vmid_bits) = kvm_arm_vmid_bits; 2456 2457 /* Propagate the FGT state to the the nVHE side */ 2458 kvm_nvhe_sym(hfgrtr_masks) = hfgrtr_masks; 2459 kvm_nvhe_sym(hfgwtr_masks) = hfgwtr_masks; 2460 kvm_nvhe_sym(hfgitr_masks) = hfgitr_masks; 2461 kvm_nvhe_sym(hdfgrtr_masks) = hdfgrtr_masks; 2462 kvm_nvhe_sym(hdfgwtr_masks) = hdfgwtr_masks; 2463 kvm_nvhe_sym(hafgrtr_masks) = hafgrtr_masks; 2464 kvm_nvhe_sym(hfgrtr2_masks) = hfgrtr2_masks; 2465 kvm_nvhe_sym(hfgwtr2_masks) = hfgwtr2_masks; 2466 kvm_nvhe_sym(hfgitr2_masks) = hfgitr2_masks; 2467 kvm_nvhe_sym(hdfgrtr2_masks)= hdfgrtr2_masks; 2468 kvm_nvhe_sym(hdfgwtr2_masks)= hdfgwtr2_masks; 2469 2470 /* 2471 * Flush entire BSS since part of its data containing init symbols is read 2472 * while the MMU is off. 2473 */ 2474 kvm_flush_dcache_to_poc(kvm_ksym_ref(__hyp_bss_start), 2475 kvm_ksym_ref(__hyp_bss_end) - kvm_ksym_ref(__hyp_bss_start)); 2476 } 2477 2478 static int __init kvm_hyp_init_protection(u32 hyp_va_bits) 2479 { 2480 void *addr = phys_to_virt(hyp_mem_base); 2481 int ret; 2482 2483 ret = create_hyp_mappings(addr, addr + hyp_mem_size, PAGE_HYP); 2484 if (ret) 2485 return ret; 2486 2487 ret = do_pkvm_init(hyp_va_bits); 2488 if (ret) 2489 return ret; 2490 2491 free_hyp_pgds(); 2492 2493 return 0; 2494 } 2495 2496 static int init_pkvm_host_sve_state(void) 2497 { 2498 int cpu; 2499 2500 if (!system_supports_sve()) 2501 return 0; 2502 2503 /* Allocate pages for host sve state in protected mode. */ 2504 for_each_possible_cpu(cpu) { 2505 struct page *page = alloc_pages(GFP_KERNEL, pkvm_host_sve_state_order()); 2506 2507 if (!page) 2508 return -ENOMEM; 2509 2510 per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state = page_address(page); 2511 } 2512 2513 /* 2514 * Don't map the pages in hyp since these are only used in protected 2515 * mode, which will (re)create its own mapping when initialized. 2516 */ 2517 2518 return 0; 2519 } 2520 2521 /* 2522 * Finalizes the initialization of hyp mode, once everything else is initialized 2523 * and the initialziation process cannot fail. 2524 */ 2525 static void finalize_init_hyp_mode(void) 2526 { 2527 int cpu; 2528 2529 if (system_supports_sve() && is_protected_kvm_enabled()) { 2530 for_each_possible_cpu(cpu) { 2531 struct cpu_sve_state *sve_state; 2532 2533 sve_state = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state; 2534 per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state = 2535 kern_hyp_va(sve_state); 2536 } 2537 } 2538 } 2539 2540 static void pkvm_hyp_init_ptrauth(void) 2541 { 2542 struct kvm_cpu_context *hyp_ctxt; 2543 int cpu; 2544 2545 for_each_possible_cpu(cpu) { 2546 hyp_ctxt = per_cpu_ptr_nvhe_sym(kvm_hyp_ctxt, cpu); 2547 hyp_ctxt->sys_regs[APIAKEYLO_EL1] = get_random_long(); 2548 hyp_ctxt->sys_regs[APIAKEYHI_EL1] = get_random_long(); 2549 hyp_ctxt->sys_regs[APIBKEYLO_EL1] = get_random_long(); 2550 hyp_ctxt->sys_regs[APIBKEYHI_EL1] = get_random_long(); 2551 hyp_ctxt->sys_regs[APDAKEYLO_EL1] = get_random_long(); 2552 hyp_ctxt->sys_regs[APDAKEYHI_EL1] = get_random_long(); 2553 hyp_ctxt->sys_regs[APDBKEYLO_EL1] = get_random_long(); 2554 hyp_ctxt->sys_regs[APDBKEYHI_EL1] = get_random_long(); 2555 hyp_ctxt->sys_regs[APGAKEYLO_EL1] = get_random_long(); 2556 hyp_ctxt->sys_regs[APGAKEYHI_EL1] = get_random_long(); 2557 } 2558 } 2559 2560 /* Inits Hyp-mode on all online CPUs */ 2561 static int __init init_hyp_mode(void) 2562 { 2563 u32 hyp_va_bits; 2564 int cpu; 2565 int err = -ENOMEM; 2566 2567 /* 2568 * The protected Hyp-mode cannot be initialized if the memory pool 2569 * allocation has failed. 2570 */ 2571 if (is_protected_kvm_enabled() && !hyp_mem_base) 2572 goto out_err; 2573 2574 /* 2575 * Allocate Hyp PGD and setup Hyp identity mapping 2576 */ 2577 err = kvm_mmu_init(&hyp_va_bits); 2578 if (err) 2579 goto out_err; 2580 2581 /* 2582 * Allocate stack pages for Hypervisor-mode 2583 */ 2584 for_each_possible_cpu(cpu) { 2585 unsigned long stack_base; 2586 2587 stack_base = __get_free_pages(GFP_KERNEL, NVHE_STACK_SHIFT - PAGE_SHIFT); 2588 if (!stack_base) { 2589 err = -ENOMEM; 2590 goto out_err; 2591 } 2592 2593 per_cpu(kvm_arm_hyp_stack_base, cpu) = stack_base; 2594 } 2595 2596 /* 2597 * Allocate and initialize pages for Hypervisor-mode percpu regions. 2598 */ 2599 for_each_possible_cpu(cpu) { 2600 struct page *page; 2601 void *page_addr; 2602 2603 page = alloc_pages(GFP_KERNEL, nvhe_percpu_order()); 2604 if (!page) { 2605 err = -ENOMEM; 2606 goto out_err; 2607 } 2608 2609 page_addr = page_address(page); 2610 memcpy(page_addr, CHOOSE_NVHE_SYM(__per_cpu_start), nvhe_percpu_size()); 2611 kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu] = (unsigned long)page_addr; 2612 } 2613 2614 /* 2615 * Map the Hyp-code called directly from the host 2616 */ 2617 err = create_hyp_mappings(kvm_ksym_ref(__hyp_text_start), 2618 kvm_ksym_ref(__hyp_text_end), PAGE_HYP_EXEC); 2619 if (err) { 2620 kvm_err("Cannot map world-switch code\n"); 2621 goto out_err; 2622 } 2623 2624 err = create_hyp_mappings(kvm_ksym_ref(__hyp_data_start), 2625 kvm_ksym_ref(__hyp_data_end), PAGE_HYP); 2626 if (err) { 2627 kvm_err("Cannot map .hyp.data section\n"); 2628 goto out_err; 2629 } 2630 2631 err = create_hyp_mappings(kvm_ksym_ref(__hyp_rodata_start), 2632 kvm_ksym_ref(__hyp_rodata_end), PAGE_HYP_RO); 2633 if (err) { 2634 kvm_err("Cannot map .hyp.rodata section\n"); 2635 goto out_err; 2636 } 2637 2638 err = create_hyp_mappings(kvm_ksym_ref(__start_rodata), 2639 kvm_ksym_ref(__end_rodata), PAGE_HYP_RO); 2640 if (err) { 2641 kvm_err("Cannot map rodata section\n"); 2642 goto out_err; 2643 } 2644 2645 /* 2646 * .hyp.bss is guaranteed to be placed at the beginning of the .bss 2647 * section thanks to an assertion in the linker script. Map it RW and 2648 * the rest of .bss RO. 2649 */ 2650 err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_start), 2651 kvm_ksym_ref(__hyp_bss_end), PAGE_HYP); 2652 if (err) { 2653 kvm_err("Cannot map hyp bss section: %d\n", err); 2654 goto out_err; 2655 } 2656 2657 err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_end), 2658 kvm_ksym_ref(__bss_stop), PAGE_HYP_RO); 2659 if (err) { 2660 kvm_err("Cannot map bss section\n"); 2661 goto out_err; 2662 } 2663 2664 /* 2665 * Map the Hyp stack pages 2666 */ 2667 for_each_possible_cpu(cpu) { 2668 struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu); 2669 char *stack_base = (char *)per_cpu(kvm_arm_hyp_stack_base, cpu); 2670 2671 err = create_hyp_stack(__pa(stack_base), ¶ms->stack_hyp_va); 2672 if (err) { 2673 kvm_err("Cannot map hyp stack\n"); 2674 goto out_err; 2675 } 2676 2677 /* 2678 * Save the stack PA in nvhe_init_params. This will be needed 2679 * to recreate the stack mapping in protected nVHE mode. 2680 * __hyp_pa() won't do the right thing there, since the stack 2681 * has been mapped in the flexible private VA space. 2682 */ 2683 params->stack_pa = __pa(stack_base); 2684 } 2685 2686 for_each_possible_cpu(cpu) { 2687 char *percpu_begin = (char *)kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu]; 2688 char *percpu_end = percpu_begin + nvhe_percpu_size(); 2689 2690 /* Map Hyp percpu pages */ 2691 err = create_hyp_mappings(percpu_begin, percpu_end, PAGE_HYP); 2692 if (err) { 2693 kvm_err("Cannot map hyp percpu region\n"); 2694 goto out_err; 2695 } 2696 2697 /* Prepare the CPU initialization parameters */ 2698 cpu_prepare_hyp_mode(cpu, hyp_va_bits); 2699 } 2700 2701 kvm_hyp_init_symbols(); 2702 2703 if (is_protected_kvm_enabled()) { 2704 if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL) && 2705 cpus_have_final_cap(ARM64_HAS_ADDRESS_AUTH)) 2706 pkvm_hyp_init_ptrauth(); 2707 2708 init_cpu_logical_map(); 2709 2710 if (!init_psci_relay()) { 2711 err = -ENODEV; 2712 goto out_err; 2713 } 2714 2715 err = init_pkvm_host_sve_state(); 2716 if (err) 2717 goto out_err; 2718 2719 err = kvm_hyp_init_protection(hyp_va_bits); 2720 if (err) { 2721 kvm_err("Failed to init hyp memory protection\n"); 2722 goto out_err; 2723 } 2724 } 2725 2726 return 0; 2727 2728 out_err: 2729 teardown_hyp_mode(); 2730 kvm_err("error initializing Hyp mode: %d\n", err); 2731 return err; 2732 } 2733 2734 struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr) 2735 { 2736 struct kvm_vcpu *vcpu = NULL; 2737 struct kvm_mpidr_data *data; 2738 unsigned long i; 2739 2740 mpidr &= MPIDR_HWID_BITMASK; 2741 2742 rcu_read_lock(); 2743 data = rcu_dereference(kvm->arch.mpidr_data); 2744 2745 if (data) { 2746 u16 idx = kvm_mpidr_index(data, mpidr); 2747 2748 vcpu = kvm_get_vcpu(kvm, data->cmpidr_to_idx[idx]); 2749 if (mpidr != kvm_vcpu_get_mpidr_aff(vcpu)) 2750 vcpu = NULL; 2751 } 2752 2753 rcu_read_unlock(); 2754 2755 if (vcpu) 2756 return vcpu; 2757 2758 kvm_for_each_vcpu(i, vcpu, kvm) { 2759 if (mpidr == kvm_vcpu_get_mpidr_aff(vcpu)) 2760 return vcpu; 2761 } 2762 return NULL; 2763 } 2764 2765 bool kvm_arch_irqchip_in_kernel(struct kvm *kvm) 2766 { 2767 return irqchip_in_kernel(kvm); 2768 } 2769 2770 int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons, 2771 struct irq_bypass_producer *prod) 2772 { 2773 struct kvm_kernel_irqfd *irqfd = 2774 container_of(cons, struct kvm_kernel_irqfd, consumer); 2775 struct kvm_kernel_irq_routing_entry *irq_entry = &irqfd->irq_entry; 2776 2777 /* 2778 * The only thing we have a chance of directly-injecting is LPIs. Maybe 2779 * one day... 2780 */ 2781 if (irq_entry->type != KVM_IRQ_ROUTING_MSI) 2782 return 0; 2783 2784 return kvm_vgic_v4_set_forwarding(irqfd->kvm, prod->irq, 2785 &irqfd->irq_entry); 2786 } 2787 2788 void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons, 2789 struct irq_bypass_producer *prod) 2790 { 2791 struct kvm_kernel_irqfd *irqfd = 2792 container_of(cons, struct kvm_kernel_irqfd, consumer); 2793 struct kvm_kernel_irq_routing_entry *irq_entry = &irqfd->irq_entry; 2794 2795 if (irq_entry->type != KVM_IRQ_ROUTING_MSI) 2796 return; 2797 2798 kvm_vgic_v4_unset_forwarding(irqfd->kvm, prod->irq); 2799 } 2800 2801 void kvm_arch_update_irqfd_routing(struct kvm_kernel_irqfd *irqfd, 2802 struct kvm_kernel_irq_routing_entry *old, 2803 struct kvm_kernel_irq_routing_entry *new) 2804 { 2805 if (old->type == KVM_IRQ_ROUTING_MSI && 2806 new->type == KVM_IRQ_ROUTING_MSI && 2807 !memcmp(&old->msi, &new->msi, sizeof(new->msi))) 2808 return; 2809 2810 /* 2811 * Remapping the vLPI requires taking the its_lock mutex to resolve 2812 * the new translation. We're in spinlock land at this point, so no 2813 * chance of resolving the translation. 2814 * 2815 * Unmap the vLPI and fall back to software LPI injection. 2816 */ 2817 return kvm_vgic_v4_unset_forwarding(irqfd->kvm, irqfd->producer->irq); 2818 } 2819 2820 void kvm_arch_irq_bypass_stop(struct irq_bypass_consumer *cons) 2821 { 2822 struct kvm_kernel_irqfd *irqfd = 2823 container_of(cons, struct kvm_kernel_irqfd, consumer); 2824 2825 kvm_arm_halt_guest(irqfd->kvm); 2826 } 2827 2828 void kvm_arch_irq_bypass_start(struct irq_bypass_consumer *cons) 2829 { 2830 struct kvm_kernel_irqfd *irqfd = 2831 container_of(cons, struct kvm_kernel_irqfd, consumer); 2832 2833 kvm_arm_resume_guest(irqfd->kvm); 2834 } 2835 2836 /* Initialize Hyp-mode and memory mappings on all CPUs */ 2837 static __init int kvm_arm_init(void) 2838 { 2839 int err; 2840 bool in_hyp_mode; 2841 2842 if (!is_hyp_mode_available()) { 2843 kvm_info("HYP mode not available\n"); 2844 return -ENODEV; 2845 } 2846 2847 if (kvm_get_mode() == KVM_MODE_NONE) { 2848 kvm_info("KVM disabled from command line\n"); 2849 return -ENODEV; 2850 } 2851 2852 err = kvm_sys_reg_table_init(); 2853 if (err) { 2854 kvm_info("Error initializing system register tables"); 2855 return err; 2856 } 2857 2858 in_hyp_mode = is_kernel_in_hyp_mode(); 2859 2860 if (cpus_have_final_cap(ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) || 2861 cpus_have_final_cap(ARM64_WORKAROUND_1508412)) 2862 kvm_info("Guests without required CPU erratum workarounds can deadlock system!\n" \ 2863 "Only trusted guests should be used on this system.\n"); 2864 2865 err = kvm_set_ipa_limit(); 2866 if (err) 2867 return err; 2868 2869 err = kvm_arm_init_sve(); 2870 if (err) 2871 return err; 2872 2873 err = kvm_arm_vmid_alloc_init(); 2874 if (err) { 2875 kvm_err("Failed to initialize VMID allocator.\n"); 2876 return err; 2877 } 2878 2879 if (!in_hyp_mode) { 2880 err = init_hyp_mode(); 2881 if (err) 2882 goto out_err; 2883 } 2884 2885 err = kvm_init_vector_slots(); 2886 if (err) { 2887 kvm_err("Cannot initialise vector slots\n"); 2888 goto out_hyp; 2889 } 2890 2891 err = init_subsystems(); 2892 if (err) 2893 goto out_hyp; 2894 2895 kvm_info("%s%sVHE%s mode initialized successfully\n", 2896 in_hyp_mode ? "" : (is_protected_kvm_enabled() ? 2897 "Protected " : "Hyp "), 2898 in_hyp_mode ? "" : (cpus_have_final_cap(ARM64_KVM_HVHE) ? 2899 "h" : "n"), 2900 cpus_have_final_cap(ARM64_HAS_NESTED_VIRT) ? "+NV2": ""); 2901 2902 /* 2903 * FIXME: Do something reasonable if kvm_init() fails after pKVM 2904 * hypervisor protection is finalized. 2905 */ 2906 err = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE); 2907 if (err) 2908 goto out_subs; 2909 2910 /* 2911 * This should be called after initialization is done and failure isn't 2912 * possible anymore. 2913 */ 2914 if (!in_hyp_mode) 2915 finalize_init_hyp_mode(); 2916 2917 kvm_arm_initialised = true; 2918 2919 return 0; 2920 2921 out_subs: 2922 teardown_subsystems(); 2923 out_hyp: 2924 if (!in_hyp_mode) 2925 teardown_hyp_mode(); 2926 out_err: 2927 kvm_arm_vmid_alloc_free(); 2928 return err; 2929 } 2930 2931 static int __init early_kvm_mode_cfg(char *arg) 2932 { 2933 if (!arg) 2934 return -EINVAL; 2935 2936 if (strcmp(arg, "none") == 0) { 2937 kvm_mode = KVM_MODE_NONE; 2938 return 0; 2939 } 2940 2941 if (!is_hyp_mode_available()) { 2942 pr_warn_once("KVM is not available. Ignoring kvm-arm.mode\n"); 2943 return 0; 2944 } 2945 2946 if (strcmp(arg, "protected") == 0) { 2947 if (!is_kernel_in_hyp_mode()) 2948 kvm_mode = KVM_MODE_PROTECTED; 2949 else 2950 pr_warn_once("Protected KVM not available with VHE\n"); 2951 2952 return 0; 2953 } 2954 2955 if (strcmp(arg, "nvhe") == 0 && !WARN_ON(is_kernel_in_hyp_mode())) { 2956 kvm_mode = KVM_MODE_DEFAULT; 2957 return 0; 2958 } 2959 2960 if (strcmp(arg, "nested") == 0 && !WARN_ON(!is_kernel_in_hyp_mode())) { 2961 kvm_mode = KVM_MODE_NV; 2962 return 0; 2963 } 2964 2965 return -EINVAL; 2966 } 2967 early_param("kvm-arm.mode", early_kvm_mode_cfg); 2968 2969 static int __init early_kvm_wfx_trap_policy_cfg(char *arg, enum kvm_wfx_trap_policy *p) 2970 { 2971 if (!arg) 2972 return -EINVAL; 2973 2974 if (strcmp(arg, "trap") == 0) { 2975 *p = KVM_WFX_TRAP; 2976 return 0; 2977 } 2978 2979 if (strcmp(arg, "notrap") == 0) { 2980 *p = KVM_WFX_NOTRAP; 2981 return 0; 2982 } 2983 2984 return -EINVAL; 2985 } 2986 2987 static int __init early_kvm_wfi_trap_policy_cfg(char *arg) 2988 { 2989 return early_kvm_wfx_trap_policy_cfg(arg, &kvm_wfi_trap_policy); 2990 } 2991 early_param("kvm-arm.wfi_trap_policy", early_kvm_wfi_trap_policy_cfg); 2992 2993 static int __init early_kvm_wfe_trap_policy_cfg(char *arg) 2994 { 2995 return early_kvm_wfx_trap_policy_cfg(arg, &kvm_wfe_trap_policy); 2996 } 2997 early_param("kvm-arm.wfe_trap_policy", early_kvm_wfe_trap_policy_cfg); 2998 2999 enum kvm_mode kvm_get_mode(void) 3000 { 3001 return kvm_mode; 3002 } 3003 3004 module_init(kvm_arm_init);