virt/kvm/async_pf.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * kvm asynchronous fault support
0004  *
0005  * Copyright 2010 Red Hat, Inc.
0006  *
0007  * Author:
0008  *      Gleb Natapov <gleb@redhat.com>
0009  */
0010
0011 #include <linux/kvm_host.h>
0012 #include <linux/slab.h>
0013 #include <linux/module.h>
0014 #include <linux/mmu_context.h>
0015 #include <linux/sched/mm.h>
0016
0017 #include "async_pf.h"
0018 #include <trace/events/kvm.h>
0019
0020 static struct kmem_cache *async_pf_cache;
0021
0022 int kvm_async_pf_init(void)
0023 {
0024     async_pf_cache = KMEM_CACHE(kvm_async_pf, 0);
0025
0026     if (!async_pf_cache)
0027         return -ENOMEM;
0028
0029     return 0;
0030 }
0031
0032 void kvm_async_pf_deinit(void)
0033 {
0034     kmem_cache_destroy(async_pf_cache);
0035     async_pf_cache = NULL;
0036 }
0037
0038 void kvm_async_pf_vcpu_init(struct kvm_vcpu *vcpu)
0039 {
0040     INIT_LIST_HEAD(&vcpu->async_pf.done);
0041     INIT_LIST_HEAD(&vcpu->async_pf.queue);
0042     spin_lock_init(&vcpu->async_pf.lock);
0043 }
0044
0045 static void async_pf_execute(struct work_struct *work)
0046 {
0047     struct kvm_async_pf *apf =
0048         container_of(work, struct kvm_async_pf, work);
0049     struct mm_struct *mm = apf->mm;
0050     struct kvm_vcpu *vcpu = apf->vcpu;
0051     unsigned long addr = apf->addr;
0052     gpa_t cr2_or_gpa = apf->cr2_or_gpa;
0053     int locked = 1;
0054     bool first;
0055
0056     might_sleep();
0057
0058     /*
0059      * This work is run asynchronously to the task which owns
0060      * mm and might be done in another context, so we must
0061      * access remotely.
0062      */
0063     mmap_read_lock(mm);
0064     get_user_pages_remote(mm, addr, 1, FOLL_WRITE, NULL, NULL,
0065             &locked);
0066     if (locked)
0067         mmap_read_unlock(mm);
0068
0069     if (IS_ENABLED(CONFIG_KVM_ASYNC_PF_SYNC))
0070         kvm_arch_async_page_present(vcpu, apf);
0071
0072     spin_lock(&vcpu->async_pf.lock);
0073     first = list_empty(&vcpu->async_pf.done);
0074     list_add_tail(&apf->link, &vcpu->async_pf.done);
0075     apf->vcpu = NULL;
0076     spin_unlock(&vcpu->async_pf.lock);
0077
0078     if (!IS_ENABLED(CONFIG_KVM_ASYNC_PF_SYNC) && first)
0079         kvm_arch_async_page_present_queued(vcpu);
0080
0081     /*
0082      * apf may be freed by kvm_check_async_pf_completion() after
0083      * this point
0084      */
0085
0086     trace_kvm_async_pf_completed(addr, cr2_or_gpa);
0087
0088     __kvm_vcpu_wake_up(vcpu);
0089
0090     mmput(mm);
0091     kvm_put_kvm(vcpu->kvm);
0092 }
0093
0094 void kvm_clear_async_pf_completion_queue(struct kvm_vcpu *vcpu)
0095 {
0096     spin_lock(&vcpu->async_pf.lock);
0097
0098     /* cancel outstanding work queue item */
0099     while (!list_empty(&vcpu->async_pf.queue)) {
0100         struct kvm_async_pf *work =
0101             list_first_entry(&vcpu->async_pf.queue,
0102                      typeof(*work), queue);
0103         list_del(&work->queue);
0104
0105         /*
0106          * We know it's present in vcpu->async_pf.done, do
0107          * nothing here.
0108          */
0109         if (!work->vcpu)
0110             continue;
0111
0112         spin_unlock(&vcpu->async_pf.lock);
0113 #ifdef CONFIG_KVM_ASYNC_PF_SYNC
0114         flush_work(&work->work);
0115 #else
0116         if (cancel_work_sync(&work->work)) {
0117             mmput(work->mm);
0118             kvm_put_kvm(vcpu->kvm); /* == work->vcpu->kvm */
0119             kmem_cache_free(async_pf_cache, work);
0120         }
0121 #endif
0122         spin_lock(&vcpu->async_pf.lock);
0123     }
0124
0125     while (!list_empty(&vcpu->async_pf.done)) {
0126         struct kvm_async_pf *work =
0127             list_first_entry(&vcpu->async_pf.done,
0128                      typeof(*work), link);
0129         list_del(&work->link);
0130         kmem_cache_free(async_pf_cache, work);
0131     }
0132     spin_unlock(&vcpu->async_pf.lock);
0133
0134     vcpu->async_pf.queued = 0;
0135 }
0136
0137 void kvm_check_async_pf_completion(struct kvm_vcpu *vcpu)
0138 {
0139     struct kvm_async_pf *work;
0140
0141     while (!list_empty_careful(&vcpu->async_pf.done) &&
0142           kvm_arch_can_dequeue_async_page_present(vcpu)) {
0143         spin_lock(&vcpu->async_pf.lock);
0144         work = list_first_entry(&vcpu->async_pf.done, typeof(*work),
0145                           link);
0146         list_del(&work->link);
0147         spin_unlock(&vcpu->async_pf.lock);
0148
0149         kvm_arch_async_page_ready(vcpu, work);
0150         if (!IS_ENABLED(CONFIG_KVM_ASYNC_PF_SYNC))
0151             kvm_arch_async_page_present(vcpu, work);
0152
0153         list_del(&work->queue);
0154         vcpu->async_pf.queued--;
0155         kmem_cache_free(async_pf_cache, work);
0156     }
0157 }
0158
0159 /*
0160  * Try to schedule a job to handle page fault asynchronously. Returns 'true' on
0161  * success, 'false' on failure (page fault has to be handled synchronously).
0162  */
0163 bool kvm_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
0164             unsigned long hva, struct kvm_arch_async_pf *arch)
0165 {
0166     struct kvm_async_pf *work;
0167
0168     if (vcpu->async_pf.queued >= ASYNC_PF_PER_VCPU)
0169         return false;
0170
0171     /* Arch specific code should not do async PF in this case */
0172     if (unlikely(kvm_is_error_hva(hva)))
0173         return false;
0174
0175     /*
0176      * do alloc nowait since if we are going to sleep anyway we
0177      * may as well sleep faulting in page
0178      */
0179     work = kmem_cache_zalloc(async_pf_cache, GFP_NOWAIT | __GFP_NOWARN);
0180     if (!work)
0181         return false;
0182
0183     work->wakeup_all = false;
0184     work->vcpu = vcpu;
0185     work->cr2_or_gpa = cr2_or_gpa;
0186     work->addr = hva;
0187     work->arch = *arch;
0188     work->mm = current->mm;
0189     mmget(work->mm);
0190     kvm_get_kvm(work->vcpu->kvm);
0191
0192     INIT_WORK(&work->work, async_pf_execute);
0193
0194     list_add_tail(&work->queue, &vcpu->async_pf.queue);
0195     vcpu->async_pf.queued++;
0196     work->notpresent_injected = kvm_arch_async_page_not_present(vcpu, work);
0197
0198     schedule_work(&work->work);
0199
0200     return true;
0201 }
0202
0203 int kvm_async_pf_wakeup_all(struct kvm_vcpu *vcpu)
0204 {
0205     struct kvm_async_pf *work;
0206     bool first;
0207
0208     if (!list_empty_careful(&vcpu->async_pf.done))
0209         return 0;
0210
0211     work = kmem_cache_zalloc(async_pf_cache, GFP_ATOMIC);
0212     if (!work)
0213         return -ENOMEM;
0214
0215     work->wakeup_all = true;
0216     INIT_LIST_HEAD(&work->queue); /* for list_del to work */
0217
0218     spin_lock(&vcpu->async_pf.lock);
0219     first = list_empty(&vcpu->async_pf.done);
0220     list_add_tail(&work->link, &vcpu->async_pf.done);
0221     spin_unlock(&vcpu->async_pf.lock);
0222
0223     if (!IS_ENABLED(CONFIG_KVM_ASYNC_PF_SYNC) && first)
0224         kvm_arch_async_page_present_queued(vcpu);
0225
0226     vcpu->async_pf.queued++;
0227     return 0;
0228 }