OSCL-LXR linux-6.0.1/​arch/​powerpc/​kvm/​book3s_hv_uvmem.c	Source navigation Diff markup Identifier search General search
	Version: linux-6.0.1 spark-3.0.0 hadoop-3.2.0-src hadoop-3.3.0-src spark-2.4.7 linux-4.15.13 hadoop-2.7.4-src Revert   Change

 // SPDX-License-Identifier: GPL-2.0
 /*
  * Secure pages management: Migration of pages between normal and secure
  * memory of KVM guests.
  *
  * Copyright 2018 Bharata B Rao, IBM Corp. <bharata@linux.ibm.com>
  */
 
 /*
  * A pseries guest can be run as secure guest on Ultravisor-enabled
  * POWER platforms. On such platforms, this driver will be used to manage
  * the movement of guest pages between the normal memory managed by
  * hypervisor (HV) and secure memory managed by Ultravisor (UV).
  *
  * The page-in or page-out requests from UV will come to HV as hcalls and
  * HV will call back into UV via ultracalls to satisfy these page requests.
  *
  * Private ZONE_DEVICE memory equal to the amount of secure memory
  * available in the platform for running secure guests is hotplugged.
  * Whenever a page belonging to the guest becomes secure, a page from this
  * private device memory is used to represent and track that secure page
  * on the HV side. Some pages (like virtio buffers, VPA pages etc) are
  * shared between UV and HV. However such pages aren't represented by
  * device private memory and mappings to shared memory exist in both
  * UV and HV page tables.
  */
 
 /*
  * Notes on locking
  *
  * kvm->arch.uvmem_lock is a per-guest lock that prevents concurrent
  * page-in and page-out requests for the same GPA. Concurrent accesses
  * can either come via UV (guest vCPUs requesting for same page)
  * or when HV and guest simultaneously access the same page.
  * This mutex serializes the migration of page from HV(normal) to
  * UV(secure) and vice versa. So the serialization points are around
  * migrate_vma routines and page-in/out routines.
  *
  * Per-guest mutex comes with a cost though. Mainly it serializes the
  * fault path as page-out can occur when HV faults on accessing secure
  * guest pages. Currently UV issues page-in requests for all the guest
  * PFNs one at a time during early boot (UV_ESM uvcall), so this is
  * not a cause for concern. Also currently the number of page-outs caused
  * by HV touching secure pages is very very low. If an when UV supports
  * overcommitting, then we might see concurrent guest driven page-outs.
  *
  * Locking order
  *
  * 1. kvm->srcu - Protects KVM memslots
  * 2. kvm->mm->mmap_lock - find_vma, migrate_vma_pages and helpers, ksm_madvise
  * 3. kvm->arch.uvmem_lock - protects read/writes to uvmem slots thus acting
  *               as sync-points for page-in/out
  */
 
 /*
  * Notes on page size
  *
  * Currently UV uses 2MB mappings internally, but will issue H_SVM_PAGE_IN
  * and H_SVM_PAGE_OUT hcalls in PAGE_SIZE(64K) granularity. HV tracks
  * secure GPAs at 64K page size and maintains one device PFN for each
  * 64K secure GPA. UV_PAGE_IN and UV_PAGE_OUT calls by HV are also issued
  * for 64K page at a time.
  *
  * HV faulting on secure pages: When HV touches any secure page, it
  * faults and issues a UV_PAGE_OUT request with 64K page size. Currently
  * UV splits and remaps the 2MB page if necessary and copies out the
  * required 64K page contents.
  *
  * Shared pages: Whenever guest shares a secure page, UV will split and
  * remap the 2MB page if required and issue H_SVM_PAGE_IN with 64K page size.
  *
  * HV invalidating a page: When a regular page belonging to secure
  * guest gets unmapped, HV informs UV with UV_PAGE_INVAL of 64K
  * page size. Using 64K page size is correct here because any non-secure
  * page will essentially be of 64K page size. Splitting by UV during sharing
  * and page-out ensures this.
  *
  * Page fault handling: When HV handles page fault of a page belonging
  * to secure guest, it sends that to UV with a 64K UV_PAGE_IN request.
  * Using 64K size is correct here too as UV would have split the 2MB page
  * into 64k mappings and would have done page-outs earlier.
  *
  * In summary, the current secure pages handling code in HV assumes
  * 64K page size and in fact fails any page-in/page-out requests of
  * non-64K size upfront. If and when UV starts supporting multiple
  * page-sizes, we need to break this assumption.
  */
 
 #include <linux/pagemap.h>
 #include <linux/migrate.h>
 #include <linux/kvm_host.h>
 #include <linux/ksm.h>
 #include <linux/of.h>
 #include <linux/memremap.h>
 #include <asm/ultravisor.h>
 #include <asm/mman.h>
 #include <asm/kvm_ppc.h>
 #include <asm/kvm_book3s_uvmem.h>
 
 static struct dev_pagemap kvmppc_uvmem_pgmap;
 static unsigned long *kvmppc_uvmem_bitmap;
 static DEFINE_SPINLOCK(kvmppc_uvmem_bitmap_lock);
 
 /*
  * States of a GFN
  * ---------------
  * The GFN can be in one of the following states.
  *
  * (a) Secure - The GFN is secure. The GFN is associated with
  *  a Secure VM, the contents of the GFN is not accessible
  *  to the Hypervisor.  This GFN can be backed by a secure-PFN,
  *  or can be backed by a normal-PFN with contents encrypted.
  *  The former is true when the GFN is paged-in into the
  *  ultravisor. The latter is true when the GFN is paged-out
  *  of the ultravisor.
  *
  * (b) Shared - The GFN is shared. The GFN is associated with a
  *  a secure VM. The contents of the GFN is accessible to
  *  Hypervisor. This GFN is backed by a normal-PFN and its
  *  content is un-encrypted.
  *
  * (c) Normal - The GFN is a normal. The GFN is associated with
  *  a normal VM. The contents of the GFN is accessible to
  *  the Hypervisor. Its content is never encrypted.
  *
  * States of a VM.
  * ---------------
  *
  * Normal VM:  A VM whose contents are always accessible to
  *  the hypervisor.  All its GFNs are normal-GFNs.
  *
  * Secure VM: A VM whose contents are not accessible to the
  *  hypervisor without the VM's consent.  Its GFNs are
  *  either Shared-GFN or Secure-GFNs.
  *
  * Transient VM: A Normal VM that is transitioning to secure VM.
  *  The transition starts on successful return of
  *  H_SVM_INIT_START, and ends on successful return
  *  of H_SVM_INIT_DONE. This transient VM, can have GFNs
  *  in any of the three states; i.e Secure-GFN, Shared-GFN,
  *  and Normal-GFN. The VM never executes in this state
  *  in supervisor-mode.
  *
  * Memory slot State.
  * -----------------------------
  *  The state of a memory slot mirrors the state of the
  *  VM the memory slot is associated with.
  *
  * VM State transition.
  * --------------------
  *
  *  A VM always starts in Normal Mode.
  *
  *  H_SVM_INIT_START moves the VM into transient state. During this
  *  time the Ultravisor may request some of its GFNs to be shared or
  *  secured. So its GFNs can be in one of the three GFN states.
  *
  *  H_SVM_INIT_DONE moves the VM entirely from transient state to
  *  secure-state. At this point any left-over normal-GFNs are
  *  transitioned to Secure-GFN.
  *
  *  H_SVM_INIT_ABORT moves the transient VM back to normal VM.
  *  All its GFNs are moved to Normal-GFNs.
  *
  *  UV_TERMINATE transitions the secure-VM back to normal-VM. All
  *  the secure-GFN and shared-GFNs are tranistioned to normal-GFN
  *  Note: The contents of the normal-GFN is undefined at this point.
  *
  * GFN state implementation:
  * -------------------------
  *
  * Secure GFN is associated with a secure-PFN; also called uvmem_pfn,
  * when the GFN is paged-in. Its pfn[] has KVMPPC_GFN_UVMEM_PFN flag
  * set, and contains the value of the secure-PFN.
  * It is associated with a normal-PFN; also called mem_pfn, when
  * the GFN is pagedout. Its pfn[] has KVMPPC_GFN_MEM_PFN flag set.
  * The value of the normal-PFN is not tracked.
  *
  * Shared GFN is associated with a normal-PFN. Its pfn[] has
  * KVMPPC_UVMEM_SHARED_PFN flag set. The value of the normal-PFN
  * is not tracked.
  *
  * Normal GFN is associated with normal-PFN. Its pfn[] has
  * no flag set. The value of the normal-PFN is not tracked.
  *
  * Life cycle of a GFN
  * --------------------
  *
  * --------------------------------------------------------------
  * |        |     Share  |  Unshare | SVM       |H_SVM_INIT_DONE|
  * |        |operation   |operation | abort/    |               |
  * |        |            |          | terminate |               |
  * -------------------------------------------------------------
  * |        |            |          |           |               |
  * | Secure |     Shared | Secure   |Normal     |Secure         |
  * |        |            |          |           |               |
  * | Shared |     Shared | Secure   |Normal     |Shared         |
  * |        |            |          |           |               |
  * | Normal |     Shared | Secure   |Normal     |Secure         |
  * --------------------------------------------------------------
  *
  * Life cycle of a VM
  * --------------------
  *
  * --------------------------------------------------------------------
  * |         |  start    |  H_SVM_  |H_SVM_   |H_SVM_     |UV_SVM_    |
  * |         |  VM       |INIT_START|INIT_DONE|INIT_ABORT |TERMINATE  |
  * |         |           |          |         |           |           |
  * --------- ----------------------------------------------------------
  * |         |           |          |         |           |           |
  * | Normal  | Normal    | Transient|Error    |Error      |Normal     |
  * |         |           |          |         |           |           |
  * | Secure  |   Error   | Error    |Error    |Error      |Normal     |
  * |         |           |          |         |           |           |
  * |Transient|   N/A     | Error    |Secure   |Normal     |Normal     |
  * --------------------------------------------------------------------
  */
 
 #define KVMPPC_GFN_UVMEM_PFN    (1UL << 63)
 #define KVMPPC_GFN_MEM_PFN  (1UL << 62)
 #define KVMPPC_GFN_SHARED   (1UL << 61)
 #define KVMPPC_GFN_SECURE   (KVMPPC_GFN_UVMEM_PFN | KVMPPC_GFN_MEM_PFN)
 #define KVMPPC_GFN_FLAG_MASK    (KVMPPC_GFN_SECURE | KVMPPC_GFN_SHARED)
 #define KVMPPC_GFN_PFN_MASK (~KVMPPC_GFN_FLAG_MASK)
 
 struct kvmppc_uvmem_slot {
     struct list_head list;
     unsigned long nr_pfns;
     unsigned long base_pfn;
     unsigned long *pfns;
 };
 struct kvmppc_uvmem_page_pvt {
     struct kvm *kvm;
     unsigned long gpa;
     bool skip_page_out;
     bool remove_gfn;
 };
 
 bool kvmppc_uvmem_available(void)
 {
     /*
      * If kvmppc_uvmem_bitmap != NULL, then there is an ultravisor
      * and our data structures have been initialized successfully.
      */
     return !!kvmppc_uvmem_bitmap;
 }
 
 int kvmppc_uvmem_slot_init(struct kvm *kvm, const struct kvm_memory_slot *slot)
 {
     struct kvmppc_uvmem_slot *p;
 
     p = kzalloc(sizeof(*p), GFP_KERNEL);
     if (!p)
         return -ENOMEM;
     p->pfns = vcalloc(slot->npages, sizeof(*p->pfns));
     if (!p->pfns) {
         kfree(p);
         return -ENOMEM;
     }
     p->nr_pfns = slot->npages;
     p->base_pfn = slot->base_gfn;
 
     mutex_lock(&kvm->arch.uvmem_lock);
     list_add(&p->list, &kvm->arch.uvmem_pfns);
     mutex_unlock(&kvm->arch.uvmem_lock);
 
     return 0;
 }
 
 /*
  * All device PFNs are already released by the time we come here.
  */
 void kvmppc_uvmem_slot_free(struct kvm *kvm, const struct kvm_memory_slot *slot)
 {
     struct kvmppc_uvmem_slot *p, *next;
 
     mutex_lock(&kvm->arch.uvmem_lock);
     list_for_each_entry_safe(p, next, &kvm->arch.uvmem_pfns, list) {
         if (p->base_pfn == slot->base_gfn) {
             vfree(p->pfns);
             list_del(&p->list);
             kfree(p);
             break;
         }
     }
     mutex_unlock(&kvm->arch.uvmem_lock);
 }
 
 static void kvmppc_mark_gfn(unsigned long gfn, struct kvm *kvm,
             unsigned long flag, unsigned long uvmem_pfn)
 {
     struct kvmppc_uvmem_slot *p;
 
     list_for_each_entry(p, &kvm->arch.uvmem_pfns, list) {
         if (gfn >= p->base_pfn && gfn < p->base_pfn + p->nr_pfns) {
             unsigned long index = gfn - p->base_pfn;
 
             if (flag == KVMPPC_GFN_UVMEM_PFN)
                 p->pfns[index] = uvmem_pfn | flag;
             else
                 p->pfns[index] = flag;
             return;
         }
     }
 }
 
 /* mark the GFN as secure-GFN associated with @uvmem pfn device-PFN. */
 static void kvmppc_gfn_secure_uvmem_pfn(unsigned long gfn,
             unsigned long uvmem_pfn, struct kvm *kvm)
 {
     kvmppc_mark_gfn(gfn, kvm, KVMPPC_GFN_UVMEM_PFN, uvmem_pfn);
 }
 
 /* mark the GFN as secure-GFN associated with a memory-PFN. */
 static void kvmppc_gfn_secure_mem_pfn(unsigned long gfn, struct kvm *kvm)
 {
     kvmppc_mark_gfn(gfn, kvm, KVMPPC_GFN_MEM_PFN, 0);
 }
 
 /* mark the GFN as a shared GFN. */
 static void kvmppc_gfn_shared(unsigned long gfn, struct kvm *kvm)
 {
     kvmppc_mark_gfn(gfn, kvm, KVMPPC_GFN_SHARED, 0);
 }
 
 /* mark the GFN as a non-existent GFN. */
 static void kvmppc_gfn_remove(unsigned long gfn, struct kvm *kvm)
 {
     kvmppc_mark_gfn(gfn, kvm, 0, 0);
 }
 
 /* return true, if the GFN is a secure-GFN backed by a secure-PFN */
 static bool kvmppc_gfn_is_uvmem_pfn(unsigned long gfn, struct kvm *kvm,
                     unsigned long *uvmem_pfn)
 {
     struct kvmppc_uvmem_slot *p;
 
     list_for_each_entry(p, &kvm->arch.uvmem_pfns, list) {
         if (gfn >= p->base_pfn && gfn < p->base_pfn + p->nr_pfns) {
             unsigned long index = gfn - p->base_pfn;
 
             if (p->pfns[index] & KVMPPC_GFN_UVMEM_PFN) {
                 if (uvmem_pfn)
                     *uvmem_pfn = p->pfns[index] &
                              KVMPPC_GFN_PFN_MASK;
                 return true;
             } else
                 return false;
         }
     }
     return false;
 }
 
 /*
  * starting from *gfn search for the next available GFN that is not yet
  * transitioned to a secure GFN.  return the value of that GFN in *gfn.  If a
  * GFN is found, return true, else return false
  *
  * Must be called with kvm->arch.uvmem_lock  held.
  */
 static bool kvmppc_next_nontransitioned_gfn(const struct kvm_memory_slot *memslot,
         struct kvm *kvm, unsigned long *gfn)
 {
     struct kvmppc_uvmem_slot *p = NULL, *iter;
     bool ret = false;
     unsigned long i;
 
     list_for_each_entry(iter, &kvm->arch.uvmem_pfns, list)
         if (*gfn >= iter->base_pfn && *gfn < iter->base_pfn + iter->nr_pfns) {
             p = iter;
             break;
         }
     if (!p)
         return ret;
     /*
      * The code below assumes, one to one correspondence between
      * kvmppc_uvmem_slot and memslot.
      */
     for (i = *gfn; i < p->base_pfn + p->nr_pfns; i++) {
         unsigned long index = i - p->base_pfn;
 
         if (!(p->pfns[index] & KVMPPC_GFN_FLAG_MASK)) {
             *gfn = i;
             ret = true;
             break;
         }
     }
     return ret;
 }
 
 static int kvmppc_memslot_page_merge(struct kvm *kvm,
         const struct kvm_memory_slot *memslot, bool merge)
 {
     unsigned long gfn = memslot->base_gfn;
     unsigned long end, start = gfn_to_hva(kvm, gfn);
     int ret = 0;
     struct vm_area_struct *vma;
     int merge_flag = (merge) ? MADV_MERGEABLE : MADV_UNMERGEABLE;
 
     if (kvm_is_error_hva(start))
         return H_STATE;
 
     end = start + (memslot->npages << PAGE_SHIFT);
 
     mmap_write_lock(kvm->mm);
     do {
         vma = find_vma_intersection(kvm->mm, start, end);
         if (!vma) {
             ret = H_STATE;
             break;
         }
         ret = ksm_madvise(vma, vma->vm_start, vma->vm_end,
               merge_flag, &vma->vm_flags);
         if (ret) {
             ret = H_STATE;
             break;
         }
         start = vma->vm_end;
     } while (end > vma->vm_end);
 
     mmap_write_unlock(kvm->mm);
     return ret;
 }
 
 static void __kvmppc_uvmem_memslot_delete(struct kvm *kvm,
         const struct kvm_memory_slot *memslot)
 {
     uv_unregister_mem_slot(kvm->arch.lpid, memslot->id);
     kvmppc_uvmem_slot_free(kvm, memslot);
     kvmppc_memslot_page_merge(kvm, memslot, true);
 }
 
 static int __kvmppc_uvmem_memslot_create(struct kvm *kvm,
         const struct kvm_memory_slot *memslot)
 {
     int ret = H_PARAMETER;
 
     if (kvmppc_memslot_page_merge(kvm, memslot, false))
         return ret;
 
     if (kvmppc_uvmem_slot_init(kvm, memslot))
         goto out1;
 
     ret = uv_register_mem_slot(kvm->arch.lpid,
                    memslot->base_gfn << PAGE_SHIFT,
                    memslot->npages * PAGE_SIZE,
                    0, memslot->id);
     if (ret < 0) {
         ret = H_PARAMETER;
         goto out;
     }
     return 0;
 out:
     kvmppc_uvmem_slot_free(kvm, memslot);
 out1:
     kvmppc_memslot_page_merge(kvm, memslot, true);
     return ret;
 }
 
 unsigned long kvmppc_h_svm_init_start(struct kvm *kvm)
 {
     struct kvm_memslots *slots;
     struct kvm_memory_slot *memslot, *m;
     int ret = H_SUCCESS;
     int srcu_idx, bkt;
 
     kvm->arch.secure_guest = KVMPPC_SECURE_INIT_START;
 
     if (!kvmppc_uvmem_bitmap)
         return H_UNSUPPORTED;
 
     /* Only radix guests can be secure guests */
     if (!kvm_is_radix(kvm))
         return H_UNSUPPORTED;
 
     /* NAK the transition to secure if not enabled */
     if (!kvm->arch.svm_enabled)
         return H_AUTHORITY;
 
     srcu_idx = srcu_read_lock(&kvm->srcu);
 
     /* register the memslot */
     slots = kvm_memslots(kvm);
     kvm_for_each_memslot(memslot, bkt, slots) {
         ret = __kvmppc_uvmem_memslot_create(kvm, memslot);
         if (ret)
             break;
     }
 
     if (ret) {
         slots = kvm_memslots(kvm);
         kvm_for_each_memslot(m, bkt, slots) {
             if (m == memslot)
                 break;
             __kvmppc_uvmem_memslot_delete(kvm, memslot);
         }
     }
 
     srcu_read_unlock(&kvm->srcu, srcu_idx);
     return ret;
 }
 
 /*
  * Provision a new page on HV side and copy over the contents
  * from secure memory using UV_PAGE_OUT uvcall.
  * Caller must held kvm->arch.uvmem_lock.
  */
 static int __kvmppc_svm_page_out(struct vm_area_struct *vma,
         unsigned long start,
         unsigned long end, unsigned long page_shift,
         struct kvm *kvm, unsigned long gpa)
 {
     unsigned long src_pfn, dst_pfn = 0;
     struct migrate_vma mig;
     struct page *dpage, *spage;
     struct kvmppc_uvmem_page_pvt *pvt;
     unsigned long pfn;
     int ret = U_SUCCESS;
 
     memset(&mig, 0, sizeof(mig));
     mig.vma = vma;
     mig.start = start;
     mig.end = end;
     mig.src = &src_pfn;
     mig.dst = &dst_pfn;
     mig.pgmap_owner = &kvmppc_uvmem_pgmap;
     mig.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
 
     /* The requested page is already paged-out, nothing to do */
     if (!kvmppc_gfn_is_uvmem_pfn(gpa >> page_shift, kvm, NULL))
         return ret;
 
     ret = migrate_vma_setup(&mig);
     if (ret)
         return -1;
 
     spage = migrate_pfn_to_page(*mig.src);
     if (!spage || !(*mig.src & MIGRATE_PFN_MIGRATE))
         goto out_finalize;
 
     if (!is_zone_device_page(spage))
         goto out_finalize;
 
     dpage = alloc_page_vma(GFP_HIGHUSER, vma, start);
     if (!dpage) {
         ret = -1;
         goto out_finalize;
     }
 
     lock_page(dpage);
     pvt = spage->zone_device_data;
     pfn = page_to_pfn(dpage);
 
     /*
      * This function is used in two cases:
      * - When HV touches a secure page, for which we do UV_PAGE_OUT
      * - When a secure page is converted to shared page, we *get*
      *   the page to essentially unmap the device page. In this
      *   case we skip page-out.
      */
     if (!pvt->skip_page_out)
         ret = uv_page_out(kvm->arch.lpid, pfn << page_shift,
                   gpa, 0, page_shift);
 
     if (ret == U_SUCCESS)
         *mig.dst = migrate_pfn(pfn);
     else {
         unlock_page(dpage);
         __free_page(dpage);
         goto out_finalize;
     }
 
     migrate_vma_pages(&mig);
 
 out_finalize:
     migrate_vma_finalize(&mig);
     return ret;
 }
 
 static inline int kvmppc_svm_page_out(struct vm_area_struct *vma,
                       unsigned long start, unsigned long end,
                       unsigned long page_shift,
                       struct kvm *kvm, unsigned long gpa)
 {
     int ret;
 
     mutex_lock(&kvm->arch.uvmem_lock);
     ret = __kvmppc_svm_page_out(vma, start, end, page_shift, kvm, gpa);
     mutex_unlock(&kvm->arch.uvmem_lock);
 
     return ret;
 }
 
 /*
  * Drop device pages that we maintain for the secure guest
  *
  * We first mark the pages to be skipped from UV_PAGE_OUT when there
  * is HV side fault on these pages. Next we *get* these pages, forcing
  * fault on them, do fault time migration to replace the device PTEs in
  * QEMU page table with normal PTEs from newly allocated pages.
  */
 void kvmppc_uvmem_drop_pages(const struct kvm_memory_slot *slot,
                  struct kvm *kvm, bool skip_page_out)
 {
     int i;
     struct kvmppc_uvmem_page_pvt *pvt;
     struct page *uvmem_page;
     struct vm_area_struct *vma = NULL;
     unsigned long uvmem_pfn, gfn;
     unsigned long addr;
 
     mmap_read_lock(kvm->mm);
 
     addr = slot->userspace_addr;
 
     gfn = slot->base_gfn;
     for (i = slot->npages; i; --i, ++gfn, addr += PAGE_SIZE) {
 
         /* Fetch the VMA if addr is not in the latest fetched one */
         if (!vma || addr >= vma->vm_end) {
             vma = vma_lookup(kvm->mm, addr);
             if (!vma) {
                 pr_err("Can't find VMA for gfn:0x%lx\n", gfn);
                 break;
             }
         }
 
         mutex_lock(&kvm->arch.uvmem_lock);
 
         if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, &uvmem_pfn)) {
             uvmem_page = pfn_to_page(uvmem_pfn);
             pvt = uvmem_page->zone_device_data;
             pvt->skip_page_out = skip_page_out;
             pvt->remove_gfn = true;
 
             if (__kvmppc_svm_page_out(vma, addr, addr + PAGE_SIZE,
                           PAGE_SHIFT, kvm, pvt->gpa))
                 pr_err("Can't page out gpa:0x%lx addr:0x%lx\n",
                        pvt->gpa, addr);
         } else {
             /* Remove the shared flag if any */
             kvmppc_gfn_remove(gfn, kvm);
         }
 
         mutex_unlock(&kvm->arch.uvmem_lock);
     }
 
     mmap_read_unlock(kvm->mm);
 }
 
 unsigned long kvmppc_h_svm_init_abort(struct kvm *kvm)
 {
     int srcu_idx, bkt;
     struct kvm_memory_slot *memslot;
 
     /*
      * Expect to be called only after INIT_START and before INIT_DONE.
      * If INIT_DONE was completed, use normal VM termination sequence.
      */
     if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START))
         return H_UNSUPPORTED;
 
     if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_DONE)
         return H_STATE;
 
     srcu_idx = srcu_read_lock(&kvm->srcu);
 
     kvm_for_each_memslot(memslot, bkt, kvm_memslots(kvm))
         kvmppc_uvmem_drop_pages(memslot, kvm, false);
 
     srcu_read_unlock(&kvm->srcu, srcu_idx);
 
     kvm->arch.secure_guest = 0;
     uv_svm_terminate(kvm->arch.lpid);
 
     return H_PARAMETER;
 }
 
 /*
  * Get a free device PFN from the pool
  *
  * Called when a normal page is moved to secure memory (UV_PAGE_IN). Device
  * PFN will be used to keep track of the secure page on HV side.
  *
  * Called with kvm->arch.uvmem_lock held
  */
 static struct page *kvmppc_uvmem_get_page(unsigned long gpa, struct kvm *kvm)
 {
     struct page *dpage = NULL;
     unsigned long bit, uvmem_pfn;
     struct kvmppc_uvmem_page_pvt *pvt;
     unsigned long pfn_last, pfn_first;
 
     pfn_first = kvmppc_uvmem_pgmap.range.start >> PAGE_SHIFT;
     pfn_last = pfn_first +
            (range_len(&kvmppc_uvmem_pgmap.range) >> PAGE_SHIFT);
 
     spin_lock(&kvmppc_uvmem_bitmap_lock);
     bit = find_first_zero_bit(kvmppc_uvmem_bitmap,
                   pfn_last - pfn_first);
     if (bit >= (pfn_last - pfn_first))
         goto out;
     bitmap_set(kvmppc_uvmem_bitmap, bit, 1);
     spin_unlock(&kvmppc_uvmem_bitmap_lock);
 
     pvt = kzalloc(sizeof(*pvt), GFP_KERNEL);
     if (!pvt)
         goto out_clear;
 
     uvmem_pfn = bit + pfn_first;
     kvmppc_gfn_secure_uvmem_pfn(gpa >> PAGE_SHIFT, uvmem_pfn, kvm);
 
     pvt->gpa = gpa;
     pvt->kvm = kvm;
 
     dpage = pfn_to_page(uvmem_pfn);
     dpage->zone_device_data = pvt;
     lock_page(dpage);
     return dpage;
 out_clear:
     spin_lock(&kvmppc_uvmem_bitmap_lock);
     bitmap_clear(kvmppc_uvmem_bitmap, bit, 1);
 out:
     spin_unlock(&kvmppc_uvmem_bitmap_lock);
     return NULL;
 }
 
 /*
  * Alloc a PFN from private device memory pool. If @pagein is true,
  * copy page from normal memory to secure memory using UV_PAGE_IN uvcall.
  */
 static int kvmppc_svm_page_in(struct vm_area_struct *vma,
         unsigned long start,
         unsigned long end, unsigned long gpa, struct kvm *kvm,
         unsigned long page_shift,
         bool pagein)
 {
     unsigned long src_pfn, dst_pfn = 0;
     struct migrate_vma mig;
     struct page *spage;
     unsigned long pfn;
     struct page *dpage;
     int ret = 0;
 
     memset(&mig, 0, sizeof(mig));
     mig.vma = vma;
     mig.start = start;
     mig.end = end;
     mig.src = &src_pfn;
     mig.dst = &dst_pfn;
     mig.flags = MIGRATE_VMA_SELECT_SYSTEM;
 
     ret = migrate_vma_setup(&mig);
     if (ret)
         return ret;
 
     if (!(*mig.src & MIGRATE_PFN_MIGRATE)) {
         ret = -1;
         goto out_finalize;
     }
 
     dpage = kvmppc_uvmem_get_page(gpa, kvm);
     if (!dpage) {
         ret = -1;
         goto out_finalize;
     }
 
     if (pagein) {
         pfn = *mig.src >> MIGRATE_PFN_SHIFT;
         spage = migrate_pfn_to_page(*mig.src);
         if (spage) {
             ret = uv_page_in(kvm->arch.lpid, pfn << page_shift,
                     gpa, 0, page_shift);
             if (ret)
                 goto out_finalize;
         }
     }
 
     *mig.dst = migrate_pfn(page_to_pfn(dpage));
     migrate_vma_pages(&mig);
 out_finalize:
     migrate_vma_finalize(&mig);
     return ret;
 }
 
 static int kvmppc_uv_migrate_mem_slot(struct kvm *kvm,
         const struct kvm_memory_slot *memslot)
 {
     unsigned long gfn = memslot->base_gfn;
     struct vm_area_struct *vma;
     unsigned long start, end;
     int ret = 0;
 
     mmap_read_lock(kvm->mm);
     mutex_lock(&kvm->arch.uvmem_lock);
     while (kvmppc_next_nontransitioned_gfn(memslot, kvm, &gfn)) {
         ret = H_STATE;
         start = gfn_to_hva(kvm, gfn);
         if (kvm_is_error_hva(start))
             break;
 
         end = start + (1UL << PAGE_SHIFT);
         vma = find_vma_intersection(kvm->mm, start, end);
         if (!vma || vma->vm_start > start || vma->vm_end < end)
             break;
 
         ret = kvmppc_svm_page_in(vma, start, end,
                 (gfn << PAGE_SHIFT), kvm, PAGE_SHIFT, false);
         if (ret) {
             ret = H_STATE;
             break;
         }
 
         /* relinquish the cpu if needed */
         cond_resched();
     }
     mutex_unlock(&kvm->arch.uvmem_lock);
     mmap_read_unlock(kvm->mm);
     return ret;
 }
 
 unsigned long kvmppc_h_svm_init_done(struct kvm *kvm)
 {
     struct kvm_memslots *slots;
     struct kvm_memory_slot *memslot;
     int srcu_idx, bkt;
     long ret = H_SUCCESS;
 
     if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START))
         return H_UNSUPPORTED;
 
     /* migrate any unmoved normal pfn to device pfns*/
     srcu_idx = srcu_read_lock(&kvm->srcu);
     slots = kvm_memslots(kvm);
     kvm_for_each_memslot(memslot, bkt, slots) {
         ret = kvmppc_uv_migrate_mem_slot(kvm, memslot);
         if (ret) {
             /*
              * The pages will remain transitioned.
              * Its the callers responsibility to
              * terminate the VM, which will undo
              * all state of the VM. Till then
              * this VM is in a erroneous state.
              * Its KVMPPC_SECURE_INIT_DONE will
              * remain unset.
              */
             ret = H_STATE;
             goto out;
         }
     }
 
     kvm->arch.secure_guest |= KVMPPC_SECURE_INIT_DONE;
     pr_info("LPID %d went secure\n", kvm->arch.lpid);
 
 out:
     srcu_read_unlock(&kvm->srcu, srcu_idx);
     return ret;
 }
 
 /*
  * Shares the page with HV, thus making it a normal page.
  *
  * - If the page is already secure, then provision a new page and share
  * - If the page is a normal page, share the existing page
  *
  * In the former case, uses dev_pagemap_ops.migrate_to_ram handler
  * to unmap the device page from QEMU's page tables.
  */
 static unsigned long kvmppc_share_page(struct kvm *kvm, unsigned long gpa,
         unsigned long page_shift)
 {
 
     int ret = H_PARAMETER;
     struct page *uvmem_page;
     struct kvmppc_uvmem_page_pvt *pvt;
     unsigned long pfn;
     unsigned long gfn = gpa >> page_shift;
     int srcu_idx;
     unsigned long uvmem_pfn;
 
     srcu_idx = srcu_read_lock(&kvm->srcu);
     mutex_lock(&kvm->arch.uvmem_lock);
     if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, &uvmem_pfn)) {
         uvmem_page = pfn_to_page(uvmem_pfn);
         pvt = uvmem_page->zone_device_data;
         pvt->skip_page_out = true;
         /*
          * do not drop the GFN. It is a valid GFN
          * that is transitioned to a shared GFN.
          */
         pvt->remove_gfn = false;
     }
 
 retry:
     mutex_unlock(&kvm->arch.uvmem_lock);
     pfn = gfn_to_pfn(kvm, gfn);
     if (is_error_noslot_pfn(pfn))
         goto out;
 
     mutex_lock(&kvm->arch.uvmem_lock);
     if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, &uvmem_pfn)) {
         uvmem_page = pfn_to_page(uvmem_pfn);
         pvt = uvmem_page->zone_device_data;
         pvt->skip_page_out = true;
         pvt->remove_gfn = false; /* it continues to be a valid GFN */
         kvm_release_pfn_clean(pfn);
         goto retry;
     }
 
     if (!uv_page_in(kvm->arch.lpid, pfn << page_shift, gpa, 0,
                 page_shift)) {
         kvmppc_gfn_shared(gfn, kvm);
         ret = H_SUCCESS;
     }
     kvm_release_pfn_clean(pfn);
     mutex_unlock(&kvm->arch.uvmem_lock);
 out:
     srcu_read_unlock(&kvm->srcu, srcu_idx);
     return ret;
 }
 
 /*
  * H_SVM_PAGE_IN: Move page from normal memory to secure memory.
  *
  * H_PAGE_IN_SHARED flag makes the page shared which means that the same
  * memory in is visible from both UV and HV.
  */
 unsigned long kvmppc_h_svm_page_in(struct kvm *kvm, unsigned long gpa,
         unsigned long flags,
         unsigned long page_shift)
 {
     unsigned long start, end;
     struct vm_area_struct *vma;
     int srcu_idx;
     unsigned long gfn = gpa >> page_shift;
     int ret;
 
     if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START))
         return H_UNSUPPORTED;
 
     if (page_shift != PAGE_SHIFT)
         return H_P3;
 
     if (flags & ~H_PAGE_IN_SHARED)
         return H_P2;
 
     if (flags & H_PAGE_IN_SHARED)
         return kvmppc_share_page(kvm, gpa, page_shift);
 
     ret = H_PARAMETER;
     srcu_idx = srcu_read_lock(&kvm->srcu);
     mmap_read_lock(kvm->mm);
 
     start = gfn_to_hva(kvm, gfn);
     if (kvm_is_error_hva(start))
         goto out;
 
     mutex_lock(&kvm->arch.uvmem_lock);
     /* Fail the page-in request of an already paged-in page */
     if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, NULL))
         goto out_unlock;
 
     end = start + (1UL << page_shift);
     vma = find_vma_intersection(kvm->mm, start, end);
     if (!vma || vma->vm_start > start || vma->vm_end < end)
         goto out_unlock;
 
     if (kvmppc_svm_page_in(vma, start, end, gpa, kvm, page_shift,
                 true))
         goto out_unlock;
 
     ret = H_SUCCESS;
 
 out_unlock:
     mutex_unlock(&kvm->arch.uvmem_lock);
 out:
     mmap_read_unlock(kvm->mm);
     srcu_read_unlock(&kvm->srcu, srcu_idx);
     return ret;
 }
 
 
 /*
  * Fault handler callback that gets called when HV touches any page that
  * has been moved to secure memory, we ask UV to give back the page by
  * issuing UV_PAGE_OUT uvcall.
  *
  * This eventually results in dropping of device PFN and the newly
  * provisioned page/PFN gets populated in QEMU page tables.
  */
 static vm_fault_t kvmppc_uvmem_migrate_to_ram(struct vm_fault *vmf)
 {
     struct kvmppc_uvmem_page_pvt *pvt = vmf->page->zone_device_data;
 
     if (kvmppc_svm_page_out(vmf->vma, vmf->address,
                 vmf->address + PAGE_SIZE, PAGE_SHIFT,
                 pvt->kvm, pvt->gpa))
         return VM_FAULT_SIGBUS;
     else
         return 0;
 }
 
 /*
  * Release the device PFN back to the pool
  *
  * Gets called when secure GFN tranistions from a secure-PFN
  * to a normal PFN during H_SVM_PAGE_OUT.
  * Gets called with kvm->arch.uvmem_lock held.
  */
 static void kvmppc_uvmem_page_free(struct page *page)
 {
     unsigned long pfn = page_to_pfn(page) -
             (kvmppc_uvmem_pgmap.range.start >> PAGE_SHIFT);
     struct kvmppc_uvmem_page_pvt *pvt;
 
     spin_lock(&kvmppc_uvmem_bitmap_lock);
     bitmap_clear(kvmppc_uvmem_bitmap, pfn, 1);
     spin_unlock(&kvmppc_uvmem_bitmap_lock);
 
     pvt = page->zone_device_data;
     page->zone_device_data = NULL;
     if (pvt->remove_gfn)
         kvmppc_gfn_remove(pvt->gpa >> PAGE_SHIFT, pvt->kvm);
     else
         kvmppc_gfn_secure_mem_pfn(pvt->gpa >> PAGE_SHIFT, pvt->kvm);
     kfree(pvt);
 }
 
 static const struct dev_pagemap_ops kvmppc_uvmem_ops = {
     .page_free = kvmppc_uvmem_page_free,
     .migrate_to_ram = kvmppc_uvmem_migrate_to_ram,
 };
 
 /*
  * H_SVM_PAGE_OUT: Move page from secure memory to normal memory.
  */
 unsigned long
 kvmppc_h_svm_page_out(struct kvm *kvm, unsigned long gpa,
               unsigned long flags, unsigned long page_shift)
 {
     unsigned long gfn = gpa >> page_shift;
     unsigned long start, end;
     struct vm_area_struct *vma;
     int srcu_idx;
     int ret;
 
     if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START))
         return H_UNSUPPORTED;
 
     if (page_shift != PAGE_SHIFT)
         return H_P3;
 
     if (flags)
         return H_P2;
 
     ret = H_PARAMETER;
     srcu_idx = srcu_read_lock(&kvm->srcu);
     mmap_read_lock(kvm->mm);
     start = gfn_to_hva(kvm, gfn);
     if (kvm_is_error_hva(start))
         goto out;
 
     end = start + (1UL << page_shift);
     vma = find_vma_intersection(kvm->mm, start, end);
     if (!vma || vma->vm_start > start || vma->vm_end < end)
         goto out;
 
     if (!kvmppc_svm_page_out(vma, start, end, page_shift, kvm, gpa))
         ret = H_SUCCESS;
 out:
     mmap_read_unlock(kvm->mm);
     srcu_read_unlock(&kvm->srcu, srcu_idx);
     return ret;
 }
 
 int kvmppc_send_page_to_uv(struct kvm *kvm, unsigned long gfn)
 {
     unsigned long pfn;
     int ret = U_SUCCESS;
 
     pfn = gfn_to_pfn(kvm, gfn);
     if (is_error_noslot_pfn(pfn))
         return -EFAULT;
 
     mutex_lock(&kvm->arch.uvmem_lock);
     if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, NULL))
         goto out;
 
     ret = uv_page_in(kvm->arch.lpid, pfn << PAGE_SHIFT, gfn << PAGE_SHIFT,
              0, PAGE_SHIFT);
 out:
     kvm_release_pfn_clean(pfn);
     mutex_unlock(&kvm->arch.uvmem_lock);
     return (ret == U_SUCCESS) ? RESUME_GUEST : -EFAULT;
 }
 
 int kvmppc_uvmem_memslot_create(struct kvm *kvm, const struct kvm_memory_slot *new)
 {
     int ret = __kvmppc_uvmem_memslot_create(kvm, new);
 
     if (!ret)
         ret = kvmppc_uv_migrate_mem_slot(kvm, new);
 
     return ret;
 }
 
 void kvmppc_uvmem_memslot_delete(struct kvm *kvm, const struct kvm_memory_slot *old)
 {
     __kvmppc_uvmem_memslot_delete(kvm, old);
 }
 
 static u64 kvmppc_get_secmem_size(void)
 {
     struct device_node *np;
     int i, len;
     const __be32 *prop;
     u64 size = 0;
 
     /*
      * First try the new ibm,secure-memory nodes which supersede the
      * secure-memory-ranges property.
      * If we found some, no need to read the deprecated ones.
      */
     for_each_compatible_node(np, NULL, "ibm,secure-memory") {
         prop = of_get_property(np, "reg", &len);
         if (!prop)
             continue;
         size += of_read_number(prop + 2, 2);
     }
     if (size)
         return size;
 
     np = of_find_compatible_node(NULL, NULL, "ibm,uv-firmware");
     if (!np)
         goto out;
 
     prop = of_get_property(np, "secure-memory-ranges", &len);
     if (!prop)
         goto out_put;
 
     for (i = 0; i < len / (sizeof(*prop) * 4); i++)
         size += of_read_number(prop + (i * 4) + 2, 2);
 
 out_put:
     of_node_put(np);
 out:
     return size;
 }
 
 int kvmppc_uvmem_init(void)
 {
     int ret = 0;
     unsigned long size;
     struct resource *res;
     void *addr;
     unsigned long pfn_last, pfn_first;
 
     size = kvmppc_get_secmem_size();
     if (!size) {
         /*
          * Don't fail the initialization of kvm-hv module if
          * the platform doesn't export ibm,uv-firmware node.
          * Let normal guests run on such PEF-disabled platform.
          */
         pr_info("KVMPPC-UVMEM: No support for secure guests\n");
         goto out;
     }
 
     res = request_free_mem_region(&iomem_resource, size, "kvmppc_uvmem");
     if (IS_ERR(res)) {
         ret = PTR_ERR(res);
         goto out;
     }
 
     kvmppc_uvmem_pgmap.type = MEMORY_DEVICE_PRIVATE;
     kvmppc_uvmem_pgmap.range.start = res->start;
     kvmppc_uvmem_pgmap.range.end = res->end;
     kvmppc_uvmem_pgmap.nr_range = 1;
     kvmppc_uvmem_pgmap.ops = &kvmppc_uvmem_ops;
     /* just one global instance: */
     kvmppc_uvmem_pgmap.owner = &kvmppc_uvmem_pgmap;
     addr = memremap_pages(&kvmppc_uvmem_pgmap, NUMA_NO_NODE);
     if (IS_ERR(addr)) {
         ret = PTR_ERR(addr);
         goto out_free_region;
     }
 
     pfn_first = res->start >> PAGE_SHIFT;
     pfn_last = pfn_first + (resource_size(res) >> PAGE_SHIFT);
     kvmppc_uvmem_bitmap = kcalloc(BITS_TO_LONGS(pfn_last - pfn_first),
                       sizeof(unsigned long), GFP_KERNEL);
     if (!kvmppc_uvmem_bitmap) {
         ret = -ENOMEM;
         goto out_unmap;
     }
 
     pr_info("KVMPPC-UVMEM: Secure Memory size 0x%lx\n", size);
     return ret;
 out_unmap:
     memunmap_pages(&kvmppc_uvmem_pgmap);
 out_free_region:
     release_mem_region(res->start, size);
 out:
     return ret;
 }
 
 void kvmppc_uvmem_free(void)
 {
     if (!kvmppc_uvmem_bitmap)
         return;
 
     memunmap_pages(&kvmppc_uvmem_pgmap);
     release_mem_region(kvmppc_uvmem_pgmap.range.start,
                range_len(&kvmppc_uvmem_pgmap.range));
     kfree(kvmppc_uvmem_bitmap);
 }

This page was automatically generated by the 2.1.0 LXR engine. The LXR team