Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  *    Copyright IBM Corp. 2007, 2011
0004  *    Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
0005  */
0006 
0007 #include <linux/sched.h>
0008 #include <linux/kernel.h>
0009 #include <linux/errno.h>
0010 #include <linux/gfp.h>
0011 #include <linux/mm.h>
0012 #include <linux/swap.h>
0013 #include <linux/smp.h>
0014 #include <linux/spinlock.h>
0015 #include <linux/rcupdate.h>
0016 #include <linux/slab.h>
0017 #include <linux/swapops.h>
0018 #include <linux/sysctl.h>
0019 #include <linux/ksm.h>
0020 #include <linux/mman.h>
0021 
0022 #include <asm/tlb.h>
0023 #include <asm/tlbflush.h>
0024 #include <asm/mmu_context.h>
0025 #include <asm/page-states.h>
0026 
0027 pgprot_t pgprot_writecombine(pgprot_t prot)
0028 {
0029     /*
0030      * mio_wb_bit_mask may be set on a different CPU, but it is only set
0031      * once at init and only read afterwards.
0032      */
0033     return __pgprot(pgprot_val(prot) | mio_wb_bit_mask);
0034 }
0035 EXPORT_SYMBOL_GPL(pgprot_writecombine);
0036 
0037 pgprot_t pgprot_writethrough(pgprot_t prot)
0038 {
0039     /*
0040      * mio_wb_bit_mask may be set on a different CPU, but it is only set
0041      * once at init and only read afterwards.
0042      */
0043     return __pgprot(pgprot_val(prot) & ~mio_wb_bit_mask);
0044 }
0045 EXPORT_SYMBOL_GPL(pgprot_writethrough);
0046 
0047 static inline void ptep_ipte_local(struct mm_struct *mm, unsigned long addr,
0048                    pte_t *ptep, int nodat)
0049 {
0050     unsigned long opt, asce;
0051 
0052     if (MACHINE_HAS_TLB_GUEST) {
0053         opt = 0;
0054         asce = READ_ONCE(mm->context.gmap_asce);
0055         if (asce == 0UL || nodat)
0056             opt |= IPTE_NODAT;
0057         if (asce != -1UL) {
0058             asce = asce ? : mm->context.asce;
0059             opt |= IPTE_GUEST_ASCE;
0060         }
0061         __ptep_ipte(addr, ptep, opt, asce, IPTE_LOCAL);
0062     } else {
0063         __ptep_ipte(addr, ptep, 0, 0, IPTE_LOCAL);
0064     }
0065 }
0066 
0067 static inline void ptep_ipte_global(struct mm_struct *mm, unsigned long addr,
0068                     pte_t *ptep, int nodat)
0069 {
0070     unsigned long opt, asce;
0071 
0072     if (MACHINE_HAS_TLB_GUEST) {
0073         opt = 0;
0074         asce = READ_ONCE(mm->context.gmap_asce);
0075         if (asce == 0UL || nodat)
0076             opt |= IPTE_NODAT;
0077         if (asce != -1UL) {
0078             asce = asce ? : mm->context.asce;
0079             opt |= IPTE_GUEST_ASCE;
0080         }
0081         __ptep_ipte(addr, ptep, opt, asce, IPTE_GLOBAL);
0082     } else {
0083         __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL);
0084     }
0085 }
0086 
0087 static inline pte_t ptep_flush_direct(struct mm_struct *mm,
0088                       unsigned long addr, pte_t *ptep,
0089                       int nodat)
0090 {
0091     pte_t old;
0092 
0093     old = *ptep;
0094     if (unlikely(pte_val(old) & _PAGE_INVALID))
0095         return old;
0096     atomic_inc(&mm->context.flush_count);
0097     if (MACHINE_HAS_TLB_LC &&
0098         cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
0099         ptep_ipte_local(mm, addr, ptep, nodat);
0100     else
0101         ptep_ipte_global(mm, addr, ptep, nodat);
0102     atomic_dec(&mm->context.flush_count);
0103     return old;
0104 }
0105 
0106 static inline pte_t ptep_flush_lazy(struct mm_struct *mm,
0107                     unsigned long addr, pte_t *ptep,
0108                     int nodat)
0109 {
0110     pte_t old;
0111 
0112     old = *ptep;
0113     if (unlikely(pte_val(old) & _PAGE_INVALID))
0114         return old;
0115     atomic_inc(&mm->context.flush_count);
0116     if (cpumask_equal(&mm->context.cpu_attach_mask,
0117               cpumask_of(smp_processor_id()))) {
0118         set_pte(ptep, set_pte_bit(*ptep, __pgprot(_PAGE_INVALID)));
0119         mm->context.flush_mm = 1;
0120     } else
0121         ptep_ipte_global(mm, addr, ptep, nodat);
0122     atomic_dec(&mm->context.flush_count);
0123     return old;
0124 }
0125 
0126 static inline pgste_t pgste_get_lock(pte_t *ptep)
0127 {
0128     unsigned long new = 0;
0129 #ifdef CONFIG_PGSTE
0130     unsigned long old;
0131 
0132     asm(
0133         "   lg  %0,%2\n"
0134         "0: lgr %1,%0\n"
0135         "   nihh    %0,0xff7f\n"    /* clear PCL bit in old */
0136         "   oihh    %1,0x0080\n"    /* set PCL bit in new */
0137         "   csg %0,%1,%2\n"
0138         "   jl  0b\n"
0139         : "=&d" (old), "=&d" (new), "=Q" (ptep[PTRS_PER_PTE])
0140         : "Q" (ptep[PTRS_PER_PTE]) : "cc", "memory");
0141 #endif
0142     return __pgste(new);
0143 }
0144 
0145 static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste)
0146 {
0147 #ifdef CONFIG_PGSTE
0148     asm(
0149         "   nihh    %1,0xff7f\n"    /* clear PCL bit */
0150         "   stg %1,%0\n"
0151         : "=Q" (ptep[PTRS_PER_PTE])
0152         : "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE])
0153         : "cc", "memory");
0154 #endif
0155 }
0156 
0157 static inline pgste_t pgste_get(pte_t *ptep)
0158 {
0159     unsigned long pgste = 0;
0160 #ifdef CONFIG_PGSTE
0161     pgste = *(unsigned long *)(ptep + PTRS_PER_PTE);
0162 #endif
0163     return __pgste(pgste);
0164 }
0165 
0166 static inline void pgste_set(pte_t *ptep, pgste_t pgste)
0167 {
0168 #ifdef CONFIG_PGSTE
0169     *(pgste_t *)(ptep + PTRS_PER_PTE) = pgste;
0170 #endif
0171 }
0172 
0173 static inline pgste_t pgste_update_all(pte_t pte, pgste_t pgste,
0174                        struct mm_struct *mm)
0175 {
0176 #ifdef CONFIG_PGSTE
0177     unsigned long address, bits, skey;
0178 
0179     if (!mm_uses_skeys(mm) || pte_val(pte) & _PAGE_INVALID)
0180         return pgste;
0181     address = pte_val(pte) & PAGE_MASK;
0182     skey = (unsigned long) page_get_storage_key(address);
0183     bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED);
0184     /* Transfer page changed & referenced bit to guest bits in pgste */
0185     pgste_val(pgste) |= bits << 48;     /* GR bit & GC bit */
0186     /* Copy page access key and fetch protection bit to pgste */
0187     pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT);
0188     pgste_val(pgste) |= (skey & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56;
0189 #endif
0190     return pgste;
0191 
0192 }
0193 
0194 static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry,
0195                  struct mm_struct *mm)
0196 {
0197 #ifdef CONFIG_PGSTE
0198     unsigned long address;
0199     unsigned long nkey;
0200 
0201     if (!mm_uses_skeys(mm) || pte_val(entry) & _PAGE_INVALID)
0202         return;
0203     VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID));
0204     address = pte_val(entry) & PAGE_MASK;
0205     /*
0206      * Set page access key and fetch protection bit from pgste.
0207      * The guest C/R information is still in the PGSTE, set real
0208      * key C/R to 0.
0209      */
0210     nkey = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56;
0211     nkey |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48;
0212     page_set_storage_key(address, nkey, 0);
0213 #endif
0214 }
0215 
0216 static inline pgste_t pgste_set_pte(pte_t *ptep, pgste_t pgste, pte_t entry)
0217 {
0218 #ifdef CONFIG_PGSTE
0219     if ((pte_val(entry) & _PAGE_PRESENT) &&
0220         (pte_val(entry) & _PAGE_WRITE) &&
0221         !(pte_val(entry) & _PAGE_INVALID)) {
0222         if (!MACHINE_HAS_ESOP) {
0223             /*
0224              * Without enhanced suppression-on-protection force
0225              * the dirty bit on for all writable ptes.
0226              */
0227             entry = set_pte_bit(entry, __pgprot(_PAGE_DIRTY));
0228             entry = clear_pte_bit(entry, __pgprot(_PAGE_PROTECT));
0229         }
0230         if (!(pte_val(entry) & _PAGE_PROTECT))
0231             /* This pte allows write access, set user-dirty */
0232             pgste_val(pgste) |= PGSTE_UC_BIT;
0233     }
0234 #endif
0235     set_pte(ptep, entry);
0236     return pgste;
0237 }
0238 
0239 static inline pgste_t pgste_pte_notify(struct mm_struct *mm,
0240                        unsigned long addr,
0241                        pte_t *ptep, pgste_t pgste)
0242 {
0243 #ifdef CONFIG_PGSTE
0244     unsigned long bits;
0245 
0246     bits = pgste_val(pgste) & (PGSTE_IN_BIT | PGSTE_VSIE_BIT);
0247     if (bits) {
0248         pgste_val(pgste) ^= bits;
0249         ptep_notify(mm, addr, ptep, bits);
0250     }
0251 #endif
0252     return pgste;
0253 }
0254 
0255 static inline pgste_t ptep_xchg_start(struct mm_struct *mm,
0256                       unsigned long addr, pte_t *ptep)
0257 {
0258     pgste_t pgste = __pgste(0);
0259 
0260     if (mm_has_pgste(mm)) {
0261         pgste = pgste_get_lock(ptep);
0262         pgste = pgste_pte_notify(mm, addr, ptep, pgste);
0263     }
0264     return pgste;
0265 }
0266 
0267 static inline pte_t ptep_xchg_commit(struct mm_struct *mm,
0268                     unsigned long addr, pte_t *ptep,
0269                     pgste_t pgste, pte_t old, pte_t new)
0270 {
0271     if (mm_has_pgste(mm)) {
0272         if (pte_val(old) & _PAGE_INVALID)
0273             pgste_set_key(ptep, pgste, new, mm);
0274         if (pte_val(new) & _PAGE_INVALID) {
0275             pgste = pgste_update_all(old, pgste, mm);
0276             if ((pgste_val(pgste) & _PGSTE_GPS_USAGE_MASK) ==
0277                 _PGSTE_GPS_USAGE_UNUSED)
0278                 old = set_pte_bit(old, __pgprot(_PAGE_UNUSED));
0279         }
0280         pgste = pgste_set_pte(ptep, pgste, new);
0281         pgste_set_unlock(ptep, pgste);
0282     } else {
0283         set_pte(ptep, new);
0284     }
0285     return old;
0286 }
0287 
0288 pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr,
0289                pte_t *ptep, pte_t new)
0290 {
0291     pgste_t pgste;
0292     pte_t old;
0293     int nodat;
0294 
0295     preempt_disable();
0296     pgste = ptep_xchg_start(mm, addr, ptep);
0297     nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
0298     old = ptep_flush_direct(mm, addr, ptep, nodat);
0299     old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new);
0300     preempt_enable();
0301     return old;
0302 }
0303 EXPORT_SYMBOL(ptep_xchg_direct);
0304 
0305 pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr,
0306              pte_t *ptep, pte_t new)
0307 {
0308     pgste_t pgste;
0309     pte_t old;
0310     int nodat;
0311 
0312     preempt_disable();
0313     pgste = ptep_xchg_start(mm, addr, ptep);
0314     nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
0315     old = ptep_flush_lazy(mm, addr, ptep, nodat);
0316     old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new);
0317     preempt_enable();
0318     return old;
0319 }
0320 EXPORT_SYMBOL(ptep_xchg_lazy);
0321 
0322 pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr,
0323                  pte_t *ptep)
0324 {
0325     pgste_t pgste;
0326     pte_t old;
0327     int nodat;
0328     struct mm_struct *mm = vma->vm_mm;
0329 
0330     preempt_disable();
0331     pgste = ptep_xchg_start(mm, addr, ptep);
0332     nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
0333     old = ptep_flush_lazy(mm, addr, ptep, nodat);
0334     if (mm_has_pgste(mm)) {
0335         pgste = pgste_update_all(old, pgste, mm);
0336         pgste_set(ptep, pgste);
0337     }
0338     return old;
0339 }
0340 
0341 void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr,
0342                  pte_t *ptep, pte_t old_pte, pte_t pte)
0343 {
0344     pgste_t pgste;
0345     struct mm_struct *mm = vma->vm_mm;
0346 
0347     if (!MACHINE_HAS_NX)
0348         pte = clear_pte_bit(pte, __pgprot(_PAGE_NOEXEC));
0349     if (mm_has_pgste(mm)) {
0350         pgste = pgste_get(ptep);
0351         pgste_set_key(ptep, pgste, pte, mm);
0352         pgste = pgste_set_pte(ptep, pgste, pte);
0353         pgste_set_unlock(ptep, pgste);
0354     } else {
0355         set_pte(ptep, pte);
0356     }
0357     preempt_enable();
0358 }
0359 
0360 static inline void pmdp_idte_local(struct mm_struct *mm,
0361                    unsigned long addr, pmd_t *pmdp)
0362 {
0363     if (MACHINE_HAS_TLB_GUEST)
0364         __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE,
0365                 mm->context.asce, IDTE_LOCAL);
0366     else
0367         __pmdp_idte(addr, pmdp, 0, 0, IDTE_LOCAL);
0368     if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m)
0369         gmap_pmdp_idte_local(mm, addr);
0370 }
0371 
0372 static inline void pmdp_idte_global(struct mm_struct *mm,
0373                     unsigned long addr, pmd_t *pmdp)
0374 {
0375     if (MACHINE_HAS_TLB_GUEST) {
0376         __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE,
0377                 mm->context.asce, IDTE_GLOBAL);
0378         if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m)
0379             gmap_pmdp_idte_global(mm, addr);
0380     } else if (MACHINE_HAS_IDTE) {
0381         __pmdp_idte(addr, pmdp, 0, 0, IDTE_GLOBAL);
0382         if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m)
0383             gmap_pmdp_idte_global(mm, addr);
0384     } else {
0385         __pmdp_csp(pmdp);
0386         if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m)
0387             gmap_pmdp_csp(mm, addr);
0388     }
0389 }
0390 
0391 static inline pmd_t pmdp_flush_direct(struct mm_struct *mm,
0392                       unsigned long addr, pmd_t *pmdp)
0393 {
0394     pmd_t old;
0395 
0396     old = *pmdp;
0397     if (pmd_val(old) & _SEGMENT_ENTRY_INVALID)
0398         return old;
0399     atomic_inc(&mm->context.flush_count);
0400     if (MACHINE_HAS_TLB_LC &&
0401         cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
0402         pmdp_idte_local(mm, addr, pmdp);
0403     else
0404         pmdp_idte_global(mm, addr, pmdp);
0405     atomic_dec(&mm->context.flush_count);
0406     return old;
0407 }
0408 
0409 static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm,
0410                     unsigned long addr, pmd_t *pmdp)
0411 {
0412     pmd_t old;
0413 
0414     old = *pmdp;
0415     if (pmd_val(old) & _SEGMENT_ENTRY_INVALID)
0416         return old;
0417     atomic_inc(&mm->context.flush_count);
0418     if (cpumask_equal(&mm->context.cpu_attach_mask,
0419               cpumask_of(smp_processor_id()))) {
0420         set_pmd(pmdp, set_pmd_bit(*pmdp, __pgprot(_SEGMENT_ENTRY_INVALID)));
0421         mm->context.flush_mm = 1;
0422         if (mm_has_pgste(mm))
0423             gmap_pmdp_invalidate(mm, addr);
0424     } else {
0425         pmdp_idte_global(mm, addr, pmdp);
0426     }
0427     atomic_dec(&mm->context.flush_count);
0428     return old;
0429 }
0430 
0431 #ifdef CONFIG_PGSTE
0432 static int pmd_lookup(struct mm_struct *mm, unsigned long addr, pmd_t **pmdp)
0433 {
0434     struct vm_area_struct *vma;
0435     pgd_t *pgd;
0436     p4d_t *p4d;
0437     pud_t *pud;
0438 
0439     /* We need a valid VMA, otherwise this is clearly a fault. */
0440     vma = vma_lookup(mm, addr);
0441     if (!vma)
0442         return -EFAULT;
0443 
0444     pgd = pgd_offset(mm, addr);
0445     if (!pgd_present(*pgd))
0446         return -ENOENT;
0447 
0448     p4d = p4d_offset(pgd, addr);
0449     if (!p4d_present(*p4d))
0450         return -ENOENT;
0451 
0452     pud = pud_offset(p4d, addr);
0453     if (!pud_present(*pud))
0454         return -ENOENT;
0455 
0456     /* Large PUDs are not supported yet. */
0457     if (pud_large(*pud))
0458         return -EFAULT;
0459 
0460     *pmdp = pmd_offset(pud, addr);
0461     return 0;
0462 }
0463 #endif
0464 
0465 pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr,
0466                pmd_t *pmdp, pmd_t new)
0467 {
0468     pmd_t old;
0469 
0470     preempt_disable();
0471     old = pmdp_flush_direct(mm, addr, pmdp);
0472     set_pmd(pmdp, new);
0473     preempt_enable();
0474     return old;
0475 }
0476 EXPORT_SYMBOL(pmdp_xchg_direct);
0477 
0478 pmd_t pmdp_xchg_lazy(struct mm_struct *mm, unsigned long addr,
0479              pmd_t *pmdp, pmd_t new)
0480 {
0481     pmd_t old;
0482 
0483     preempt_disable();
0484     old = pmdp_flush_lazy(mm, addr, pmdp);
0485     set_pmd(pmdp, new);
0486     preempt_enable();
0487     return old;
0488 }
0489 EXPORT_SYMBOL(pmdp_xchg_lazy);
0490 
0491 static inline void pudp_idte_local(struct mm_struct *mm,
0492                    unsigned long addr, pud_t *pudp)
0493 {
0494     if (MACHINE_HAS_TLB_GUEST)
0495         __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE,
0496                 mm->context.asce, IDTE_LOCAL);
0497     else
0498         __pudp_idte(addr, pudp, 0, 0, IDTE_LOCAL);
0499 }
0500 
0501 static inline void pudp_idte_global(struct mm_struct *mm,
0502                     unsigned long addr, pud_t *pudp)
0503 {
0504     if (MACHINE_HAS_TLB_GUEST)
0505         __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE,
0506                 mm->context.asce, IDTE_GLOBAL);
0507     else if (MACHINE_HAS_IDTE)
0508         __pudp_idte(addr, pudp, 0, 0, IDTE_GLOBAL);
0509     else
0510         /*
0511          * Invalid bit position is the same for pmd and pud, so we can
0512          * re-use _pmd_csp() here
0513          */
0514         __pmdp_csp((pmd_t *) pudp);
0515 }
0516 
0517 static inline pud_t pudp_flush_direct(struct mm_struct *mm,
0518                       unsigned long addr, pud_t *pudp)
0519 {
0520     pud_t old;
0521 
0522     old = *pudp;
0523     if (pud_val(old) & _REGION_ENTRY_INVALID)
0524         return old;
0525     atomic_inc(&mm->context.flush_count);
0526     if (MACHINE_HAS_TLB_LC &&
0527         cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
0528         pudp_idte_local(mm, addr, pudp);
0529     else
0530         pudp_idte_global(mm, addr, pudp);
0531     atomic_dec(&mm->context.flush_count);
0532     return old;
0533 }
0534 
0535 pud_t pudp_xchg_direct(struct mm_struct *mm, unsigned long addr,
0536                pud_t *pudp, pud_t new)
0537 {
0538     pud_t old;
0539 
0540     preempt_disable();
0541     old = pudp_flush_direct(mm, addr, pudp);
0542     set_pud(pudp, new);
0543     preempt_enable();
0544     return old;
0545 }
0546 EXPORT_SYMBOL(pudp_xchg_direct);
0547 
0548 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
0549 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
0550                 pgtable_t pgtable)
0551 {
0552     struct list_head *lh = (struct list_head *) pgtable;
0553 
0554     assert_spin_locked(pmd_lockptr(mm, pmdp));
0555 
0556     /* FIFO */
0557     if (!pmd_huge_pte(mm, pmdp))
0558         INIT_LIST_HEAD(lh);
0559     else
0560         list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
0561     pmd_huge_pte(mm, pmdp) = pgtable;
0562 }
0563 
0564 pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
0565 {
0566     struct list_head *lh;
0567     pgtable_t pgtable;
0568     pte_t *ptep;
0569 
0570     assert_spin_locked(pmd_lockptr(mm, pmdp));
0571 
0572     /* FIFO */
0573     pgtable = pmd_huge_pte(mm, pmdp);
0574     lh = (struct list_head *) pgtable;
0575     if (list_empty(lh))
0576         pmd_huge_pte(mm, pmdp) = NULL;
0577     else {
0578         pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
0579         list_del(lh);
0580     }
0581     ptep = (pte_t *) pgtable;
0582     set_pte(ptep, __pte(_PAGE_INVALID));
0583     ptep++;
0584     set_pte(ptep, __pte(_PAGE_INVALID));
0585     return pgtable;
0586 }
0587 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
0588 
0589 #ifdef CONFIG_PGSTE
0590 void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr,
0591              pte_t *ptep, pte_t entry)
0592 {
0593     pgste_t pgste;
0594 
0595     /* the mm_has_pgste() check is done in set_pte_at() */
0596     preempt_disable();
0597     pgste = pgste_get_lock(ptep);
0598     pgste_val(pgste) &= ~_PGSTE_GPS_ZERO;
0599     pgste_set_key(ptep, pgste, entry, mm);
0600     pgste = pgste_set_pte(ptep, pgste, entry);
0601     pgste_set_unlock(ptep, pgste);
0602     preempt_enable();
0603 }
0604 
0605 void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
0606 {
0607     pgste_t pgste;
0608 
0609     preempt_disable();
0610     pgste = pgste_get_lock(ptep);
0611     pgste_val(pgste) |= PGSTE_IN_BIT;
0612     pgste_set_unlock(ptep, pgste);
0613     preempt_enable();
0614 }
0615 
0616 /**
0617  * ptep_force_prot - change access rights of a locked pte
0618  * @mm: pointer to the process mm_struct
0619  * @addr: virtual address in the guest address space
0620  * @ptep: pointer to the page table entry
0621  * @prot: indicates guest access rights: PROT_NONE, PROT_READ or PROT_WRITE
0622  * @bit: pgste bit to set (e.g. for notification)
0623  *
0624  * Returns 0 if the access rights were changed and -EAGAIN if the current
0625  * and requested access rights are incompatible.
0626  */
0627 int ptep_force_prot(struct mm_struct *mm, unsigned long addr,
0628             pte_t *ptep, int prot, unsigned long bit)
0629 {
0630     pte_t entry;
0631     pgste_t pgste;
0632     int pte_i, pte_p, nodat;
0633 
0634     pgste = pgste_get_lock(ptep);
0635     entry = *ptep;
0636     /* Check pte entry after all locks have been acquired */
0637     pte_i = pte_val(entry) & _PAGE_INVALID;
0638     pte_p = pte_val(entry) & _PAGE_PROTECT;
0639     if ((pte_i && (prot != PROT_NONE)) ||
0640         (pte_p && (prot & PROT_WRITE))) {
0641         pgste_set_unlock(ptep, pgste);
0642         return -EAGAIN;
0643     }
0644     /* Change access rights and set pgste bit */
0645     nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
0646     if (prot == PROT_NONE && !pte_i) {
0647         ptep_flush_direct(mm, addr, ptep, nodat);
0648         pgste = pgste_update_all(entry, pgste, mm);
0649         entry = set_pte_bit(entry, __pgprot(_PAGE_INVALID));
0650     }
0651     if (prot == PROT_READ && !pte_p) {
0652         ptep_flush_direct(mm, addr, ptep, nodat);
0653         entry = clear_pte_bit(entry, __pgprot(_PAGE_INVALID));
0654         entry = set_pte_bit(entry, __pgprot(_PAGE_PROTECT));
0655     }
0656     pgste_val(pgste) |= bit;
0657     pgste = pgste_set_pte(ptep, pgste, entry);
0658     pgste_set_unlock(ptep, pgste);
0659     return 0;
0660 }
0661 
0662 int ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr,
0663             pte_t *sptep, pte_t *tptep, pte_t pte)
0664 {
0665     pgste_t spgste, tpgste;
0666     pte_t spte, tpte;
0667     int rc = -EAGAIN;
0668 
0669     if (!(pte_val(*tptep) & _PAGE_INVALID))
0670         return 0;   /* already shadowed */
0671     spgste = pgste_get_lock(sptep);
0672     spte = *sptep;
0673     if (!(pte_val(spte) & _PAGE_INVALID) &&
0674         !((pte_val(spte) & _PAGE_PROTECT) &&
0675           !(pte_val(pte) & _PAGE_PROTECT))) {
0676         pgste_val(spgste) |= PGSTE_VSIE_BIT;
0677         tpgste = pgste_get_lock(tptep);
0678         tpte = __pte((pte_val(spte) & PAGE_MASK) |
0679                  (pte_val(pte) & _PAGE_PROTECT));
0680         /* don't touch the storage key - it belongs to parent pgste */
0681         tpgste = pgste_set_pte(tptep, tpgste, tpte);
0682         pgste_set_unlock(tptep, tpgste);
0683         rc = 1;
0684     }
0685     pgste_set_unlock(sptep, spgste);
0686     return rc;
0687 }
0688 
0689 void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep)
0690 {
0691     pgste_t pgste;
0692     int nodat;
0693 
0694     pgste = pgste_get_lock(ptep);
0695     /* notifier is called by the caller */
0696     nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
0697     ptep_flush_direct(mm, saddr, ptep, nodat);
0698     /* don't touch the storage key - it belongs to parent pgste */
0699     pgste = pgste_set_pte(ptep, pgste, __pte(_PAGE_INVALID));
0700     pgste_set_unlock(ptep, pgste);
0701 }
0702 
0703 static void ptep_zap_swap_entry(struct mm_struct *mm, swp_entry_t entry)
0704 {
0705     if (!non_swap_entry(entry))
0706         dec_mm_counter(mm, MM_SWAPENTS);
0707     else if (is_migration_entry(entry)) {
0708         struct page *page = pfn_swap_entry_to_page(entry);
0709 
0710         dec_mm_counter(mm, mm_counter(page));
0711     }
0712     free_swap_and_cache(entry);
0713 }
0714 
0715 void ptep_zap_unused(struct mm_struct *mm, unsigned long addr,
0716              pte_t *ptep, int reset)
0717 {
0718     unsigned long pgstev;
0719     pgste_t pgste;
0720     pte_t pte;
0721 
0722     /* Zap unused and logically-zero pages */
0723     preempt_disable();
0724     pgste = pgste_get_lock(ptep);
0725     pgstev = pgste_val(pgste);
0726     pte = *ptep;
0727     if (!reset && pte_swap(pte) &&
0728         ((pgstev & _PGSTE_GPS_USAGE_MASK) == _PGSTE_GPS_USAGE_UNUSED ||
0729          (pgstev & _PGSTE_GPS_ZERO))) {
0730         ptep_zap_swap_entry(mm, pte_to_swp_entry(pte));
0731         pte_clear(mm, addr, ptep);
0732     }
0733     if (reset)
0734         pgste_val(pgste) &= ~_PGSTE_GPS_USAGE_MASK;
0735     pgste_set_unlock(ptep, pgste);
0736     preempt_enable();
0737 }
0738 
0739 void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
0740 {
0741     unsigned long ptev;
0742     pgste_t pgste;
0743 
0744     /* Clear storage key ACC and F, but set R/C */
0745     preempt_disable();
0746     pgste = pgste_get_lock(ptep);
0747     pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT);
0748     pgste_val(pgste) |= PGSTE_GR_BIT | PGSTE_GC_BIT;
0749     ptev = pte_val(*ptep);
0750     if (!(ptev & _PAGE_INVALID) && (ptev & _PAGE_WRITE))
0751         page_set_storage_key(ptev & PAGE_MASK, PAGE_DEFAULT_KEY, 0);
0752     pgste_set_unlock(ptep, pgste);
0753     preempt_enable();
0754 }
0755 
0756 /*
0757  * Test and reset if a guest page is dirty
0758  */
0759 bool ptep_test_and_clear_uc(struct mm_struct *mm, unsigned long addr,
0760                pte_t *ptep)
0761 {
0762     pgste_t pgste;
0763     pte_t pte;
0764     bool dirty;
0765     int nodat;
0766 
0767     pgste = pgste_get_lock(ptep);
0768     dirty = !!(pgste_val(pgste) & PGSTE_UC_BIT);
0769     pgste_val(pgste) &= ~PGSTE_UC_BIT;
0770     pte = *ptep;
0771     if (dirty && (pte_val(pte) & _PAGE_PRESENT)) {
0772         pgste = pgste_pte_notify(mm, addr, ptep, pgste);
0773         nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
0774         ptep_ipte_global(mm, addr, ptep, nodat);
0775         if (MACHINE_HAS_ESOP || !(pte_val(pte) & _PAGE_WRITE))
0776             pte = set_pte_bit(pte, __pgprot(_PAGE_PROTECT));
0777         else
0778             pte = set_pte_bit(pte, __pgprot(_PAGE_INVALID));
0779         set_pte(ptep, pte);
0780     }
0781     pgste_set_unlock(ptep, pgste);
0782     return dirty;
0783 }
0784 EXPORT_SYMBOL_GPL(ptep_test_and_clear_uc);
0785 
0786 int set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
0787               unsigned char key, bool nq)
0788 {
0789     unsigned long keyul, paddr;
0790     spinlock_t *ptl;
0791     pgste_t old, new;
0792     pmd_t *pmdp;
0793     pte_t *ptep;
0794 
0795     /*
0796      * If we don't have a PTE table and if there is no huge page mapped,
0797      * we can ignore attempts to set the key to 0, because it already is 0.
0798      */
0799     switch (pmd_lookup(mm, addr, &pmdp)) {
0800     case -ENOENT:
0801         return key ? -EFAULT : 0;
0802     case 0:
0803         break;
0804     default:
0805         return -EFAULT;
0806     }
0807 
0808     ptl = pmd_lock(mm, pmdp);
0809     if (!pmd_present(*pmdp)) {
0810         spin_unlock(ptl);
0811         return key ? -EFAULT : 0;
0812     }
0813 
0814     if (pmd_large(*pmdp)) {
0815         paddr = pmd_val(*pmdp) & HPAGE_MASK;
0816         paddr |= addr & ~HPAGE_MASK;
0817         /*
0818          * Huge pmds need quiescing operations, they are
0819          * always mapped.
0820          */
0821         page_set_storage_key(paddr, key, 1);
0822         spin_unlock(ptl);
0823         return 0;
0824     }
0825     spin_unlock(ptl);
0826 
0827     ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
0828     new = old = pgste_get_lock(ptep);
0829     pgste_val(new) &= ~(PGSTE_GR_BIT | PGSTE_GC_BIT |
0830                 PGSTE_ACC_BITS | PGSTE_FP_BIT);
0831     keyul = (unsigned long) key;
0832     pgste_val(new) |= (keyul & (_PAGE_CHANGED | _PAGE_REFERENCED)) << 48;
0833     pgste_val(new) |= (keyul & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56;
0834     if (!(pte_val(*ptep) & _PAGE_INVALID)) {
0835         unsigned long bits, skey;
0836 
0837         paddr = pte_val(*ptep) & PAGE_MASK;
0838         skey = (unsigned long) page_get_storage_key(paddr);
0839         bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED);
0840         skey = key & (_PAGE_ACC_BITS | _PAGE_FP_BIT);
0841         /* Set storage key ACC and FP */
0842         page_set_storage_key(paddr, skey, !nq);
0843         /* Merge host changed & referenced into pgste  */
0844         pgste_val(new) |= bits << 52;
0845     }
0846     /* changing the guest storage key is considered a change of the page */
0847     if ((pgste_val(new) ^ pgste_val(old)) &
0848         (PGSTE_ACC_BITS | PGSTE_FP_BIT | PGSTE_GR_BIT | PGSTE_GC_BIT))
0849         pgste_val(new) |= PGSTE_UC_BIT;
0850 
0851     pgste_set_unlock(ptep, new);
0852     pte_unmap_unlock(ptep, ptl);
0853     return 0;
0854 }
0855 EXPORT_SYMBOL(set_guest_storage_key);
0856 
0857 /*
0858  * Conditionally set a guest storage key (handling csske).
0859  * oldkey will be updated when either mr or mc is set and a pointer is given.
0860  *
0861  * Returns 0 if a guests storage key update wasn't necessary, 1 if the guest
0862  * storage key was updated and -EFAULT on access errors.
0863  */
0864 int cond_set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
0865                    unsigned char key, unsigned char *oldkey,
0866                    bool nq, bool mr, bool mc)
0867 {
0868     unsigned char tmp, mask = _PAGE_ACC_BITS | _PAGE_FP_BIT;
0869     int rc;
0870 
0871     /* we can drop the pgste lock between getting and setting the key */
0872     if (mr | mc) {
0873         rc = get_guest_storage_key(current->mm, addr, &tmp);
0874         if (rc)
0875             return rc;
0876         if (oldkey)
0877             *oldkey = tmp;
0878         if (!mr)
0879             mask |= _PAGE_REFERENCED;
0880         if (!mc)
0881             mask |= _PAGE_CHANGED;
0882         if (!((tmp ^ key) & mask))
0883             return 0;
0884     }
0885     rc = set_guest_storage_key(current->mm, addr, key, nq);
0886     return rc < 0 ? rc : 1;
0887 }
0888 EXPORT_SYMBOL(cond_set_guest_storage_key);
0889 
0890 /*
0891  * Reset a guest reference bit (rrbe), returning the reference and changed bit.
0892  *
0893  * Returns < 0 in case of error, otherwise the cc to be reported to the guest.
0894  */
0895 int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr)
0896 {
0897     spinlock_t *ptl;
0898     unsigned long paddr;
0899     pgste_t old, new;
0900     pmd_t *pmdp;
0901     pte_t *ptep;
0902     int cc = 0;
0903 
0904     /*
0905      * If we don't have a PTE table and if there is no huge page mapped,
0906      * the storage key is 0 and there is nothing for us to do.
0907      */
0908     switch (pmd_lookup(mm, addr, &pmdp)) {
0909     case -ENOENT:
0910         return 0;
0911     case 0:
0912         break;
0913     default:
0914         return -EFAULT;
0915     }
0916 
0917     ptl = pmd_lock(mm, pmdp);
0918     if (!pmd_present(*pmdp)) {
0919         spin_unlock(ptl);
0920         return 0;
0921     }
0922 
0923     if (pmd_large(*pmdp)) {
0924         paddr = pmd_val(*pmdp) & HPAGE_MASK;
0925         paddr |= addr & ~HPAGE_MASK;
0926         cc = page_reset_referenced(paddr);
0927         spin_unlock(ptl);
0928         return cc;
0929     }
0930     spin_unlock(ptl);
0931 
0932     ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
0933     new = old = pgste_get_lock(ptep);
0934     /* Reset guest reference bit only */
0935     pgste_val(new) &= ~PGSTE_GR_BIT;
0936 
0937     if (!(pte_val(*ptep) & _PAGE_INVALID)) {
0938         paddr = pte_val(*ptep) & PAGE_MASK;
0939         cc = page_reset_referenced(paddr);
0940         /* Merge real referenced bit into host-set */
0941         pgste_val(new) |= ((unsigned long) cc << 53) & PGSTE_HR_BIT;
0942     }
0943     /* Reflect guest's logical view, not physical */
0944     cc |= (pgste_val(old) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 49;
0945     /* Changing the guest storage key is considered a change of the page */
0946     if ((pgste_val(new) ^ pgste_val(old)) & PGSTE_GR_BIT)
0947         pgste_val(new) |= PGSTE_UC_BIT;
0948 
0949     pgste_set_unlock(ptep, new);
0950     pte_unmap_unlock(ptep, ptl);
0951     return cc;
0952 }
0953 EXPORT_SYMBOL(reset_guest_reference_bit);
0954 
0955 int get_guest_storage_key(struct mm_struct *mm, unsigned long addr,
0956               unsigned char *key)
0957 {
0958     unsigned long paddr;
0959     spinlock_t *ptl;
0960     pgste_t pgste;
0961     pmd_t *pmdp;
0962     pte_t *ptep;
0963 
0964     /*
0965      * If we don't have a PTE table and if there is no huge page mapped,
0966      * the storage key is 0.
0967      */
0968     *key = 0;
0969 
0970     switch (pmd_lookup(mm, addr, &pmdp)) {
0971     case -ENOENT:
0972         return 0;
0973     case 0:
0974         break;
0975     default:
0976         return -EFAULT;
0977     }
0978 
0979     ptl = pmd_lock(mm, pmdp);
0980     if (!pmd_present(*pmdp)) {
0981         spin_unlock(ptl);
0982         return 0;
0983     }
0984 
0985     if (pmd_large(*pmdp)) {
0986         paddr = pmd_val(*pmdp) & HPAGE_MASK;
0987         paddr |= addr & ~HPAGE_MASK;
0988         *key = page_get_storage_key(paddr);
0989         spin_unlock(ptl);
0990         return 0;
0991     }
0992     spin_unlock(ptl);
0993 
0994     ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
0995     pgste = pgste_get_lock(ptep);
0996     *key = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56;
0997     paddr = pte_val(*ptep) & PAGE_MASK;
0998     if (!(pte_val(*ptep) & _PAGE_INVALID))
0999         *key = page_get_storage_key(paddr);
1000     /* Reflect guest's logical view, not physical */
1001     *key |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48;
1002     pgste_set_unlock(ptep, pgste);
1003     pte_unmap_unlock(ptep, ptl);
1004     return 0;
1005 }
1006 EXPORT_SYMBOL(get_guest_storage_key);
1007 
1008 /**
1009  * pgste_perform_essa - perform ESSA actions on the PGSTE.
1010  * @mm: the memory context. It must have PGSTEs, no check is performed here!
1011  * @hva: the host virtual address of the page whose PGSTE is to be processed
1012  * @orc: the specific action to perform, see the ESSA_SET_* macros.
1013  * @oldpte: the PTE will be saved there if the pointer is not NULL.
1014  * @oldpgste: the old PGSTE will be saved there if the pointer is not NULL.
1015  *
1016  * Return: 1 if the page is to be added to the CBRL, otherwise 0,
1017  *     or < 0 in case of error. -EINVAL is returned for invalid values
1018  *     of orc, -EFAULT for invalid addresses.
1019  */
1020 int pgste_perform_essa(struct mm_struct *mm, unsigned long hva, int orc,
1021             unsigned long *oldpte, unsigned long *oldpgste)
1022 {
1023     struct vm_area_struct *vma;
1024     unsigned long pgstev;
1025     spinlock_t *ptl;
1026     pgste_t pgste;
1027     pte_t *ptep;
1028     int res = 0;
1029 
1030     WARN_ON_ONCE(orc > ESSA_MAX);
1031     if (unlikely(orc > ESSA_MAX))
1032         return -EINVAL;
1033 
1034     vma = vma_lookup(mm, hva);
1035     if (!vma || is_vm_hugetlb_page(vma))
1036         return -EFAULT;
1037     ptep = get_locked_pte(mm, hva, &ptl);
1038     if (unlikely(!ptep))
1039         return -EFAULT;
1040     pgste = pgste_get_lock(ptep);
1041     pgstev = pgste_val(pgste);
1042     if (oldpte)
1043         *oldpte = pte_val(*ptep);
1044     if (oldpgste)
1045         *oldpgste = pgstev;
1046 
1047     switch (orc) {
1048     case ESSA_GET_STATE:
1049         break;
1050     case ESSA_SET_STABLE:
1051         pgstev &= ~(_PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT);
1052         pgstev |= _PGSTE_GPS_USAGE_STABLE;
1053         break;
1054     case ESSA_SET_UNUSED:
1055         pgstev &= ~_PGSTE_GPS_USAGE_MASK;
1056         pgstev |= _PGSTE_GPS_USAGE_UNUSED;
1057         if (pte_val(*ptep) & _PAGE_INVALID)
1058             res = 1;
1059         break;
1060     case ESSA_SET_VOLATILE:
1061         pgstev &= ~_PGSTE_GPS_USAGE_MASK;
1062         pgstev |= _PGSTE_GPS_USAGE_VOLATILE;
1063         if (pte_val(*ptep) & _PAGE_INVALID)
1064             res = 1;
1065         break;
1066     case ESSA_SET_POT_VOLATILE:
1067         pgstev &= ~_PGSTE_GPS_USAGE_MASK;
1068         if (!(pte_val(*ptep) & _PAGE_INVALID)) {
1069             pgstev |= _PGSTE_GPS_USAGE_POT_VOLATILE;
1070             break;
1071         }
1072         if (pgstev & _PGSTE_GPS_ZERO) {
1073             pgstev |= _PGSTE_GPS_USAGE_VOLATILE;
1074             break;
1075         }
1076         if (!(pgstev & PGSTE_GC_BIT)) {
1077             pgstev |= _PGSTE_GPS_USAGE_VOLATILE;
1078             res = 1;
1079             break;
1080         }
1081         break;
1082     case ESSA_SET_STABLE_RESIDENT:
1083         pgstev &= ~_PGSTE_GPS_USAGE_MASK;
1084         pgstev |= _PGSTE_GPS_USAGE_STABLE;
1085         /*
1086          * Since the resident state can go away any time after this
1087          * call, we will not make this page resident. We can revisit
1088          * this decision if a guest will ever start using this.
1089          */
1090         break;
1091     case ESSA_SET_STABLE_IF_RESIDENT:
1092         if (!(pte_val(*ptep) & _PAGE_INVALID)) {
1093             pgstev &= ~_PGSTE_GPS_USAGE_MASK;
1094             pgstev |= _PGSTE_GPS_USAGE_STABLE;
1095         }
1096         break;
1097     case ESSA_SET_STABLE_NODAT:
1098         pgstev &= ~_PGSTE_GPS_USAGE_MASK;
1099         pgstev |= _PGSTE_GPS_USAGE_STABLE | _PGSTE_GPS_NODAT;
1100         break;
1101     default:
1102         /* we should never get here! */
1103         break;
1104     }
1105     /* If we are discarding a page, set it to logical zero */
1106     if (res)
1107         pgstev |= _PGSTE_GPS_ZERO;
1108 
1109     pgste_val(pgste) = pgstev;
1110     pgste_set_unlock(ptep, pgste);
1111     pte_unmap_unlock(ptep, ptl);
1112     return res;
1113 }
1114 EXPORT_SYMBOL(pgste_perform_essa);
1115 
1116 /**
1117  * set_pgste_bits - set specific PGSTE bits.
1118  * @mm: the memory context. It must have PGSTEs, no check is performed here!
1119  * @hva: the host virtual address of the page whose PGSTE is to be processed
1120  * @bits: a bitmask representing the bits that will be touched
1121  * @value: the values of the bits to be written. Only the bits in the mask
1122  *     will be written.
1123  *
1124  * Return: 0 on success, < 0 in case of error.
1125  */
1126 int set_pgste_bits(struct mm_struct *mm, unsigned long hva,
1127             unsigned long bits, unsigned long value)
1128 {
1129     struct vm_area_struct *vma;
1130     spinlock_t *ptl;
1131     pgste_t new;
1132     pte_t *ptep;
1133 
1134     vma = vma_lookup(mm, hva);
1135     if (!vma || is_vm_hugetlb_page(vma))
1136         return -EFAULT;
1137     ptep = get_locked_pte(mm, hva, &ptl);
1138     if (unlikely(!ptep))
1139         return -EFAULT;
1140     new = pgste_get_lock(ptep);
1141 
1142     pgste_val(new) &= ~bits;
1143     pgste_val(new) |= value & bits;
1144 
1145     pgste_set_unlock(ptep, new);
1146     pte_unmap_unlock(ptep, ptl);
1147     return 0;
1148 }
1149 EXPORT_SYMBOL(set_pgste_bits);
1150 
1151 /**
1152  * get_pgste - get the current PGSTE for the given address.
1153  * @mm: the memory context. It must have PGSTEs, no check is performed here!
1154  * @hva: the host virtual address of the page whose PGSTE is to be processed
1155  * @pgstep: will be written with the current PGSTE for the given address.
1156  *
1157  * Return: 0 on success, < 0 in case of error.
1158  */
1159 int get_pgste(struct mm_struct *mm, unsigned long hva, unsigned long *pgstep)
1160 {
1161     struct vm_area_struct *vma;
1162     spinlock_t *ptl;
1163     pte_t *ptep;
1164 
1165     vma = vma_lookup(mm, hva);
1166     if (!vma || is_vm_hugetlb_page(vma))
1167         return -EFAULT;
1168     ptep = get_locked_pte(mm, hva, &ptl);
1169     if (unlikely(!ptep))
1170         return -EFAULT;
1171     *pgstep = pgste_val(pgste_get(ptep));
1172     pte_unmap_unlock(ptep, ptl);
1173     return 0;
1174 }
1175 EXPORT_SYMBOL(get_pgste);
1176 #endif
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!