include/linux/io-pgtable.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 #ifndef __IO_PGTABLE_H
0003 #define __IO_PGTABLE_H
0004
0005 #include <linux/bitops.h>
0006 #include <linux/iommu.h>
0007
0008 /*
0009  * Public API for use by IOMMU drivers
0010  */
0011 enum io_pgtable_fmt {
0012     ARM_32_LPAE_S1,
0013     ARM_32_LPAE_S2,
0014     ARM_64_LPAE_S1,
0015     ARM_64_LPAE_S2,
0016     ARM_V7S,
0017     ARM_MALI_LPAE,
0018     AMD_IOMMU_V1,
0019     APPLE_DART,
0020     IO_PGTABLE_NUM_FMTS,
0021 };
0022
0023 /**
0024  * struct iommu_flush_ops - IOMMU callbacks for TLB and page table management.
0025  *
0026  * @tlb_flush_all:  Synchronously invalidate the entire TLB context.
0027  * @tlb_flush_walk: Synchronously invalidate all intermediate TLB state
0028  *                  (sometimes referred to as the "walk cache") for a virtual
0029  *                  address range.
0030  * @tlb_add_page:   Optional callback to queue up leaf TLB invalidation for a
0031  *                  single page.  IOMMUs that cannot batch TLB invalidation
0032  *                  operations efficiently will typically issue them here, but
0033  *                  others may decide to update the iommu_iotlb_gather structure
0034  *                  and defer the invalidation until iommu_iotlb_sync() instead.
0035  *
0036  * Note that these can all be called in atomic context and must therefore
0037  * not block.
0038  */
0039 struct iommu_flush_ops {
0040     void (*tlb_flush_all)(void *cookie);
0041     void (*tlb_flush_walk)(unsigned long iova, size_t size, size_t granule,
0042                    void *cookie);
0043     void (*tlb_add_page)(struct iommu_iotlb_gather *gather,
0044                  unsigned long iova, size_t granule, void *cookie);
0045 };
0046
0047 /**
0048  * struct io_pgtable_cfg - Configuration data for a set of page tables.
0049  *
0050  * @quirks:        A bitmap of hardware quirks that require some special
0051  *                 action by the low-level page table allocator.
0052  * @pgsize_bitmap: A bitmap of page sizes supported by this set of page
0053  *                 tables.
0054  * @ias:           Input address (iova) size, in bits.
0055  * @oas:           Output address (paddr) size, in bits.
0056  * @coherent_walk  A flag to indicate whether or not page table walks made
0057  *                 by the IOMMU are coherent with the CPU caches.
0058  * @tlb:           TLB management callbacks for this set of tables.
0059  * @iommu_dev:     The device representing the DMA configuration for the
0060  *                 page table walker.
0061  */
0062 struct io_pgtable_cfg {
0063     /*
0064      * IO_PGTABLE_QUIRK_ARM_NS: (ARM formats) Set NS and NSTABLE bits in
0065      *  stage 1 PTEs, for hardware which insists on validating them
0066      *  even in non-secure state where they should normally be ignored.
0067      *
0068      * IO_PGTABLE_QUIRK_NO_PERMS: Ignore the IOMMU_READ, IOMMU_WRITE and
0069      *  IOMMU_NOEXEC flags and map everything with full access, for
0070      *  hardware which does not implement the permissions of a given
0071      *  format, and/or requires some format-specific default value.
0072      *
0073      * IO_PGTABLE_QUIRK_ARM_MTK_EXT: (ARM v7s format) MediaTek IOMMUs extend
0074      *  to support up to 35 bits PA where the bit32, bit33 and bit34 are
0075      *  encoded in the bit9, bit4 and bit5 of the PTE respectively.
0076      *
0077      * IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT: (ARM v7s format) MediaTek IOMMUs
0078      *  extend the translation table base support up to 35 bits PA, the
0079      *  encoding format is same with IO_PGTABLE_QUIRK_ARM_MTK_EXT.
0080      *
0081      * IO_PGTABLE_QUIRK_ARM_TTBR1: (ARM LPAE format) Configure the table
0082      *  for use in the upper half of a split address space.
0083      *
0084      * IO_PGTABLE_QUIRK_ARM_OUTER_WBWA: Override the outer-cacheability
0085      *  attributes set in the TCR for a non-coherent page-table walker.
0086      */
0087     #define IO_PGTABLE_QUIRK_ARM_NS         BIT(0)
0088     #define IO_PGTABLE_QUIRK_NO_PERMS       BIT(1)
0089     #define IO_PGTABLE_QUIRK_ARM_MTK_EXT        BIT(3)
0090     #define IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT   BIT(4)
0091     #define IO_PGTABLE_QUIRK_ARM_TTBR1      BIT(5)
0092     #define IO_PGTABLE_QUIRK_ARM_OUTER_WBWA     BIT(6)
0093     unsigned long           quirks;
0094     unsigned long           pgsize_bitmap;
0095     unsigned int            ias;
0096     unsigned int            oas;
0097     bool                coherent_walk;
0098     const struct iommu_flush_ops    *tlb;
0099     struct device           *iommu_dev;
0100
0101     /* Low-level data specific to the table format */
0102     union {
0103         struct {
0104             u64 ttbr;
0105             struct {
0106                 u32 ips:3;
0107                 u32 tg:2;
0108                 u32 sh:2;
0109                 u32 orgn:2;
0110                 u32 irgn:2;
0111                 u32 tsz:6;
0112             }   tcr;
0113             u64 mair;
0114         } arm_lpae_s1_cfg;
0115
0116         struct {
0117             u64 vttbr;
0118             struct {
0119                 u32 ps:3;
0120                 u32 tg:2;
0121                 u32 sh:2;
0122                 u32 orgn:2;
0123                 u32 irgn:2;
0124                 u32 sl:2;
0125                 u32 tsz:6;
0126             }   vtcr;
0127         } arm_lpae_s2_cfg;
0128
0129         struct {
0130             u32 ttbr;
0131             u32 tcr;
0132             u32 nmrr;
0133             u32 prrr;
0134         } arm_v7s_cfg;
0135
0136         struct {
0137             u64 transtab;
0138             u64 memattr;
0139         } arm_mali_lpae_cfg;
0140
0141         struct {
0142             u64 ttbr[4];
0143             u32 n_ttbrs;
0144         } apple_dart_cfg;
0145     };
0146 };
0147
0148 /**
0149  * struct io_pgtable_ops - Page table manipulation API for IOMMU drivers.
0150  *
0151  * @map:          Map a physically contiguous memory region.
0152  * @map_pages:    Map a physically contiguous range of pages of the same size.
0153  * @unmap:        Unmap a physically contiguous memory region.
0154  * @unmap_pages:  Unmap a range of virtually contiguous pages of the same size.
0155  * @iova_to_phys: Translate iova to physical address.
0156  *
0157  * These functions map directly onto the iommu_ops member functions with
0158  * the same names.
0159  */
0160 struct io_pgtable_ops {
0161     int (*map)(struct io_pgtable_ops *ops, unsigned long iova,
0162            phys_addr_t paddr, size_t size, int prot, gfp_t gfp);
0163     int (*map_pages)(struct io_pgtable_ops *ops, unsigned long iova,
0164              phys_addr_t paddr, size_t pgsize, size_t pgcount,
0165              int prot, gfp_t gfp, size_t *mapped);
0166     size_t (*unmap)(struct io_pgtable_ops *ops, unsigned long iova,
0167             size_t size, struct iommu_iotlb_gather *gather);
0168     size_t (*unmap_pages)(struct io_pgtable_ops *ops, unsigned long iova,
0169                   size_t pgsize, size_t pgcount,
0170                   struct iommu_iotlb_gather *gather);
0171     phys_addr_t (*iova_to_phys)(struct io_pgtable_ops *ops,
0172                     unsigned long iova);
0173 };
0174
0175 /**
0176  * alloc_io_pgtable_ops() - Allocate a page table allocator for use by an IOMMU.
0177  *
0178  * @fmt:    The page table format.
0179  * @cfg:    The page table configuration. This will be modified to represent
0180  *          the configuration actually provided by the allocator (e.g. the
0181  *          pgsize_bitmap may be restricted).
0182  * @cookie: An opaque token provided by the IOMMU driver and passed back to
0183  *          the callback routines in cfg->tlb.
0184  */
0185 struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt,
0186                         struct io_pgtable_cfg *cfg,
0187                         void *cookie);
0188
0189 /**
0190  * free_io_pgtable_ops() - Free an io_pgtable_ops structure. The caller
0191  *                         *must* ensure that the page table is no longer
0192  *                         live, but the TLB can be dirty.
0193  *
0194  * @ops: The ops returned from alloc_io_pgtable_ops.
0195  */
0196 void free_io_pgtable_ops(struct io_pgtable_ops *ops);
0197
0198
0199 /*
0200  * Internal structures for page table allocator implementations.
0201  */
0202
0203 /**
0204  * struct io_pgtable - Internal structure describing a set of page tables.
0205  *
0206  * @fmt:    The page table format.
0207  * @cookie: An opaque token provided by the IOMMU driver and passed back to
0208  *          any callback routines.
0209  * @cfg:    A copy of the page table configuration.
0210  * @ops:    The page table operations in use for this set of page tables.
0211  */
0212 struct io_pgtable {
0213     enum io_pgtable_fmt fmt;
0214     void            *cookie;
0215     struct io_pgtable_cfg   cfg;
0216     struct io_pgtable_ops   ops;
0217 };
0218
0219 #define io_pgtable_ops_to_pgtable(x) container_of((x), struct io_pgtable, ops)
0220
0221 static inline void io_pgtable_tlb_flush_all(struct io_pgtable *iop)
0222 {
0223     if (iop->cfg.tlb && iop->cfg.tlb->tlb_flush_all)
0224         iop->cfg.tlb->tlb_flush_all(iop->cookie);
0225 }
0226
0227 static inline void
0228 io_pgtable_tlb_flush_walk(struct io_pgtable *iop, unsigned long iova,
0229               size_t size, size_t granule)
0230 {
0231     if (iop->cfg.tlb && iop->cfg.tlb->tlb_flush_walk)
0232         iop->cfg.tlb->tlb_flush_walk(iova, size, granule, iop->cookie);
0233 }
0234
0235 static inline void
0236 io_pgtable_tlb_add_page(struct io_pgtable *iop,
0237             struct iommu_iotlb_gather * gather, unsigned long iova,
0238             size_t granule)
0239 {
0240     if (iop->cfg.tlb && iop->cfg.tlb->tlb_add_page)
0241         iop->cfg.tlb->tlb_add_page(gather, iova, granule, iop->cookie);
0242 }
0243
0244 /**
0245  * struct io_pgtable_init_fns - Alloc/free a set of page tables for a
0246  *                              particular format.
0247  *
0248  * @alloc: Allocate a set of page tables described by cfg.
0249  * @free:  Free the page tables associated with iop.
0250  */
0251 struct io_pgtable_init_fns {
0252     struct io_pgtable *(*alloc)(struct io_pgtable_cfg *cfg, void *cookie);
0253     void (*free)(struct io_pgtable *iop);
0254 };
0255
0256 extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns;
0257 extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns;
0258 extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns;
0259 extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns;
0260 extern struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns;
0261 extern struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns;
0262 extern struct io_pgtable_init_fns io_pgtable_amd_iommu_v1_init_fns;
0263 extern struct io_pgtable_init_fns io_pgtable_apple_dart_init_fns;
0264
0265 #endif /* __IO_PGTABLE_H */