include/linux/dma-map-ops.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /*
0003  * This header is for implementations of dma_map_ops and related code.
0004  * It should not be included in drivers just using the DMA API.
0005  */
0006 #ifndef _LINUX_DMA_MAP_OPS_H
0007 #define _LINUX_DMA_MAP_OPS_H
0008
0009 #include <linux/dma-mapping.h>
0010 #include <linux/pgtable.h>
0011
0012 struct cma;
0013
0014 /*
0015  * Values for struct dma_map_ops.flags:
0016  *
0017  * DMA_F_PCI_P2PDMA_SUPPORTED: Indicates the dma_map_ops implementation can
0018  * handle PCI P2PDMA pages in the map_sg/unmap_sg operation.
0019  */
0020 #define DMA_F_PCI_P2PDMA_SUPPORTED     (1 << 0)
0021
0022 struct dma_map_ops {
0023     unsigned int flags;
0024
0025     void *(*alloc)(struct device *dev, size_t size,
0026             dma_addr_t *dma_handle, gfp_t gfp,
0027             unsigned long attrs);
0028     void (*free)(struct device *dev, size_t size, void *vaddr,
0029             dma_addr_t dma_handle, unsigned long attrs);
0030     struct page *(*alloc_pages)(struct device *dev, size_t size,
0031             dma_addr_t *dma_handle, enum dma_data_direction dir,
0032             gfp_t gfp);
0033     void (*free_pages)(struct device *dev, size_t size, struct page *vaddr,
0034             dma_addr_t dma_handle, enum dma_data_direction dir);
0035     struct sg_table *(*alloc_noncontiguous)(struct device *dev, size_t size,
0036             enum dma_data_direction dir, gfp_t gfp,
0037             unsigned long attrs);
0038     void (*free_noncontiguous)(struct device *dev, size_t size,
0039             struct sg_table *sgt, enum dma_data_direction dir);
0040     int (*mmap)(struct device *, struct vm_area_struct *,
0041             void *, dma_addr_t, size_t, unsigned long attrs);
0042
0043     int (*get_sgtable)(struct device *dev, struct sg_table *sgt,
0044             void *cpu_addr, dma_addr_t dma_addr, size_t size,
0045             unsigned long attrs);
0046
0047     dma_addr_t (*map_page)(struct device *dev, struct page *page,
0048             unsigned long offset, size_t size,
0049             enum dma_data_direction dir, unsigned long attrs);
0050     void (*unmap_page)(struct device *dev, dma_addr_t dma_handle,
0051             size_t size, enum dma_data_direction dir,
0052             unsigned long attrs);
0053     /*
0054      * map_sg should return a negative error code on error. See
0055      * dma_map_sgtable() for a list of appropriate error codes
0056      * and their meanings.
0057      */
0058     int (*map_sg)(struct device *dev, struct scatterlist *sg, int nents,
0059             enum dma_data_direction dir, unsigned long attrs);
0060     void (*unmap_sg)(struct device *dev, struct scatterlist *sg, int nents,
0061             enum dma_data_direction dir, unsigned long attrs);
0062     dma_addr_t (*map_resource)(struct device *dev, phys_addr_t phys_addr,
0063             size_t size, enum dma_data_direction dir,
0064             unsigned long attrs);
0065     void (*unmap_resource)(struct device *dev, dma_addr_t dma_handle,
0066             size_t size, enum dma_data_direction dir,
0067             unsigned long attrs);
0068     void (*sync_single_for_cpu)(struct device *dev, dma_addr_t dma_handle,
0069             size_t size, enum dma_data_direction dir);
0070     void (*sync_single_for_device)(struct device *dev,
0071             dma_addr_t dma_handle, size_t size,
0072             enum dma_data_direction dir);
0073     void (*sync_sg_for_cpu)(struct device *dev, struct scatterlist *sg,
0074             int nents, enum dma_data_direction dir);
0075     void (*sync_sg_for_device)(struct device *dev, struct scatterlist *sg,
0076             int nents, enum dma_data_direction dir);
0077     void (*cache_sync)(struct device *dev, void *vaddr, size_t size,
0078             enum dma_data_direction direction);
0079     int (*dma_supported)(struct device *dev, u64 mask);
0080     u64 (*get_required_mask)(struct device *dev);
0081     size_t (*max_mapping_size)(struct device *dev);
0082     size_t (*opt_mapping_size)(void);
0083     unsigned long (*get_merge_boundary)(struct device *dev);
0084 };
0085
0086 #ifdef CONFIG_DMA_OPS
0087 #include <asm/dma-mapping.h>
0088
0089 static inline const struct dma_map_ops *get_dma_ops(struct device *dev)
0090 {
0091     if (dev->dma_ops)
0092         return dev->dma_ops;
0093     return get_arch_dma_ops(dev->bus);
0094 }
0095
0096 static inline void set_dma_ops(struct device *dev,
0097                    const struct dma_map_ops *dma_ops)
0098 {
0099     dev->dma_ops = dma_ops;
0100 }
0101 #else /* CONFIG_DMA_OPS */
0102 static inline const struct dma_map_ops *get_dma_ops(struct device *dev)
0103 {
0104     return NULL;
0105 }
0106 static inline void set_dma_ops(struct device *dev,
0107                    const struct dma_map_ops *dma_ops)
0108 {
0109 }
0110 #endif /* CONFIG_DMA_OPS */
0111
0112 #ifdef CONFIG_DMA_CMA
0113 extern struct cma *dma_contiguous_default_area;
0114
0115 static inline struct cma *dev_get_cma_area(struct device *dev)
0116 {
0117     if (dev && dev->cma_area)
0118         return dev->cma_area;
0119     return dma_contiguous_default_area;
0120 }
0121
0122 void dma_contiguous_reserve(phys_addr_t addr_limit);
0123 int __init dma_contiguous_reserve_area(phys_addr_t size, phys_addr_t base,
0124         phys_addr_t limit, struct cma **res_cma, bool fixed);
0125
0126 struct page *dma_alloc_from_contiguous(struct device *dev, size_t count,
0127                        unsigned int order, bool no_warn);
0128 bool dma_release_from_contiguous(struct device *dev, struct page *pages,
0129                  int count);
0130 struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp);
0131 void dma_free_contiguous(struct device *dev, struct page *page, size_t size);
0132
0133 void dma_contiguous_early_fixup(phys_addr_t base, unsigned long size);
0134 #else /* CONFIG_DMA_CMA */
0135 static inline struct cma *dev_get_cma_area(struct device *dev)
0136 {
0137     return NULL;
0138 }
0139 static inline void dma_contiguous_reserve(phys_addr_t limit)
0140 {
0141 }
0142 static inline int dma_contiguous_reserve_area(phys_addr_t size,
0143         phys_addr_t base, phys_addr_t limit, struct cma **res_cma,
0144         bool fixed)
0145 {
0146     return -ENOSYS;
0147 }
0148 static inline struct page *dma_alloc_from_contiguous(struct device *dev,
0149         size_t count, unsigned int order, bool no_warn)
0150 {
0151     return NULL;
0152 }
0153 static inline bool dma_release_from_contiguous(struct device *dev,
0154         struct page *pages, int count)
0155 {
0156     return false;
0157 }
0158 /* Use fallback alloc() and free() when CONFIG_DMA_CMA=n */
0159 static inline struct page *dma_alloc_contiguous(struct device *dev, size_t size,
0160         gfp_t gfp)
0161 {
0162     return NULL;
0163 }
0164 static inline void dma_free_contiguous(struct device *dev, struct page *page,
0165         size_t size)
0166 {
0167     __free_pages(page, get_order(size));
0168 }
0169 #endif /* CONFIG_DMA_CMA*/
0170
0171 #ifdef CONFIG_DMA_PERNUMA_CMA
0172 void dma_pernuma_cma_reserve(void);
0173 #else
0174 static inline void dma_pernuma_cma_reserve(void) { }
0175 #endif /* CONFIG_DMA_PERNUMA_CMA */
0176
0177 #ifdef CONFIG_DMA_DECLARE_COHERENT
0178 int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
0179         dma_addr_t device_addr, size_t size);
0180 void dma_release_coherent_memory(struct device *dev);
0181 int dma_alloc_from_dev_coherent(struct device *dev, ssize_t size,
0182         dma_addr_t *dma_handle, void **ret);
0183 int dma_release_from_dev_coherent(struct device *dev, int order, void *vaddr);
0184 int dma_mmap_from_dev_coherent(struct device *dev, struct vm_area_struct *vma,
0185         void *cpu_addr, size_t size, int *ret);
0186 #else
0187 static inline int dma_declare_coherent_memory(struct device *dev,
0188         phys_addr_t phys_addr, dma_addr_t device_addr, size_t size)
0189 {
0190     return -ENOSYS;
0191 }
0192
0193 #define dma_alloc_from_dev_coherent(dev, size, handle, ret) (0)
0194 #define dma_release_from_dev_coherent(dev, order, vaddr) (0)
0195 #define dma_mmap_from_dev_coherent(dev, vma, vaddr, order, ret) (0)
0196 static inline void dma_release_coherent_memory(struct device *dev) { }
0197 #endif /* CONFIG_DMA_DECLARE_COHERENT */
0198
0199 #ifdef CONFIG_DMA_GLOBAL_POOL
0200 void *dma_alloc_from_global_coherent(struct device *dev, ssize_t size,
0201         dma_addr_t *dma_handle);
0202 int dma_release_from_global_coherent(int order, void *vaddr);
0203 int dma_mmap_from_global_coherent(struct vm_area_struct *vma, void *cpu_addr,
0204         size_t size, int *ret);
0205 int dma_init_global_coherent(phys_addr_t phys_addr, size_t size);
0206 #else
0207 static inline void *dma_alloc_from_global_coherent(struct device *dev,
0208         ssize_t size, dma_addr_t *dma_handle)
0209 {
0210     return NULL;
0211 }
0212 static inline int dma_release_from_global_coherent(int order, void *vaddr)
0213 {
0214     return 0;
0215 }
0216 static inline int dma_mmap_from_global_coherent(struct vm_area_struct *vma,
0217         void *cpu_addr, size_t size, int *ret)
0218 {
0219     return 0;
0220 }
0221 #endif /* CONFIG_DMA_GLOBAL_POOL */
0222
0223 /*
0224  * This is the actual return value from the ->alloc_noncontiguous method.
0225  * The users of the DMA API should only care about the sg_table, but to make
0226  * the DMA-API internal vmaping and freeing easier we stash away the page
0227  * array as well (except for the fallback case).  This can go away any time,
0228  * e.g. when a vmap-variant that takes a scatterlist comes along.
0229  */
0230 struct dma_sgt_handle {
0231     struct sg_table sgt;
0232     struct page **pages;
0233 };
0234 #define sgt_handle(sgt) \
0235     container_of((sgt), struct dma_sgt_handle, sgt)
0236
0237 int dma_common_get_sgtable(struct device *dev, struct sg_table *sgt,
0238         void *cpu_addr, dma_addr_t dma_addr, size_t size,
0239         unsigned long attrs);
0240 int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
0241         void *cpu_addr, dma_addr_t dma_addr, size_t size,
0242         unsigned long attrs);
0243 struct page *dma_common_alloc_pages(struct device *dev, size_t size,
0244         dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp);
0245 void dma_common_free_pages(struct device *dev, size_t size, struct page *vaddr,
0246         dma_addr_t dma_handle, enum dma_data_direction dir);
0247
0248 struct page **dma_common_find_pages(void *cpu_addr);
0249 void *dma_common_contiguous_remap(struct page *page, size_t size, pgprot_t prot,
0250         const void *caller);
0251 void *dma_common_pages_remap(struct page **pages, size_t size, pgprot_t prot,
0252         const void *caller);
0253 void dma_common_free_remap(void *cpu_addr, size_t size);
0254
0255 struct page *dma_alloc_from_pool(struct device *dev, size_t size,
0256         void **cpu_addr, gfp_t flags,
0257         bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t));
0258 bool dma_free_from_pool(struct device *dev, void *start, size_t size);
0259
0260 int dma_direct_set_offset(struct device *dev, phys_addr_t cpu_start,
0261         dma_addr_t dma_start, u64 size);
0262
0263 #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
0264     defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
0265     defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
0266 extern bool dma_default_coherent;
0267 static inline bool dev_is_dma_coherent(struct device *dev)
0268 {
0269     return dev->dma_coherent;
0270 }
0271 #else
0272 static inline bool dev_is_dma_coherent(struct device *dev)
0273 {
0274     return true;
0275 }
0276 #endif /* CONFIG_ARCH_HAS_DMA_COHERENCE_H */
0277
0278 void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
0279         gfp_t gfp, unsigned long attrs);
0280 void arch_dma_free(struct device *dev, size_t size, void *cpu_addr,
0281         dma_addr_t dma_addr, unsigned long attrs);
0282
0283 #ifdef CONFIG_MMU
0284 /*
0285  * Page protection so that devices that can't snoop CPU caches can use the
0286  * memory coherently.  We default to pgprot_noncached which is usually used
0287  * for ioremap as a safe bet, but architectures can override this with less
0288  * strict semantics if possible.
0289  */
0290 #ifndef pgprot_dmacoherent
0291 #define pgprot_dmacoherent(prot)    pgprot_noncached(prot)
0292 #endif
0293
0294 pgprot_t dma_pgprot(struct device *dev, pgprot_t prot, unsigned long attrs);
0295 #else
0296 static inline pgprot_t dma_pgprot(struct device *dev, pgprot_t prot,
0297         unsigned long attrs)
0298 {
0299     return prot;    /* no protection bits supported without page tables */
0300 }
0301 #endif /* CONFIG_MMU */
0302
0303 #ifdef CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE
0304 void arch_sync_dma_for_device(phys_addr_t paddr, size_t size,
0305         enum dma_data_direction dir);
0306 #else
0307 static inline void arch_sync_dma_for_device(phys_addr_t paddr, size_t size,
0308         enum dma_data_direction dir)
0309 {
0310 }
0311 #endif /* ARCH_HAS_SYNC_DMA_FOR_DEVICE */
0312
0313 #ifdef CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU
0314 void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size,
0315         enum dma_data_direction dir);
0316 #else
0317 static inline void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size,
0318         enum dma_data_direction dir)
0319 {
0320 }
0321 #endif /* ARCH_HAS_SYNC_DMA_FOR_CPU */
0322
0323 #ifdef CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL
0324 void arch_sync_dma_for_cpu_all(void);
0325 #else
0326 static inline void arch_sync_dma_for_cpu_all(void)
0327 {
0328 }
0329 #endif /* CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL */
0330
0331 #ifdef CONFIG_ARCH_HAS_DMA_PREP_COHERENT
0332 void arch_dma_prep_coherent(struct page *page, size_t size);
0333 #else
0334 static inline void arch_dma_prep_coherent(struct page *page, size_t size)
0335 {
0336 }
0337 #endif /* CONFIG_ARCH_HAS_DMA_PREP_COHERENT */
0338
0339 #ifdef CONFIG_ARCH_HAS_DMA_MARK_CLEAN
0340 void arch_dma_mark_clean(phys_addr_t paddr, size_t size);
0341 #else
0342 static inline void arch_dma_mark_clean(phys_addr_t paddr, size_t size)
0343 {
0344 }
0345 #endif /* ARCH_HAS_DMA_MARK_CLEAN */
0346
0347 void *arch_dma_set_uncached(void *addr, size_t size);
0348 void arch_dma_clear_uncached(void *addr, size_t size);
0349
0350 #ifdef CONFIG_ARCH_HAS_DMA_MAP_DIRECT
0351 bool arch_dma_map_page_direct(struct device *dev, phys_addr_t addr);
0352 bool arch_dma_unmap_page_direct(struct device *dev, dma_addr_t dma_handle);
0353 bool arch_dma_map_sg_direct(struct device *dev, struct scatterlist *sg,
0354         int nents);
0355 bool arch_dma_unmap_sg_direct(struct device *dev, struct scatterlist *sg,
0356         int nents);
0357 #else
0358 #define arch_dma_map_page_direct(d, a)      (false)
0359 #define arch_dma_unmap_page_direct(d, a)    (false)
0360 #define arch_dma_map_sg_direct(d, s, n)     (false)
0361 #define arch_dma_unmap_sg_direct(d, s, n)   (false)
0362 #endif
0363
0364 #ifdef CONFIG_ARCH_HAS_SETUP_DMA_OPS
0365 void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
0366         const struct iommu_ops *iommu, bool coherent);
0367 #else
0368 static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base,
0369         u64 size, const struct iommu_ops *iommu, bool coherent)
0370 {
0371 }
0372 #endif /* CONFIG_ARCH_HAS_SETUP_DMA_OPS */
0373
0374 #ifdef CONFIG_ARCH_HAS_TEARDOWN_DMA_OPS
0375 void arch_teardown_dma_ops(struct device *dev);
0376 #else
0377 static inline void arch_teardown_dma_ops(struct device *dev)
0378 {
0379 }
0380 #endif /* CONFIG_ARCH_HAS_TEARDOWN_DMA_OPS */
0381
0382 #ifdef CONFIG_DMA_API_DEBUG
0383 void dma_debug_add_bus(struct bus_type *bus);
0384 void debug_dma_dump_mappings(struct device *dev);
0385 #else
0386 static inline void dma_debug_add_bus(struct bus_type *bus)
0387 {
0388 }
0389 static inline void debug_dma_dump_mappings(struct device *dev)
0390 {
0391 }
0392 #endif /* CONFIG_DMA_API_DEBUG */
0393
0394 extern const struct dma_map_ops dma_dummy_ops;
0395
0396 enum pci_p2pdma_map_type {
0397     /*
0398      * PCI_P2PDMA_MAP_UNKNOWN: Used internally for indicating the mapping
0399      * type hasn't been calculated yet. Functions that return this enum
0400      * never return this value.
0401      */
0402     PCI_P2PDMA_MAP_UNKNOWN = 0,
0403
0404     /*
0405      * PCI_P2PDMA_MAP_NOT_SUPPORTED: Indicates the transaction will
0406      * traverse the host bridge and the host bridge is not in the
0407      * allowlist. DMA Mapping routines should return an error when
0408      * this is returned.
0409      */
0410     PCI_P2PDMA_MAP_NOT_SUPPORTED,
0411
0412     /*
0413      * PCI_P2PDMA_BUS_ADDR: Indicates that two devices can talk to
0414      * each other directly through a PCI switch and the transaction will
0415      * not traverse the host bridge. Such a mapping should program
0416      * the DMA engine with PCI bus addresses.
0417      */
0418     PCI_P2PDMA_MAP_BUS_ADDR,
0419
0420     /*
0421      * PCI_P2PDMA_MAP_THRU_HOST_BRIDGE: Indicates two devices can talk
0422      * to each other, but the transaction traverses a host bridge on the
0423      * allowlist. In this case, a normal mapping either with CPU physical
0424      * addresses (in the case of dma-direct) or IOVA addresses (in the
0425      * case of IOMMUs) should be used to program the DMA engine.
0426      */
0427     PCI_P2PDMA_MAP_THRU_HOST_BRIDGE,
0428 };
0429
0430 struct pci_p2pdma_map_state {
0431     struct dev_pagemap *pgmap;
0432     int map;
0433     u64 bus_off;
0434 };
0435
0436 #ifdef CONFIG_PCI_P2PDMA
0437 enum pci_p2pdma_map_type
0438 pci_p2pdma_map_segment(struct pci_p2pdma_map_state *state, struct device *dev,
0439                struct scatterlist *sg);
0440 #else /* CONFIG_PCI_P2PDMA */
0441 static inline enum pci_p2pdma_map_type
0442 pci_p2pdma_map_segment(struct pci_p2pdma_map_state *state, struct device *dev,
0443                struct scatterlist *sg)
0444 {
0445     return PCI_P2PDMA_MAP_NOT_SUPPORTED;
0446 }
0447 #endif /* CONFIG_PCI_P2PDMA */
0448
0449 #endif /* _LINUX_DMA_MAP_OPS_H */