Back to home page

LXR

 
 

    


0001 /*
0002  * Copyright (C) 2008 Advanced Micro Devices, Inc.
0003  *
0004  * Author: Joerg Roedel <joerg.roedel@amd.com>
0005  *
0006  * This program is free software; you can redistribute it and/or modify it
0007  * under the terms of the GNU General Public License version 2 as published
0008  * by the Free Software Foundation.
0009  *
0010  * This program is distributed in the hope that it will be useful,
0011  * but WITHOUT ANY WARRANTY; without even the implied warranty of
0012  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
0013  * GNU General Public License for more details.
0014  *
0015  * You should have received a copy of the GNU General Public License
0016  * along with this program; if not, write to the Free Software
0017  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
0018  */
0019 
0020 #include <linux/scatterlist.h>
0021 #include <linux/dma-mapping.h>
0022 #include <linux/stacktrace.h>
0023 #include <linux/dma-debug.h>
0024 #include <linux/spinlock.h>
0025 #include <linux/vmalloc.h>
0026 #include <linux/debugfs.h>
0027 #include <linux/uaccess.h>
0028 #include <linux/export.h>
0029 #include <linux/device.h>
0030 #include <linux/types.h>
0031 #include <linux/sched.h>
0032 #include <linux/ctype.h>
0033 #include <linux/list.h>
0034 #include <linux/slab.h>
0035 
0036 #include <asm/sections.h>
0037 
0038 #define HASH_SIZE       1024ULL
0039 #define HASH_FN_SHIFT   13
0040 #define HASH_FN_MASK    (HASH_SIZE - 1)
0041 
0042 enum {
0043     dma_debug_single,
0044     dma_debug_page,
0045     dma_debug_sg,
0046     dma_debug_coherent,
0047     dma_debug_resource,
0048 };
0049 
0050 enum map_err_types {
0051     MAP_ERR_CHECK_NOT_APPLICABLE,
0052     MAP_ERR_NOT_CHECKED,
0053     MAP_ERR_CHECKED,
0054 };
0055 
0056 #define DMA_DEBUG_STACKTRACE_ENTRIES 5
0057 
0058 /**
0059  * struct dma_debug_entry - track a dma_map* or dma_alloc_coherent mapping
0060  * @list: node on pre-allocated free_entries list
0061  * @dev: 'dev' argument to dma_map_{page|single|sg} or dma_alloc_coherent
0062  * @type: single, page, sg, coherent
0063  * @pfn: page frame of the start address
0064  * @offset: offset of mapping relative to pfn
0065  * @size: length of the mapping
0066  * @direction: enum dma_data_direction
0067  * @sg_call_ents: 'nents' from dma_map_sg
0068  * @sg_mapped_ents: 'mapped_ents' from dma_map_sg
0069  * @map_err_type: track whether dma_mapping_error() was checked
0070  * @stacktrace: support backtraces when a violation is detected
0071  */
0072 struct dma_debug_entry {
0073     struct list_head list;
0074     struct device    *dev;
0075     int              type;
0076     unsigned long    pfn;
0077     size_t       offset;
0078     u64              dev_addr;
0079     u64              size;
0080     int              direction;
0081     int      sg_call_ents;
0082     int      sg_mapped_ents;
0083     enum map_err_types  map_err_type;
0084 #ifdef CONFIG_STACKTRACE
0085     struct       stack_trace stacktrace;
0086     unsigned long    st_entries[DMA_DEBUG_STACKTRACE_ENTRIES];
0087 #endif
0088 };
0089 
0090 typedef bool (*match_fn)(struct dma_debug_entry *, struct dma_debug_entry *);
0091 
0092 struct hash_bucket {
0093     struct list_head list;
0094     spinlock_t lock;
0095 } ____cacheline_aligned_in_smp;
0096 
0097 /* Hash list to save the allocated dma addresses */
0098 static struct hash_bucket dma_entry_hash[HASH_SIZE];
0099 /* List of pre-allocated dma_debug_entry's */
0100 static LIST_HEAD(free_entries);
0101 /* Lock for the list above */
0102 static DEFINE_SPINLOCK(free_entries_lock);
0103 
0104 /* Global disable flag - will be set in case of an error */
0105 static bool global_disable __read_mostly;
0106 
0107 /* Early initialization disable flag, set at the end of dma_debug_init */
0108 static bool dma_debug_initialized __read_mostly;
0109 
0110 static inline bool dma_debug_disabled(void)
0111 {
0112     return global_disable || !dma_debug_initialized;
0113 }
0114 
0115 /* Global error count */
0116 static u32 error_count;
0117 
0118 /* Global error show enable*/
0119 static u32 show_all_errors __read_mostly;
0120 /* Number of errors to show */
0121 static u32 show_num_errors = 1;
0122 
0123 static u32 num_free_entries;
0124 static u32 min_free_entries;
0125 static u32 nr_total_entries;
0126 
0127 /* number of preallocated entries requested by kernel cmdline */
0128 static u32 req_entries;
0129 
0130 /* debugfs dentry's for the stuff above */
0131 static struct dentry *dma_debug_dent        __read_mostly;
0132 static struct dentry *global_disable_dent   __read_mostly;
0133 static struct dentry *error_count_dent      __read_mostly;
0134 static struct dentry *show_all_errors_dent  __read_mostly;
0135 static struct dentry *show_num_errors_dent  __read_mostly;
0136 static struct dentry *num_free_entries_dent __read_mostly;
0137 static struct dentry *min_free_entries_dent __read_mostly;
0138 static struct dentry *filter_dent           __read_mostly;
0139 
0140 /* per-driver filter related state */
0141 
0142 #define NAME_MAX_LEN    64
0143 
0144 static char                  current_driver_name[NAME_MAX_LEN] __read_mostly;
0145 static struct device_driver *current_driver                    __read_mostly;
0146 
0147 static DEFINE_RWLOCK(driver_name_lock);
0148 
0149 static const char *const maperr2str[] = {
0150     [MAP_ERR_CHECK_NOT_APPLICABLE] = "dma map error check not applicable",
0151     [MAP_ERR_NOT_CHECKED] = "dma map error not checked",
0152     [MAP_ERR_CHECKED] = "dma map error checked",
0153 };
0154 
0155 static const char *type2name[5] = { "single", "page",
0156                     "scather-gather", "coherent",
0157                     "resource" };
0158 
0159 static const char *dir2name[4] = { "DMA_BIDIRECTIONAL", "DMA_TO_DEVICE",
0160                    "DMA_FROM_DEVICE", "DMA_NONE" };
0161 
0162 /*
0163  * The access to some variables in this macro is racy. We can't use atomic_t
0164  * here because all these variables are exported to debugfs. Some of them even
0165  * writeable. This is also the reason why a lock won't help much. But anyway,
0166  * the races are no big deal. Here is why:
0167  *
0168  *   error_count: the addition is racy, but the worst thing that can happen is
0169  *                that we don't count some errors
0170  *   show_num_errors: the subtraction is racy. Also no big deal because in
0171  *                    worst case this will result in one warning more in the
0172  *                    system log than the user configured. This variable is
0173  *                    writeable via debugfs.
0174  */
0175 static inline void dump_entry_trace(struct dma_debug_entry *entry)
0176 {
0177 #ifdef CONFIG_STACKTRACE
0178     if (entry) {
0179         pr_warning("Mapped at:\n");
0180         print_stack_trace(&entry->stacktrace, 0);
0181     }
0182 #endif
0183 }
0184 
0185 static bool driver_filter(struct device *dev)
0186 {
0187     struct device_driver *drv;
0188     unsigned long flags;
0189     bool ret;
0190 
0191     /* driver filter off */
0192     if (likely(!current_driver_name[0]))
0193         return true;
0194 
0195     /* driver filter on and initialized */
0196     if (current_driver && dev && dev->driver == current_driver)
0197         return true;
0198 
0199     /* driver filter on, but we can't filter on a NULL device... */
0200     if (!dev)
0201         return false;
0202 
0203     if (current_driver || !current_driver_name[0])
0204         return false;
0205 
0206     /* driver filter on but not yet initialized */
0207     drv = dev->driver;
0208     if (!drv)
0209         return false;
0210 
0211     /* lock to protect against change of current_driver_name */
0212     read_lock_irqsave(&driver_name_lock, flags);
0213 
0214     ret = false;
0215     if (drv->name &&
0216         strncmp(current_driver_name, drv->name, NAME_MAX_LEN - 1) == 0) {
0217         current_driver = drv;
0218         ret = true;
0219     }
0220 
0221     read_unlock_irqrestore(&driver_name_lock, flags);
0222 
0223     return ret;
0224 }
0225 
0226 #define err_printk(dev, entry, format, arg...) do {         \
0227         error_count += 1;                   \
0228         if (driver_filter(dev) &&               \
0229             (show_all_errors || show_num_errors > 0)) {     \
0230             WARN(1, "%s %s: " format,           \
0231                  dev ? dev_driver_string(dev) : "NULL", \
0232                  dev ? dev_name(dev) : "NULL", ## arg); \
0233             dump_entry_trace(entry);            \
0234         }                           \
0235         if (!show_all_errors && show_num_errors > 0)        \
0236             show_num_errors -= 1;               \
0237     } while (0);
0238 
0239 /*
0240  * Hash related functions
0241  *
0242  * Every DMA-API request is saved into a struct dma_debug_entry. To
0243  * have quick access to these structs they are stored into a hash.
0244  */
0245 static int hash_fn(struct dma_debug_entry *entry)
0246 {
0247     /*
0248      * Hash function is based on the dma address.
0249      * We use bits 20-27 here as the index into the hash
0250      */
0251     return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK;
0252 }
0253 
0254 /*
0255  * Request exclusive access to a hash bucket for a given dma_debug_entry.
0256  */
0257 static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry,
0258                        unsigned long *flags)
0259     __acquires(&dma_entry_hash[idx].lock)
0260 {
0261     int idx = hash_fn(entry);
0262     unsigned long __flags;
0263 
0264     spin_lock_irqsave(&dma_entry_hash[idx].lock, __flags);
0265     *flags = __flags;
0266     return &dma_entry_hash[idx];
0267 }
0268 
0269 /*
0270  * Give up exclusive access to the hash bucket
0271  */
0272 static void put_hash_bucket(struct hash_bucket *bucket,
0273                 unsigned long *flags)
0274     __releases(&bucket->lock)
0275 {
0276     unsigned long __flags = *flags;
0277 
0278     spin_unlock_irqrestore(&bucket->lock, __flags);
0279 }
0280 
0281 static bool exact_match(struct dma_debug_entry *a, struct dma_debug_entry *b)
0282 {
0283     return ((a->dev_addr == b->dev_addr) &&
0284         (a->dev == b->dev)) ? true : false;
0285 }
0286 
0287 static bool containing_match(struct dma_debug_entry *a,
0288                  struct dma_debug_entry *b)
0289 {
0290     if (a->dev != b->dev)
0291         return false;
0292 
0293     if ((b->dev_addr <= a->dev_addr) &&
0294         ((b->dev_addr + b->size) >= (a->dev_addr + a->size)))
0295         return true;
0296 
0297     return false;
0298 }
0299 
0300 /*
0301  * Search a given entry in the hash bucket list
0302  */
0303 static struct dma_debug_entry *__hash_bucket_find(struct hash_bucket *bucket,
0304                           struct dma_debug_entry *ref,
0305                           match_fn match)
0306 {
0307     struct dma_debug_entry *entry, *ret = NULL;
0308     int matches = 0, match_lvl, last_lvl = -1;
0309 
0310     list_for_each_entry(entry, &bucket->list, list) {
0311         if (!match(ref, entry))
0312             continue;
0313 
0314         /*
0315          * Some drivers map the same physical address multiple
0316          * times. Without a hardware IOMMU this results in the
0317          * same device addresses being put into the dma-debug
0318          * hash multiple times too. This can result in false
0319          * positives being reported. Therefore we implement a
0320          * best-fit algorithm here which returns the entry from
0321          * the hash which fits best to the reference value
0322          * instead of the first-fit.
0323          */
0324         matches += 1;
0325         match_lvl = 0;
0326         entry->size         == ref->size         ? ++match_lvl : 0;
0327         entry->type         == ref->type         ? ++match_lvl : 0;
0328         entry->direction    == ref->direction    ? ++match_lvl : 0;
0329         entry->sg_call_ents == ref->sg_call_ents ? ++match_lvl : 0;
0330 
0331         if (match_lvl == 4) {
0332             /* perfect-fit - return the result */
0333             return entry;
0334         } else if (match_lvl > last_lvl) {
0335             /*
0336              * We found an entry that fits better then the
0337              * previous one or it is the 1st match.
0338              */
0339             last_lvl = match_lvl;
0340             ret      = entry;
0341         }
0342     }
0343 
0344     /*
0345      * If we have multiple matches but no perfect-fit, just return
0346      * NULL.
0347      */
0348     ret = (matches == 1) ? ret : NULL;
0349 
0350     return ret;
0351 }
0352 
0353 static struct dma_debug_entry *bucket_find_exact(struct hash_bucket *bucket,
0354                          struct dma_debug_entry *ref)
0355 {
0356     return __hash_bucket_find(bucket, ref, exact_match);
0357 }
0358 
0359 static struct dma_debug_entry *bucket_find_contain(struct hash_bucket **bucket,
0360                            struct dma_debug_entry *ref,
0361                            unsigned long *flags)
0362 {
0363 
0364     unsigned int max_range = dma_get_max_seg_size(ref->dev);
0365     struct dma_debug_entry *entry, index = *ref;
0366     unsigned int range = 0;
0367 
0368     while (range <= max_range) {
0369         entry = __hash_bucket_find(*bucket, ref, containing_match);
0370 
0371         if (entry)
0372             return entry;
0373 
0374         /*
0375          * Nothing found, go back a hash bucket
0376          */
0377         put_hash_bucket(*bucket, flags);
0378         range          += (1 << HASH_FN_SHIFT);
0379         index.dev_addr -= (1 << HASH_FN_SHIFT);
0380         *bucket = get_hash_bucket(&index, flags);
0381     }
0382 
0383     return NULL;
0384 }
0385 
0386 /*
0387  * Add an entry to a hash bucket
0388  */
0389 static void hash_bucket_add(struct hash_bucket *bucket,
0390                 struct dma_debug_entry *entry)
0391 {
0392     list_add_tail(&entry->list, &bucket->list);
0393 }
0394 
0395 /*
0396  * Remove entry from a hash bucket list
0397  */
0398 static void hash_bucket_del(struct dma_debug_entry *entry)
0399 {
0400     list_del(&entry->list);
0401 }
0402 
0403 static unsigned long long phys_addr(struct dma_debug_entry *entry)
0404 {
0405     if (entry->type == dma_debug_resource)
0406         return __pfn_to_phys(entry->pfn) + entry->offset;
0407 
0408     return page_to_phys(pfn_to_page(entry->pfn)) + entry->offset;
0409 }
0410 
0411 /*
0412  * Dump mapping entries for debugging purposes
0413  */
0414 void debug_dma_dump_mappings(struct device *dev)
0415 {
0416     int idx;
0417 
0418     for (idx = 0; idx < HASH_SIZE; idx++) {
0419         struct hash_bucket *bucket = &dma_entry_hash[idx];
0420         struct dma_debug_entry *entry;
0421         unsigned long flags;
0422 
0423         spin_lock_irqsave(&bucket->lock, flags);
0424 
0425         list_for_each_entry(entry, &bucket->list, list) {
0426             if (!dev || dev == entry->dev) {
0427                 dev_info(entry->dev,
0428                      "%s idx %d P=%Lx N=%lx D=%Lx L=%Lx %s %s\n",
0429                      type2name[entry->type], idx,
0430                      phys_addr(entry), entry->pfn,
0431                      entry->dev_addr, entry->size,
0432                      dir2name[entry->direction],
0433                      maperr2str[entry->map_err_type]);
0434             }
0435         }
0436 
0437         spin_unlock_irqrestore(&bucket->lock, flags);
0438     }
0439 }
0440 EXPORT_SYMBOL(debug_dma_dump_mappings);
0441 
0442 /*
0443  * For each mapping (initial cacheline in the case of
0444  * dma_alloc_coherent/dma_map_page, initial cacheline in each page of a
0445  * scatterlist, or the cacheline specified in dma_map_single) insert
0446  * into this tree using the cacheline as the key. At
0447  * dma_unmap_{single|sg|page} or dma_free_coherent delete the entry.  If
0448  * the entry already exists at insertion time add a tag as a reference
0449  * count for the overlapping mappings.  For now, the overlap tracking
0450  * just ensures that 'unmaps' balance 'maps' before marking the
0451  * cacheline idle, but we should also be flagging overlaps as an API
0452  * violation.
0453  *
0454  * Memory usage is mostly constrained by the maximum number of available
0455  * dma-debug entries in that we need a free dma_debug_entry before
0456  * inserting into the tree.  In the case of dma_map_page and
0457  * dma_alloc_coherent there is only one dma_debug_entry and one
0458  * dma_active_cacheline entry to track per event.  dma_map_sg(), on the
0459  * other hand, consumes a single dma_debug_entry, but inserts 'nents'
0460  * entries into the tree.
0461  *
0462  * At any time debug_dma_assert_idle() can be called to trigger a
0463  * warning if any cachelines in the given page are in the active set.
0464  */
0465 static RADIX_TREE(dma_active_cacheline, GFP_NOWAIT);
0466 static DEFINE_SPINLOCK(radix_lock);
0467 #define ACTIVE_CACHELINE_MAX_OVERLAP ((1 << RADIX_TREE_MAX_TAGS) - 1)
0468 #define CACHELINE_PER_PAGE_SHIFT (PAGE_SHIFT - L1_CACHE_SHIFT)
0469 #define CACHELINES_PER_PAGE (1 << CACHELINE_PER_PAGE_SHIFT)
0470 
0471 static phys_addr_t to_cacheline_number(struct dma_debug_entry *entry)
0472 {
0473     return (entry->pfn << CACHELINE_PER_PAGE_SHIFT) +
0474         (entry->offset >> L1_CACHE_SHIFT);
0475 }
0476 
0477 static int active_cacheline_read_overlap(phys_addr_t cln)
0478 {
0479     int overlap = 0, i;
0480 
0481     for (i = RADIX_TREE_MAX_TAGS - 1; i >= 0; i--)
0482         if (radix_tree_tag_get(&dma_active_cacheline, cln, i))
0483             overlap |= 1 << i;
0484     return overlap;
0485 }
0486 
0487 static int active_cacheline_set_overlap(phys_addr_t cln, int overlap)
0488 {
0489     int i;
0490 
0491     if (overlap > ACTIVE_CACHELINE_MAX_OVERLAP || overlap < 0)
0492         return overlap;
0493 
0494     for (i = RADIX_TREE_MAX_TAGS - 1; i >= 0; i--)
0495         if (overlap & 1 << i)
0496             radix_tree_tag_set(&dma_active_cacheline, cln, i);
0497         else
0498             radix_tree_tag_clear(&dma_active_cacheline, cln, i);
0499 
0500     return overlap;
0501 }
0502 
0503 static void active_cacheline_inc_overlap(phys_addr_t cln)
0504 {
0505     int overlap = active_cacheline_read_overlap(cln);
0506 
0507     overlap = active_cacheline_set_overlap(cln, ++overlap);
0508 
0509     /* If we overflowed the overlap counter then we're potentially
0510      * leaking dma-mappings.  Otherwise, if maps and unmaps are
0511      * balanced then this overflow may cause false negatives in
0512      * debug_dma_assert_idle() as the cacheline may be marked idle
0513      * prematurely.
0514      */
0515     WARN_ONCE(overlap > ACTIVE_CACHELINE_MAX_OVERLAP,
0516           "DMA-API: exceeded %d overlapping mappings of cacheline %pa\n",
0517           ACTIVE_CACHELINE_MAX_OVERLAP, &cln);
0518 }
0519 
0520 static int active_cacheline_dec_overlap(phys_addr_t cln)
0521 {
0522     int overlap = active_cacheline_read_overlap(cln);
0523 
0524     return active_cacheline_set_overlap(cln, --overlap);
0525 }
0526 
0527 static int active_cacheline_insert(struct dma_debug_entry *entry)
0528 {
0529     phys_addr_t cln = to_cacheline_number(entry);
0530     unsigned long flags;
0531     int rc;
0532 
0533     /* If the device is not writing memory then we don't have any
0534      * concerns about the cpu consuming stale data.  This mitigates
0535      * legitimate usages of overlapping mappings.
0536      */
0537     if (entry->direction == DMA_TO_DEVICE)
0538         return 0;
0539 
0540     spin_lock_irqsave(&radix_lock, flags);
0541     rc = radix_tree_insert(&dma_active_cacheline, cln, entry);
0542     if (rc == -EEXIST)
0543         active_cacheline_inc_overlap(cln);
0544     spin_unlock_irqrestore(&radix_lock, flags);
0545 
0546     return rc;
0547 }
0548 
0549 static void active_cacheline_remove(struct dma_debug_entry *entry)
0550 {
0551     phys_addr_t cln = to_cacheline_number(entry);
0552     unsigned long flags;
0553 
0554     /* ...mirror the insert case */
0555     if (entry->direction == DMA_TO_DEVICE)
0556         return;
0557 
0558     spin_lock_irqsave(&radix_lock, flags);
0559     /* since we are counting overlaps the final put of the
0560      * cacheline will occur when the overlap count is 0.
0561      * active_cacheline_dec_overlap() returns -1 in that case
0562      */
0563     if (active_cacheline_dec_overlap(cln) < 0)
0564         radix_tree_delete(&dma_active_cacheline, cln);
0565     spin_unlock_irqrestore(&radix_lock, flags);
0566 }
0567 
0568 /**
0569  * debug_dma_assert_idle() - assert that a page is not undergoing dma
0570  * @page: page to lookup in the dma_active_cacheline tree
0571  *
0572  * Place a call to this routine in cases where the cpu touching the page
0573  * before the dma completes (page is dma_unmapped) will lead to data
0574  * corruption.
0575  */
0576 void debug_dma_assert_idle(struct page *page)
0577 {
0578     static struct dma_debug_entry *ents[CACHELINES_PER_PAGE];
0579     struct dma_debug_entry *entry = NULL;
0580     void **results = (void **) &ents;
0581     unsigned int nents, i;
0582     unsigned long flags;
0583     phys_addr_t cln;
0584 
0585     if (dma_debug_disabled())
0586         return;
0587 
0588     if (!page)
0589         return;
0590 
0591     cln = (phys_addr_t) page_to_pfn(page) << CACHELINE_PER_PAGE_SHIFT;
0592     spin_lock_irqsave(&radix_lock, flags);
0593     nents = radix_tree_gang_lookup(&dma_active_cacheline, results, cln,
0594                        CACHELINES_PER_PAGE);
0595     for (i = 0; i < nents; i++) {
0596         phys_addr_t ent_cln = to_cacheline_number(ents[i]);
0597 
0598         if (ent_cln == cln) {
0599             entry = ents[i];
0600             break;
0601         } else if (ent_cln >= cln + CACHELINES_PER_PAGE)
0602             break;
0603     }
0604     spin_unlock_irqrestore(&radix_lock, flags);
0605 
0606     if (!entry)
0607         return;
0608 
0609     cln = to_cacheline_number(entry);
0610     err_printk(entry->dev, entry,
0611            "DMA-API: cpu touching an active dma mapped cacheline [cln=%pa]\n",
0612            &cln);
0613 }
0614 
0615 /*
0616  * Wrapper function for adding an entry to the hash.
0617  * This function takes care of locking itself.
0618  */
0619 static void add_dma_entry(struct dma_debug_entry *entry)
0620 {
0621     struct hash_bucket *bucket;
0622     unsigned long flags;
0623     int rc;
0624 
0625     bucket = get_hash_bucket(entry, &flags);
0626     hash_bucket_add(bucket, entry);
0627     put_hash_bucket(bucket, &flags);
0628 
0629     rc = active_cacheline_insert(entry);
0630     if (rc == -ENOMEM) {
0631         pr_err("DMA-API: cacheline tracking ENOMEM, dma-debug disabled\n");
0632         global_disable = true;
0633     }
0634 
0635     /* TODO: report -EEXIST errors here as overlapping mappings are
0636      * not supported by the DMA API
0637      */
0638 }
0639 
0640 static struct dma_debug_entry *__dma_entry_alloc(void)
0641 {
0642     struct dma_debug_entry *entry;
0643 
0644     entry = list_entry(free_entries.next, struct dma_debug_entry, list);
0645     list_del(&entry->list);
0646     memset(entry, 0, sizeof(*entry));
0647 
0648     num_free_entries -= 1;
0649     if (num_free_entries < min_free_entries)
0650         min_free_entries = num_free_entries;
0651 
0652     return entry;
0653 }
0654 
0655 /* struct dma_entry allocator
0656  *
0657  * The next two functions implement the allocator for
0658  * struct dma_debug_entries.
0659  */
0660 static struct dma_debug_entry *dma_entry_alloc(void)
0661 {
0662     struct dma_debug_entry *entry;
0663     unsigned long flags;
0664 
0665     spin_lock_irqsave(&free_entries_lock, flags);
0666 
0667     if (list_empty(&free_entries)) {
0668         global_disable = true;
0669         spin_unlock_irqrestore(&free_entries_lock, flags);
0670         pr_err("DMA-API: debugging out of memory - disabling\n");
0671         return NULL;
0672     }
0673 
0674     entry = __dma_entry_alloc();
0675 
0676     spin_unlock_irqrestore(&free_entries_lock, flags);
0677 
0678 #ifdef CONFIG_STACKTRACE
0679     entry->stacktrace.max_entries = DMA_DEBUG_STACKTRACE_ENTRIES;
0680     entry->stacktrace.entries = entry->st_entries;
0681     entry->stacktrace.skip = 2;
0682     save_stack_trace(&entry->stacktrace);
0683 #endif
0684 
0685     return entry;
0686 }
0687 
0688 static void dma_entry_free(struct dma_debug_entry *entry)
0689 {
0690     unsigned long flags;
0691 
0692     active_cacheline_remove(entry);
0693 
0694     /*
0695      * add to beginning of the list - this way the entries are
0696      * more likely cache hot when they are reallocated.
0697      */
0698     spin_lock_irqsave(&free_entries_lock, flags);
0699     list_add(&entry->list, &free_entries);
0700     num_free_entries += 1;
0701     spin_unlock_irqrestore(&free_entries_lock, flags);
0702 }
0703 
0704 int dma_debug_resize_entries(u32 num_entries)
0705 {
0706     int i, delta, ret = 0;
0707     unsigned long flags;
0708     struct dma_debug_entry *entry;
0709     LIST_HEAD(tmp);
0710 
0711     spin_lock_irqsave(&free_entries_lock, flags);
0712 
0713     if (nr_total_entries < num_entries) {
0714         delta = num_entries - nr_total_entries;
0715 
0716         spin_unlock_irqrestore(&free_entries_lock, flags);
0717 
0718         for (i = 0; i < delta; i++) {
0719             entry = kzalloc(sizeof(*entry), GFP_KERNEL);
0720             if (!entry)
0721                 break;
0722 
0723             list_add_tail(&entry->list, &tmp);
0724         }
0725 
0726         spin_lock_irqsave(&free_entries_lock, flags);
0727 
0728         list_splice(&tmp, &free_entries);
0729         nr_total_entries += i;
0730         num_free_entries += i;
0731     } else {
0732         delta = nr_total_entries - num_entries;
0733 
0734         for (i = 0; i < delta && !list_empty(&free_entries); i++) {
0735             entry = __dma_entry_alloc();
0736             kfree(entry);
0737         }
0738 
0739         nr_total_entries -= i;
0740     }
0741 
0742     if (nr_total_entries != num_entries)
0743         ret = 1;
0744 
0745     spin_unlock_irqrestore(&free_entries_lock, flags);
0746 
0747     return ret;
0748 }
0749 EXPORT_SYMBOL(dma_debug_resize_entries);
0750 
0751 /*
0752  * DMA-API debugging init code
0753  *
0754  * The init code does two things:
0755  *   1. Initialize core data structures
0756  *   2. Preallocate a given number of dma_debug_entry structs
0757  */
0758 
0759 static int prealloc_memory(u32 num_entries)
0760 {
0761     struct dma_debug_entry *entry, *next_entry;
0762     int i;
0763 
0764     for (i = 0; i < num_entries; ++i) {
0765         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
0766         if (!entry)
0767             goto out_err;
0768 
0769         list_add_tail(&entry->list, &free_entries);
0770     }
0771 
0772     num_free_entries = num_entries;
0773     min_free_entries = num_entries;
0774 
0775     pr_info("DMA-API: preallocated %d debug entries\n", num_entries);
0776 
0777     return 0;
0778 
0779 out_err:
0780 
0781     list_for_each_entry_safe(entry, next_entry, &free_entries, list) {
0782         list_del(&entry->list);
0783         kfree(entry);
0784     }
0785 
0786     return -ENOMEM;
0787 }
0788 
0789 static ssize_t filter_read(struct file *file, char __user *user_buf,
0790                size_t count, loff_t *ppos)
0791 {
0792     char buf[NAME_MAX_LEN + 1];
0793     unsigned long flags;
0794     int len;
0795 
0796     if (!current_driver_name[0])
0797         return 0;
0798 
0799     /*
0800      * We can't copy to userspace directly because current_driver_name can
0801      * only be read under the driver_name_lock with irqs disabled. So
0802      * create a temporary copy first.
0803      */
0804     read_lock_irqsave(&driver_name_lock, flags);
0805     len = scnprintf(buf, NAME_MAX_LEN + 1, "%s\n", current_driver_name);
0806     read_unlock_irqrestore(&driver_name_lock, flags);
0807 
0808     return simple_read_from_buffer(user_buf, count, ppos, buf, len);
0809 }
0810 
0811 static ssize_t filter_write(struct file *file, const char __user *userbuf,
0812                 size_t count, loff_t *ppos)
0813 {
0814     char buf[NAME_MAX_LEN];
0815     unsigned long flags;
0816     size_t len;
0817     int i;
0818 
0819     /*
0820      * We can't copy from userspace directly. Access to
0821      * current_driver_name is protected with a write_lock with irqs
0822      * disabled. Since copy_from_user can fault and may sleep we
0823      * need to copy to temporary buffer first
0824      */
0825     len = min(count, (size_t)(NAME_MAX_LEN - 1));
0826     if (copy_from_user(buf, userbuf, len))
0827         return -EFAULT;
0828 
0829     buf[len] = 0;
0830 
0831     write_lock_irqsave(&driver_name_lock, flags);
0832 
0833     /*
0834      * Now handle the string we got from userspace very carefully.
0835      * The rules are:
0836      *         - only use the first token we got
0837      *         - token delimiter is everything looking like a space
0838      *           character (' ', '\n', '\t' ...)
0839      *
0840      */
0841     if (!isalnum(buf[0])) {
0842         /*
0843          * If the first character userspace gave us is not
0844          * alphanumerical then assume the filter should be
0845          * switched off.
0846          */
0847         if (current_driver_name[0])
0848             pr_info("DMA-API: switching off dma-debug driver filter\n");
0849         current_driver_name[0] = 0;
0850         current_driver = NULL;
0851         goto out_unlock;
0852     }
0853 
0854     /*
0855      * Now parse out the first token and use it as the name for the
0856      * driver to filter for.
0857      */
0858     for (i = 0; i < NAME_MAX_LEN - 1; ++i) {
0859         current_driver_name[i] = buf[i];
0860         if (isspace(buf[i]) || buf[i] == ' ' || buf[i] == 0)
0861             break;
0862     }
0863     current_driver_name[i] = 0;
0864     current_driver = NULL;
0865 
0866     pr_info("DMA-API: enable driver filter for driver [%s]\n",
0867         current_driver_name);
0868 
0869 out_unlock:
0870     write_unlock_irqrestore(&driver_name_lock, flags);
0871 
0872     return count;
0873 }
0874 
0875 static const struct file_operations filter_fops = {
0876     .read  = filter_read,
0877     .write = filter_write,
0878     .llseek = default_llseek,
0879 };
0880 
0881 static int dma_debug_fs_init(void)
0882 {
0883     dma_debug_dent = debugfs_create_dir("dma-api", NULL);
0884     if (!dma_debug_dent) {
0885         pr_err("DMA-API: can not create debugfs directory\n");
0886         return -ENOMEM;
0887     }
0888 
0889     global_disable_dent = debugfs_create_bool("disabled", 0444,
0890             dma_debug_dent,
0891             &global_disable);
0892     if (!global_disable_dent)
0893         goto out_err;
0894 
0895     error_count_dent = debugfs_create_u32("error_count", 0444,
0896             dma_debug_dent, &error_count);
0897     if (!error_count_dent)
0898         goto out_err;
0899 
0900     show_all_errors_dent = debugfs_create_u32("all_errors", 0644,
0901             dma_debug_dent,
0902             &show_all_errors);
0903     if (!show_all_errors_dent)
0904         goto out_err;
0905 
0906     show_num_errors_dent = debugfs_create_u32("num_errors", 0644,
0907             dma_debug_dent,
0908             &show_num_errors);
0909     if (!show_num_errors_dent)
0910         goto out_err;
0911 
0912     num_free_entries_dent = debugfs_create_u32("num_free_entries", 0444,
0913             dma_debug_dent,
0914             &num_free_entries);
0915     if (!num_free_entries_dent)
0916         goto out_err;
0917 
0918     min_free_entries_dent = debugfs_create_u32("min_free_entries", 0444,
0919             dma_debug_dent,
0920             &min_free_entries);
0921     if (!min_free_entries_dent)
0922         goto out_err;
0923 
0924     filter_dent = debugfs_create_file("driver_filter", 0644,
0925                       dma_debug_dent, NULL, &filter_fops);
0926     if (!filter_dent)
0927         goto out_err;
0928 
0929     return 0;
0930 
0931 out_err:
0932     debugfs_remove_recursive(dma_debug_dent);
0933 
0934     return -ENOMEM;
0935 }
0936 
0937 static int device_dma_allocations(struct device *dev, struct dma_debug_entry **out_entry)
0938 {
0939     struct dma_debug_entry *entry;
0940     unsigned long flags;
0941     int count = 0, i;
0942 
0943     local_irq_save(flags);
0944 
0945     for (i = 0; i < HASH_SIZE; ++i) {
0946         spin_lock(&dma_entry_hash[i].lock);
0947         list_for_each_entry(entry, &dma_entry_hash[i].list, list) {
0948             if (entry->dev == dev) {
0949                 count += 1;
0950                 *out_entry = entry;
0951             }
0952         }
0953         spin_unlock(&dma_entry_hash[i].lock);
0954     }
0955 
0956     local_irq_restore(flags);
0957 
0958     return count;
0959 }
0960 
0961 static int dma_debug_device_change(struct notifier_block *nb, unsigned long action, void *data)
0962 {
0963     struct device *dev = data;
0964     struct dma_debug_entry *uninitialized_var(entry);
0965     int count;
0966 
0967     if (dma_debug_disabled())
0968         return 0;
0969 
0970     switch (action) {
0971     case BUS_NOTIFY_UNBOUND_DRIVER:
0972         count = device_dma_allocations(dev, &entry);
0973         if (count == 0)
0974             break;
0975         err_printk(dev, entry, "DMA-API: device driver has pending "
0976                 "DMA allocations while released from device "
0977                 "[count=%d]\n"
0978                 "One of leaked entries details: "
0979                 "[device address=0x%016llx] [size=%llu bytes] "
0980                 "[mapped with %s] [mapped as %s]\n",
0981             count, entry->dev_addr, entry->size,
0982             dir2name[entry->direction], type2name[entry->type]);
0983         break;
0984     default:
0985         break;
0986     }
0987 
0988     return 0;
0989 }
0990 
0991 void dma_debug_add_bus(struct bus_type *bus)
0992 {
0993     struct notifier_block *nb;
0994 
0995     if (dma_debug_disabled())
0996         return;
0997 
0998     nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL);
0999     if (nb == NULL) {
1000         pr_err("dma_debug_add_bus: out of memory\n");
1001         return;
1002     }
1003 
1004     nb->notifier_call = dma_debug_device_change;
1005 
1006     bus_register_notifier(bus, nb);
1007 }
1008 
1009 /*
1010  * Let the architectures decide how many entries should be preallocated.
1011  */
1012 void dma_debug_init(u32 num_entries)
1013 {
1014     int i;
1015 
1016     /* Do not use dma_debug_initialized here, since we really want to be
1017      * called to set dma_debug_initialized
1018      */
1019     if (global_disable)
1020         return;
1021 
1022     for (i = 0; i < HASH_SIZE; ++i) {
1023         INIT_LIST_HEAD(&dma_entry_hash[i].list);
1024         spin_lock_init(&dma_entry_hash[i].lock);
1025     }
1026 
1027     if (dma_debug_fs_init() != 0) {
1028         pr_err("DMA-API: error creating debugfs entries - disabling\n");
1029         global_disable = true;
1030 
1031         return;
1032     }
1033 
1034     if (req_entries)
1035         num_entries = req_entries;
1036 
1037     if (prealloc_memory(num_entries) != 0) {
1038         pr_err("DMA-API: debugging out of memory error - disabled\n");
1039         global_disable = true;
1040 
1041         return;
1042     }
1043 
1044     nr_total_entries = num_free_entries;
1045 
1046     dma_debug_initialized = true;
1047 
1048     pr_info("DMA-API: debugging enabled by kernel config\n");
1049 }
1050 
1051 static __init int dma_debug_cmdline(char *str)
1052 {
1053     if (!str)
1054         return -EINVAL;
1055 
1056     if (strncmp(str, "off", 3) == 0) {
1057         pr_info("DMA-API: debugging disabled on kernel command line\n");
1058         global_disable = true;
1059     }
1060 
1061     return 0;
1062 }
1063 
1064 static __init int dma_debug_entries_cmdline(char *str)
1065 {
1066     int res;
1067 
1068     if (!str)
1069         return -EINVAL;
1070 
1071     res = get_option(&str, &req_entries);
1072 
1073     if (!res)
1074         req_entries = 0;
1075 
1076     return 0;
1077 }
1078 
1079 __setup("dma_debug=", dma_debug_cmdline);
1080 __setup("dma_debug_entries=", dma_debug_entries_cmdline);
1081 
1082 static void check_unmap(struct dma_debug_entry *ref)
1083 {
1084     struct dma_debug_entry *entry;
1085     struct hash_bucket *bucket;
1086     unsigned long flags;
1087 
1088     bucket = get_hash_bucket(ref, &flags);
1089     entry = bucket_find_exact(bucket, ref);
1090 
1091     if (!entry) {
1092         /* must drop lock before calling dma_mapping_error */
1093         put_hash_bucket(bucket, &flags);
1094 
1095         if (dma_mapping_error(ref->dev, ref->dev_addr)) {
1096             err_printk(ref->dev, NULL,
1097                    "DMA-API: device driver tries to free an "
1098                    "invalid DMA memory address\n");
1099         } else {
1100             err_printk(ref->dev, NULL,
1101                    "DMA-API: device driver tries to free DMA "
1102                    "memory it has not allocated [device "
1103                    "address=0x%016llx] [size=%llu bytes]\n",
1104                    ref->dev_addr, ref->size);
1105         }
1106         return;
1107     }
1108 
1109     if (ref->size != entry->size) {
1110         err_printk(ref->dev, entry, "DMA-API: device driver frees "
1111                "DMA memory with different size "
1112                "[device address=0x%016llx] [map size=%llu bytes] "
1113                "[unmap size=%llu bytes]\n",
1114                ref->dev_addr, entry->size, ref->size);
1115     }
1116 
1117     if (ref->type != entry->type) {
1118         err_printk(ref->dev, entry, "DMA-API: device driver frees "
1119                "DMA memory with wrong function "
1120                "[device address=0x%016llx] [size=%llu bytes] "
1121                "[mapped as %s] [unmapped as %s]\n",
1122                ref->dev_addr, ref->size,
1123                type2name[entry->type], type2name[ref->type]);
1124     } else if ((entry->type == dma_debug_coherent) &&
1125            (phys_addr(ref) != phys_addr(entry))) {
1126         err_printk(ref->dev, entry, "DMA-API: device driver frees "
1127                "DMA memory with different CPU address "
1128                "[device address=0x%016llx] [size=%llu bytes] "
1129                "[cpu alloc address=0x%016llx] "
1130                "[cpu free address=0x%016llx]",
1131                ref->dev_addr, ref->size,
1132                phys_addr(entry),
1133                phys_addr(ref));
1134     }
1135 
1136     if (ref->sg_call_ents && ref->type == dma_debug_sg &&
1137         ref->sg_call_ents != entry->sg_call_ents) {
1138         err_printk(ref->dev, entry, "DMA-API: device driver frees "
1139                "DMA sg list with different entry count "
1140                "[map count=%d] [unmap count=%d]\n",
1141                entry->sg_call_ents, ref->sg_call_ents);
1142     }
1143 
1144     /*
1145      * This may be no bug in reality - but most implementations of the
1146      * DMA API don't handle this properly, so check for it here
1147      */
1148     if (ref->direction != entry->direction) {
1149         err_printk(ref->dev, entry, "DMA-API: device driver frees "
1150                "DMA memory with different direction "
1151                "[device address=0x%016llx] [size=%llu bytes] "
1152                "[mapped with %s] [unmapped with %s]\n",
1153                ref->dev_addr, ref->size,
1154                dir2name[entry->direction],
1155                dir2name[ref->direction]);
1156     }
1157 
1158     if (entry->map_err_type == MAP_ERR_NOT_CHECKED) {
1159         err_printk(ref->dev, entry,
1160                "DMA-API: device driver failed to check map error"
1161                "[device address=0x%016llx] [size=%llu bytes] "
1162                "[mapped as %s]",
1163                ref->dev_addr, ref->size,
1164                type2name[entry->type]);
1165     }
1166 
1167     hash_bucket_del(entry);
1168     dma_entry_free(entry);
1169 
1170     put_hash_bucket(bucket, &flags);
1171 }
1172 
1173 static void check_for_stack(struct device *dev,
1174                 struct page *page, size_t offset)
1175 {
1176     void *addr;
1177     struct vm_struct *stack_vm_area = task_stack_vm_area(current);
1178 
1179     if (!stack_vm_area) {
1180         /* Stack is direct-mapped. */
1181         if (PageHighMem(page))
1182             return;
1183         addr = page_address(page) + offset;
1184         if (object_is_on_stack(addr))
1185             err_printk(dev, NULL, "DMA-API: device driver maps memory from stack [addr=%p]\n", addr);
1186     } else {
1187         /* Stack is vmalloced. */
1188         int i;
1189 
1190         for (i = 0; i < stack_vm_area->nr_pages; i++) {
1191             if (page != stack_vm_area->pages[i])
1192                 continue;
1193 
1194             addr = (u8 *)current->stack + i * PAGE_SIZE + offset;
1195             err_printk(dev, NULL, "DMA-API: device driver maps memory from stack [probable addr=%p]\n", addr);
1196             break;
1197         }
1198     }
1199 }
1200 
1201 static inline bool overlap(void *addr, unsigned long len, void *start, void *end)
1202 {
1203     unsigned long a1 = (unsigned long)addr;
1204     unsigned long b1 = a1 + len;
1205     unsigned long a2 = (unsigned long)start;
1206     unsigned long b2 = (unsigned long)end;
1207 
1208     return !(b1 <= a2 || a1 >= b2);
1209 }
1210 
1211 static void check_for_illegal_area(struct device *dev, void *addr, unsigned long len)
1212 {
1213     if (overlap(addr, len, _stext, _etext) ||
1214         overlap(addr, len, __start_rodata, __end_rodata))
1215         err_printk(dev, NULL, "DMA-API: device driver maps memory from kernel text or rodata [addr=%p] [len=%lu]\n", addr, len);
1216 }
1217 
1218 static void check_sync(struct device *dev,
1219                struct dma_debug_entry *ref,
1220                bool to_cpu)
1221 {
1222     struct dma_debug_entry *entry;
1223     struct hash_bucket *bucket;
1224     unsigned long flags;
1225 
1226     bucket = get_hash_bucket(ref, &flags);
1227 
1228     entry = bucket_find_contain(&bucket, ref, &flags);
1229 
1230     if (!entry) {
1231         err_printk(dev, NULL, "DMA-API: device driver tries "
1232                 "to sync DMA memory it has not allocated "
1233                 "[device address=0x%016llx] [size=%llu bytes]\n",
1234                 (unsigned long long)ref->dev_addr, ref->size);
1235         goto out;
1236     }
1237 
1238     if (ref->size > entry->size) {
1239         err_printk(dev, entry, "DMA-API: device driver syncs"
1240                 " DMA memory outside allocated range "
1241                 "[device address=0x%016llx] "
1242                 "[allocation size=%llu bytes] "
1243                 "[sync offset+size=%llu]\n",
1244                 entry->dev_addr, entry->size,
1245                 ref->size);
1246     }
1247 
1248     if (entry->direction == DMA_BIDIRECTIONAL)
1249         goto out;
1250 
1251     if (ref->direction != entry->direction) {
1252         err_printk(dev, entry, "DMA-API: device driver syncs "
1253                 "DMA memory with different direction "
1254                 "[device address=0x%016llx] [size=%llu bytes] "
1255                 "[mapped with %s] [synced with %s]\n",
1256                 (unsigned long long)ref->dev_addr, entry->size,
1257                 dir2name[entry->direction],
1258                 dir2name[ref->direction]);
1259     }
1260 
1261     if (to_cpu && !(entry->direction == DMA_FROM_DEVICE) &&
1262               !(ref->direction == DMA_TO_DEVICE))
1263         err_printk(dev, entry, "DMA-API: device driver syncs "
1264                 "device read-only DMA memory for cpu "
1265                 "[device address=0x%016llx] [size=%llu bytes] "
1266                 "[mapped with %s] [synced with %s]\n",
1267                 (unsigned long long)ref->dev_addr, entry->size,
1268                 dir2name[entry->direction],
1269                 dir2name[ref->direction]);
1270 
1271     if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) &&
1272                !(ref->direction == DMA_FROM_DEVICE))
1273         err_printk(dev, entry, "DMA-API: device driver syncs "
1274                 "device write-only DMA memory to device "
1275                 "[device address=0x%016llx] [size=%llu bytes] "
1276                 "[mapped with %s] [synced with %s]\n",
1277                 (unsigned long long)ref->dev_addr, entry->size,
1278                 dir2name[entry->direction],
1279                 dir2name[ref->direction]);
1280 
1281     if (ref->sg_call_ents && ref->type == dma_debug_sg &&
1282         ref->sg_call_ents != entry->sg_call_ents) {
1283         err_printk(ref->dev, entry, "DMA-API: device driver syncs "
1284                "DMA sg list with different entry count "
1285                "[map count=%d] [sync count=%d]\n",
1286                entry->sg_call_ents, ref->sg_call_ents);
1287     }
1288 
1289 out:
1290     put_hash_bucket(bucket, &flags);
1291 }
1292 
1293 void debug_dma_map_page(struct device *dev, struct page *page, size_t offset,
1294             size_t size, int direction, dma_addr_t dma_addr,
1295             bool map_single)
1296 {
1297     struct dma_debug_entry *entry;
1298 
1299     if (unlikely(dma_debug_disabled()))
1300         return;
1301 
1302     if (dma_mapping_error(dev, dma_addr))
1303         return;
1304 
1305     entry = dma_entry_alloc();
1306     if (!entry)
1307         return;
1308 
1309     entry->dev       = dev;
1310     entry->type      = dma_debug_page;
1311     entry->pfn   = page_to_pfn(page);
1312     entry->offset    = offset,
1313     entry->dev_addr  = dma_addr;
1314     entry->size      = size;
1315     entry->direction = direction;
1316     entry->map_err_type = MAP_ERR_NOT_CHECKED;
1317 
1318     if (map_single)
1319         entry->type = dma_debug_single;
1320 
1321     check_for_stack(dev, page, offset);
1322 
1323     if (!PageHighMem(page)) {
1324         void *addr = page_address(page) + offset;
1325 
1326         check_for_illegal_area(dev, addr, size);
1327     }
1328 
1329     add_dma_entry(entry);
1330 }
1331 EXPORT_SYMBOL(debug_dma_map_page);
1332 
1333 void debug_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
1334 {
1335     struct dma_debug_entry ref;
1336     struct dma_debug_entry *entry;
1337     struct hash_bucket *bucket;
1338     unsigned long flags;
1339 
1340     if (unlikely(dma_debug_disabled()))
1341         return;
1342 
1343     ref.dev = dev;
1344     ref.dev_addr = dma_addr;
1345     bucket = get_hash_bucket(&ref, &flags);
1346 
1347     list_for_each_entry(entry, &bucket->list, list) {
1348         if (!exact_match(&ref, entry))
1349             continue;
1350 
1351         /*
1352          * The same physical address can be mapped multiple
1353          * times. Without a hardware IOMMU this results in the
1354          * same device addresses being put into the dma-debug
1355          * hash multiple times too. This can result in false
1356          * positives being reported. Therefore we implement a
1357          * best-fit algorithm here which updates the first entry
1358          * from the hash which fits the reference value and is
1359          * not currently listed as being checked.
1360          */
1361         if (entry->map_err_type == MAP_ERR_NOT_CHECKED) {
1362             entry->map_err_type = MAP_ERR_CHECKED;
1363             break;
1364         }
1365     }
1366 
1367     put_hash_bucket(bucket, &flags);
1368 }
1369 EXPORT_SYMBOL(debug_dma_mapping_error);
1370 
1371 void debug_dma_unmap_page(struct device *dev, dma_addr_t addr,
1372               size_t size, int direction, bool map_single)
1373 {
1374     struct dma_debug_entry ref = {
1375         .type           = dma_debug_page,
1376         .dev            = dev,
1377         .dev_addr       = addr,
1378         .size           = size,
1379         .direction      = direction,
1380     };
1381 
1382     if (unlikely(dma_debug_disabled()))
1383         return;
1384 
1385     if (map_single)
1386         ref.type = dma_debug_single;
1387 
1388     check_unmap(&ref);
1389 }
1390 EXPORT_SYMBOL(debug_dma_unmap_page);
1391 
1392 void debug_dma_map_sg(struct device *dev, struct scatterlist *sg,
1393               int nents, int mapped_ents, int direction)
1394 {
1395     struct dma_debug_entry *entry;
1396     struct scatterlist *s;
1397     int i;
1398 
1399     if (unlikely(dma_debug_disabled()))
1400         return;
1401 
1402     for_each_sg(sg, s, mapped_ents, i) {
1403         entry = dma_entry_alloc();
1404         if (!entry)
1405             return;
1406 
1407         entry->type           = dma_debug_sg;
1408         entry->dev            = dev;
1409         entry->pfn        = page_to_pfn(sg_page(s));
1410         entry->offset         = s->offset,
1411         entry->size           = sg_dma_len(s);
1412         entry->dev_addr       = sg_dma_address(s);
1413         entry->direction      = direction;
1414         entry->sg_call_ents   = nents;
1415         entry->sg_mapped_ents = mapped_ents;
1416 
1417         check_for_stack(dev, sg_page(s), s->offset);
1418 
1419         if (!PageHighMem(sg_page(s))) {
1420             check_for_illegal_area(dev, sg_virt(s), sg_dma_len(s));
1421         }
1422 
1423         add_dma_entry(entry);
1424     }
1425 }
1426 EXPORT_SYMBOL(debug_dma_map_sg);
1427 
1428 static int get_nr_mapped_entries(struct device *dev,
1429                  struct dma_debug_entry *ref)
1430 {
1431     struct dma_debug_entry *entry;
1432     struct hash_bucket *bucket;
1433     unsigned long flags;
1434     int mapped_ents;
1435 
1436     bucket       = get_hash_bucket(ref, &flags);
1437     entry        = bucket_find_exact(bucket, ref);
1438     mapped_ents  = 0;
1439 
1440     if (entry)
1441         mapped_ents = entry->sg_mapped_ents;
1442     put_hash_bucket(bucket, &flags);
1443 
1444     return mapped_ents;
1445 }
1446 
1447 void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
1448             int nelems, int dir)
1449 {
1450     struct scatterlist *s;
1451     int mapped_ents = 0, i;
1452 
1453     if (unlikely(dma_debug_disabled()))
1454         return;
1455 
1456     for_each_sg(sglist, s, nelems, i) {
1457 
1458         struct dma_debug_entry ref = {
1459             .type           = dma_debug_sg,
1460             .dev            = dev,
1461             .pfn        = page_to_pfn(sg_page(s)),
1462             .offset     = s->offset,
1463             .dev_addr       = sg_dma_address(s),
1464             .size           = sg_dma_len(s),
1465             .direction      = dir,
1466             .sg_call_ents   = nelems,
1467         };
1468 
1469         if (mapped_ents && i >= mapped_ents)
1470             break;
1471 
1472         if (!i)
1473             mapped_ents = get_nr_mapped_entries(dev, &ref);
1474 
1475         check_unmap(&ref);
1476     }
1477 }
1478 EXPORT_SYMBOL(debug_dma_unmap_sg);
1479 
1480 void debug_dma_alloc_coherent(struct device *dev, size_t size,
1481                   dma_addr_t dma_addr, void *virt)
1482 {
1483     struct dma_debug_entry *entry;
1484 
1485     if (unlikely(dma_debug_disabled()))
1486         return;
1487 
1488     if (unlikely(virt == NULL))
1489         return;
1490 
1491     entry = dma_entry_alloc();
1492     if (!entry)
1493         return;
1494 
1495     entry->type      = dma_debug_coherent;
1496     entry->dev       = dev;
1497     entry->pfn   = page_to_pfn(virt_to_page(virt));
1498     entry->offset    = (size_t) virt & ~PAGE_MASK;
1499     entry->size      = size;
1500     entry->dev_addr  = dma_addr;
1501     entry->direction = DMA_BIDIRECTIONAL;
1502 
1503     add_dma_entry(entry);
1504 }
1505 EXPORT_SYMBOL(debug_dma_alloc_coherent);
1506 
1507 void debug_dma_free_coherent(struct device *dev, size_t size,
1508              void *virt, dma_addr_t addr)
1509 {
1510     struct dma_debug_entry ref = {
1511         .type           = dma_debug_coherent,
1512         .dev            = dev,
1513         .pfn        = page_to_pfn(virt_to_page(virt)),
1514         .offset     = (size_t) virt & ~PAGE_MASK,
1515         .dev_addr       = addr,
1516         .size           = size,
1517         .direction      = DMA_BIDIRECTIONAL,
1518     };
1519 
1520     if (unlikely(dma_debug_disabled()))
1521         return;
1522 
1523     check_unmap(&ref);
1524 }
1525 EXPORT_SYMBOL(debug_dma_free_coherent);
1526 
1527 void debug_dma_map_resource(struct device *dev, phys_addr_t addr, size_t size,
1528                 int direction, dma_addr_t dma_addr)
1529 {
1530     struct dma_debug_entry *entry;
1531 
1532     if (unlikely(dma_debug_disabled()))
1533         return;
1534 
1535     entry = dma_entry_alloc();
1536     if (!entry)
1537         return;
1538 
1539     entry->type     = dma_debug_resource;
1540     entry->dev      = dev;
1541     entry->pfn      = PHYS_PFN(addr);
1542     entry->offset       = offset_in_page(addr);
1543     entry->size     = size;
1544     entry->dev_addr     = dma_addr;
1545     entry->direction    = direction;
1546     entry->map_err_type = MAP_ERR_NOT_CHECKED;
1547 
1548     add_dma_entry(entry);
1549 }
1550 EXPORT_SYMBOL(debug_dma_map_resource);
1551 
1552 void debug_dma_unmap_resource(struct device *dev, dma_addr_t dma_addr,
1553                   size_t size, int direction)
1554 {
1555     struct dma_debug_entry ref = {
1556         .type           = dma_debug_resource,
1557         .dev            = dev,
1558         .dev_addr       = dma_addr,
1559         .size           = size,
1560         .direction      = direction,
1561     };
1562 
1563     if (unlikely(dma_debug_disabled()))
1564         return;
1565 
1566     check_unmap(&ref);
1567 }
1568 EXPORT_SYMBOL(debug_dma_unmap_resource);
1569 
1570 void debug_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle,
1571                    size_t size, int direction)
1572 {
1573     struct dma_debug_entry ref;
1574 
1575     if (unlikely(dma_debug_disabled()))
1576         return;
1577 
1578     ref.type         = dma_debug_single;
1579     ref.dev          = dev;
1580     ref.dev_addr     = dma_handle;
1581     ref.size         = size;
1582     ref.direction    = direction;
1583     ref.sg_call_ents = 0;
1584 
1585     check_sync(dev, &ref, true);
1586 }
1587 EXPORT_SYMBOL(debug_dma_sync_single_for_cpu);
1588 
1589 void debug_dma_sync_single_for_device(struct device *dev,
1590                       dma_addr_t dma_handle, size_t size,
1591                       int direction)
1592 {
1593     struct dma_debug_entry ref;
1594 
1595     if (unlikely(dma_debug_disabled()))
1596         return;
1597 
1598     ref.type         = dma_debug_single;
1599     ref.dev          = dev;
1600     ref.dev_addr     = dma_handle;
1601     ref.size         = size;
1602     ref.direction    = direction;
1603     ref.sg_call_ents = 0;
1604 
1605     check_sync(dev, &ref, false);
1606 }
1607 EXPORT_SYMBOL(debug_dma_sync_single_for_device);
1608 
1609 void debug_dma_sync_single_range_for_cpu(struct device *dev,
1610                      dma_addr_t dma_handle,
1611                      unsigned long offset, size_t size,
1612                      int direction)
1613 {
1614     struct dma_debug_entry ref;
1615 
1616     if (unlikely(dma_debug_disabled()))
1617         return;
1618 
1619     ref.type         = dma_debug_single;
1620     ref.dev          = dev;
1621     ref.dev_addr     = dma_handle;
1622     ref.size         = offset + size;
1623     ref.direction    = direction;
1624     ref.sg_call_ents = 0;
1625 
1626     check_sync(dev, &ref, true);
1627 }
1628 EXPORT_SYMBOL(debug_dma_sync_single_range_for_cpu);
1629 
1630 void debug_dma_sync_single_range_for_device(struct device *dev,
1631                         dma_addr_t dma_handle,
1632                         unsigned long offset,
1633                         size_t size, int direction)
1634 {
1635     struct dma_debug_entry ref;
1636 
1637     if (unlikely(dma_debug_disabled()))
1638         return;
1639 
1640     ref.type         = dma_debug_single;
1641     ref.dev          = dev;
1642     ref.dev_addr     = dma_handle;
1643     ref.size         = offset + size;
1644     ref.direction    = direction;
1645     ref.sg_call_ents = 0;
1646 
1647     check_sync(dev, &ref, false);
1648 }
1649 EXPORT_SYMBOL(debug_dma_sync_single_range_for_device);
1650 
1651 void debug_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
1652                    int nelems, int direction)
1653 {
1654     struct scatterlist *s;
1655     int mapped_ents = 0, i;
1656 
1657     if (unlikely(dma_debug_disabled()))
1658         return;
1659 
1660     for_each_sg(sg, s, nelems, i) {
1661 
1662         struct dma_debug_entry ref = {
1663             .type           = dma_debug_sg,
1664             .dev            = dev,
1665             .pfn        = page_to_pfn(sg_page(s)),
1666             .offset     = s->offset,
1667             .dev_addr       = sg_dma_address(s),
1668             .size           = sg_dma_len(s),
1669             .direction      = direction,
1670             .sg_call_ents   = nelems,
1671         };
1672 
1673         if (!i)
1674             mapped_ents = get_nr_mapped_entries(dev, &ref);
1675 
1676         if (i >= mapped_ents)
1677             break;
1678 
1679         check_sync(dev, &ref, true);
1680     }
1681 }
1682 EXPORT_SYMBOL(debug_dma_sync_sg_for_cpu);
1683 
1684 void debug_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
1685                   int nelems, int direction)
1686 {
1687     struct scatterlist *s;
1688     int mapped_ents = 0, i;
1689 
1690     if (unlikely(dma_debug_disabled()))
1691         return;
1692 
1693     for_each_sg(sg, s, nelems, i) {
1694 
1695         struct dma_debug_entry ref = {
1696             .type           = dma_debug_sg,
1697             .dev            = dev,
1698             .pfn        = page_to_pfn(sg_page(s)),
1699             .offset     = s->offset,
1700             .dev_addr       = sg_dma_address(s),
1701             .size           = sg_dma_len(s),
1702             .direction      = direction,
1703             .sg_call_ents   = nelems,
1704         };
1705         if (!i)
1706             mapped_ents = get_nr_mapped_entries(dev, &ref);
1707 
1708         if (i >= mapped_ents)
1709             break;
1710 
1711         check_sync(dev, &ref, false);
1712     }
1713 }
1714 EXPORT_SYMBOL(debug_dma_sync_sg_for_device);
1715 
1716 static int __init dma_debug_driver_setup(char *str)
1717 {
1718     int i;
1719 
1720     for (i = 0; i < NAME_MAX_LEN - 1; ++i, ++str) {
1721         current_driver_name[i] = *str;
1722         if (*str == 0)
1723             break;
1724     }
1725 
1726     if (current_driver_name[0])
1727         pr_info("DMA-API: enable driver filter for driver [%s]\n",
1728             current_driver_name);
1729 
1730 
1731     return 1;
1732 }
1733 __setup("dma_debug_driver=", dma_debug_driver_setup);