Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Copyright(C) 2016 Linaro Limited. All rights reserved.
0004  * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
0005  */
0006 
0007 #include <linux/atomic.h>
0008 #include <linux/coresight.h>
0009 #include <linux/dma-mapping.h>
0010 #include <linux/iommu.h>
0011 #include <linux/idr.h>
0012 #include <linux/mutex.h>
0013 #include <linux/refcount.h>
0014 #include <linux/slab.h>
0015 #include <linux/types.h>
0016 #include <linux/vmalloc.h>
0017 #include "coresight-catu.h"
0018 #include "coresight-etm-perf.h"
0019 #include "coresight-priv.h"
0020 #include "coresight-tmc.h"
0021 
0022 struct etr_flat_buf {
0023     struct device   *dev;
0024     dma_addr_t  daddr;
0025     void        *vaddr;
0026     size_t      size;
0027 };
0028 
0029 /*
0030  * etr_perf_buffer - Perf buffer used for ETR
0031  * @drvdata     - The ETR drvdaga this buffer has been allocated for.
0032  * @etr_buf     - Actual buffer used by the ETR
0033  * @pid         - The PID this etr_perf_buffer belongs to.
0034  * @snaphost        - Perf session mode
0035  * @nr_pages        - Number of pages in the ring buffer.
0036  * @pages       - Array of Pages in the ring buffer.
0037  */
0038 struct etr_perf_buffer {
0039     struct tmc_drvdata  *drvdata;
0040     struct etr_buf      *etr_buf;
0041     pid_t           pid;
0042     bool            snapshot;
0043     int         nr_pages;
0044     void            **pages;
0045 };
0046 
0047 /* Convert the perf index to an offset within the ETR buffer */
0048 #define PERF_IDX2OFF(idx, buf)  ((idx) % ((buf)->nr_pages << PAGE_SHIFT))
0049 
0050 /* Lower limit for ETR hardware buffer */
0051 #define TMC_ETR_PERF_MIN_BUF_SIZE   SZ_1M
0052 
0053 /*
0054  * The TMC ETR SG has a page size of 4K. The SG table contains pointers
0055  * to 4KB buffers. However, the OS may use a PAGE_SIZE different from
0056  * 4K (i.e, 16KB or 64KB). This implies that a single OS page could
0057  * contain more than one SG buffer and tables.
0058  *
0059  * A table entry has the following format:
0060  *
0061  * ---Bit31------------Bit4-------Bit1-----Bit0--
0062  * |     Address[39:12]    | SBZ |  Entry Type  |
0063  * ----------------------------------------------
0064  *
0065  * Address: Bits [39:12] of a physical page address. Bits [11:0] are
0066  *      always zero.
0067  *
0068  * Entry type:
0069  *  b00 - Reserved.
0070  *  b01 - Last entry in the tables, points to 4K page buffer.
0071  *  b10 - Normal entry, points to 4K page buffer.
0072  *  b11 - Link. The address points to the base of next table.
0073  */
0074 
0075 typedef u32 sgte_t;
0076 
0077 #define ETR_SG_PAGE_SHIFT       12
0078 #define ETR_SG_PAGE_SIZE        (1UL << ETR_SG_PAGE_SHIFT)
0079 #define ETR_SG_PAGES_PER_SYSPAGE    (PAGE_SIZE / ETR_SG_PAGE_SIZE)
0080 #define ETR_SG_PTRS_PER_PAGE        (ETR_SG_PAGE_SIZE / sizeof(sgte_t))
0081 #define ETR_SG_PTRS_PER_SYSPAGE     (PAGE_SIZE / sizeof(sgte_t))
0082 
0083 #define ETR_SG_ET_MASK          0x3
0084 #define ETR_SG_ET_LAST          0x1
0085 #define ETR_SG_ET_NORMAL        0x2
0086 #define ETR_SG_ET_LINK          0x3
0087 
0088 #define ETR_SG_ADDR_SHIFT       4
0089 
0090 #define ETR_SG_ENTRY(addr, type) \
0091     (sgte_t)((((addr) >> ETR_SG_PAGE_SHIFT) << ETR_SG_ADDR_SHIFT) | \
0092          (type & ETR_SG_ET_MASK))
0093 
0094 #define ETR_SG_ADDR(entry) \
0095     (((dma_addr_t)(entry) >> ETR_SG_ADDR_SHIFT) << ETR_SG_PAGE_SHIFT)
0096 #define ETR_SG_ET(entry)        ((entry) & ETR_SG_ET_MASK)
0097 
0098 /*
0099  * struct etr_sg_table : ETR SG Table
0100  * @sg_table:       Generic SG Table holding the data/table pages.
0101  * @hwaddr:     hwaddress used by the TMC, which is the base
0102  *          address of the table.
0103  */
0104 struct etr_sg_table {
0105     struct tmc_sg_table *sg_table;
0106     dma_addr_t      hwaddr;
0107 };
0108 
0109 /*
0110  * tmc_etr_sg_table_entries: Total number of table entries required to map
0111  * @nr_pages system pages.
0112  *
0113  * We need to map @nr_pages * ETR_SG_PAGES_PER_SYSPAGE data pages.
0114  * Each TMC page can map (ETR_SG_PTRS_PER_PAGE - 1) buffer pointers,
0115  * with the last entry pointing to another page of table entries.
0116  * If we spill over to a new page for mapping 1 entry, we could as
0117  * well replace the link entry of the previous page with the last entry.
0118  */
0119 static inline unsigned long __attribute_const__
0120 tmc_etr_sg_table_entries(int nr_pages)
0121 {
0122     unsigned long nr_sgpages = nr_pages * ETR_SG_PAGES_PER_SYSPAGE;
0123     unsigned long nr_sglinks = nr_sgpages / (ETR_SG_PTRS_PER_PAGE - 1);
0124     /*
0125      * If we spill over to a new page for 1 entry, we could as well
0126      * make it the LAST entry in the previous page, skipping the Link
0127      * address.
0128      */
0129     if (nr_sglinks && (nr_sgpages % (ETR_SG_PTRS_PER_PAGE - 1) < 2))
0130         nr_sglinks--;
0131     return nr_sgpages + nr_sglinks;
0132 }
0133 
0134 /*
0135  * tmc_pages_get_offset:  Go through all the pages in the tmc_pages
0136  * and map the device address @addr to an offset within the virtual
0137  * contiguous buffer.
0138  */
0139 static long
0140 tmc_pages_get_offset(struct tmc_pages *tmc_pages, dma_addr_t addr)
0141 {
0142     int i;
0143     dma_addr_t page_start;
0144 
0145     for (i = 0; i < tmc_pages->nr_pages; i++) {
0146         page_start = tmc_pages->daddrs[i];
0147         if (addr >= page_start && addr < (page_start + PAGE_SIZE))
0148             return i * PAGE_SIZE + (addr - page_start);
0149     }
0150 
0151     return -EINVAL;
0152 }
0153 
0154 /*
0155  * tmc_pages_free : Unmap and free the pages used by tmc_pages.
0156  * If the pages were not allocated in tmc_pages_alloc(), we would
0157  * simply drop the refcount.
0158  */
0159 static void tmc_pages_free(struct tmc_pages *tmc_pages,
0160                struct device *dev, enum dma_data_direction dir)
0161 {
0162     int i;
0163     struct device *real_dev = dev->parent;
0164 
0165     for (i = 0; i < tmc_pages->nr_pages; i++) {
0166         if (tmc_pages->daddrs && tmc_pages->daddrs[i])
0167             dma_unmap_page(real_dev, tmc_pages->daddrs[i],
0168                      PAGE_SIZE, dir);
0169         if (tmc_pages->pages && tmc_pages->pages[i])
0170             __free_page(tmc_pages->pages[i]);
0171     }
0172 
0173     kfree(tmc_pages->pages);
0174     kfree(tmc_pages->daddrs);
0175     tmc_pages->pages = NULL;
0176     tmc_pages->daddrs = NULL;
0177     tmc_pages->nr_pages = 0;
0178 }
0179 
0180 /*
0181  * tmc_pages_alloc : Allocate and map pages for a given @tmc_pages.
0182  * If @pages is not NULL, the list of page virtual addresses are
0183  * used as the data pages. The pages are then dma_map'ed for @dev
0184  * with dma_direction @dir.
0185  *
0186  * Returns 0 upon success, else the error number.
0187  */
0188 static int tmc_pages_alloc(struct tmc_pages *tmc_pages,
0189                struct device *dev, int node,
0190                enum dma_data_direction dir, void **pages)
0191 {
0192     int i, nr_pages;
0193     dma_addr_t paddr;
0194     struct page *page;
0195     struct device *real_dev = dev->parent;
0196 
0197     nr_pages = tmc_pages->nr_pages;
0198     tmc_pages->daddrs = kcalloc(nr_pages, sizeof(*tmc_pages->daddrs),
0199                      GFP_KERNEL);
0200     if (!tmc_pages->daddrs)
0201         return -ENOMEM;
0202     tmc_pages->pages = kcalloc(nr_pages, sizeof(*tmc_pages->pages),
0203                      GFP_KERNEL);
0204     if (!tmc_pages->pages) {
0205         kfree(tmc_pages->daddrs);
0206         tmc_pages->daddrs = NULL;
0207         return -ENOMEM;
0208     }
0209 
0210     for (i = 0; i < nr_pages; i++) {
0211         if (pages && pages[i]) {
0212             page = virt_to_page(pages[i]);
0213             /* Hold a refcount on the page */
0214             get_page(page);
0215         } else {
0216             page = alloc_pages_node(node,
0217                         GFP_KERNEL | __GFP_ZERO, 0);
0218             if (!page)
0219                 goto err;
0220         }
0221         paddr = dma_map_page(real_dev, page, 0, PAGE_SIZE, dir);
0222         if (dma_mapping_error(real_dev, paddr))
0223             goto err;
0224         tmc_pages->daddrs[i] = paddr;
0225         tmc_pages->pages[i] = page;
0226     }
0227     return 0;
0228 err:
0229     tmc_pages_free(tmc_pages, dev, dir);
0230     return -ENOMEM;
0231 }
0232 
0233 static inline long
0234 tmc_sg_get_data_page_offset(struct tmc_sg_table *sg_table, dma_addr_t addr)
0235 {
0236     return tmc_pages_get_offset(&sg_table->data_pages, addr);
0237 }
0238 
0239 static inline void tmc_free_table_pages(struct tmc_sg_table *sg_table)
0240 {
0241     if (sg_table->table_vaddr)
0242         vunmap(sg_table->table_vaddr);
0243     tmc_pages_free(&sg_table->table_pages, sg_table->dev, DMA_TO_DEVICE);
0244 }
0245 
0246 static void tmc_free_data_pages(struct tmc_sg_table *sg_table)
0247 {
0248     if (sg_table->data_vaddr)
0249         vunmap(sg_table->data_vaddr);
0250     tmc_pages_free(&sg_table->data_pages, sg_table->dev, DMA_FROM_DEVICE);
0251 }
0252 
0253 void tmc_free_sg_table(struct tmc_sg_table *sg_table)
0254 {
0255     tmc_free_table_pages(sg_table);
0256     tmc_free_data_pages(sg_table);
0257 }
0258 EXPORT_SYMBOL_GPL(tmc_free_sg_table);
0259 
0260 /*
0261  * Alloc pages for the table. Since this will be used by the device,
0262  * allocate the pages closer to the device (i.e, dev_to_node(dev)
0263  * rather than the CPU node).
0264  */
0265 static int tmc_alloc_table_pages(struct tmc_sg_table *sg_table)
0266 {
0267     int rc;
0268     struct tmc_pages *table_pages = &sg_table->table_pages;
0269 
0270     rc = tmc_pages_alloc(table_pages, sg_table->dev,
0271                  dev_to_node(sg_table->dev),
0272                  DMA_TO_DEVICE, NULL);
0273     if (rc)
0274         return rc;
0275     sg_table->table_vaddr = vmap(table_pages->pages,
0276                      table_pages->nr_pages,
0277                      VM_MAP,
0278                      PAGE_KERNEL);
0279     if (!sg_table->table_vaddr)
0280         rc = -ENOMEM;
0281     else
0282         sg_table->table_daddr = table_pages->daddrs[0];
0283     return rc;
0284 }
0285 
0286 static int tmc_alloc_data_pages(struct tmc_sg_table *sg_table, void **pages)
0287 {
0288     int rc;
0289 
0290     /* Allocate data pages on the node requested by the caller */
0291     rc = tmc_pages_alloc(&sg_table->data_pages,
0292                  sg_table->dev, sg_table->node,
0293                  DMA_FROM_DEVICE, pages);
0294     if (!rc) {
0295         sg_table->data_vaddr = vmap(sg_table->data_pages.pages,
0296                         sg_table->data_pages.nr_pages,
0297                         VM_MAP,
0298                         PAGE_KERNEL);
0299         if (!sg_table->data_vaddr)
0300             rc = -ENOMEM;
0301     }
0302     return rc;
0303 }
0304 
0305 /*
0306  * tmc_alloc_sg_table: Allocate and setup dma pages for the TMC SG table
0307  * and data buffers. TMC writes to the data buffers and reads from the SG
0308  * Table pages.
0309  *
0310  * @dev     - Coresight device to which page should be DMA mapped.
0311  * @node    - Numa node for mem allocations
0312  * @nr_tpages   - Number of pages for the table entries.
0313  * @nr_dpages   - Number of pages for Data buffer.
0314  * @pages   - Optional list of virtual address of pages.
0315  */
0316 struct tmc_sg_table *tmc_alloc_sg_table(struct device *dev,
0317                     int node,
0318                     int nr_tpages,
0319                     int nr_dpages,
0320                     void **pages)
0321 {
0322     long rc;
0323     struct tmc_sg_table *sg_table;
0324 
0325     sg_table = kzalloc(sizeof(*sg_table), GFP_KERNEL);
0326     if (!sg_table)
0327         return ERR_PTR(-ENOMEM);
0328     sg_table->data_pages.nr_pages = nr_dpages;
0329     sg_table->table_pages.nr_pages = nr_tpages;
0330     sg_table->node = node;
0331     sg_table->dev = dev;
0332 
0333     rc  = tmc_alloc_data_pages(sg_table, pages);
0334     if (!rc)
0335         rc = tmc_alloc_table_pages(sg_table);
0336     if (rc) {
0337         tmc_free_sg_table(sg_table);
0338         kfree(sg_table);
0339         return ERR_PTR(rc);
0340     }
0341 
0342     return sg_table;
0343 }
0344 EXPORT_SYMBOL_GPL(tmc_alloc_sg_table);
0345 
0346 /*
0347  * tmc_sg_table_sync_data_range: Sync the data buffer written
0348  * by the device from @offset upto a @size bytes.
0349  */
0350 void tmc_sg_table_sync_data_range(struct tmc_sg_table *table,
0351                   u64 offset, u64 size)
0352 {
0353     int i, index, start;
0354     int npages = DIV_ROUND_UP(size, PAGE_SIZE);
0355     struct device *real_dev = table->dev->parent;
0356     struct tmc_pages *data = &table->data_pages;
0357 
0358     start = offset >> PAGE_SHIFT;
0359     for (i = start; i < (start + npages); i++) {
0360         index = i % data->nr_pages;
0361         dma_sync_single_for_cpu(real_dev, data->daddrs[index],
0362                     PAGE_SIZE, DMA_FROM_DEVICE);
0363     }
0364 }
0365 EXPORT_SYMBOL_GPL(tmc_sg_table_sync_data_range);
0366 
0367 /* tmc_sg_sync_table: Sync the page table */
0368 void tmc_sg_table_sync_table(struct tmc_sg_table *sg_table)
0369 {
0370     int i;
0371     struct device *real_dev = sg_table->dev->parent;
0372     struct tmc_pages *table_pages = &sg_table->table_pages;
0373 
0374     for (i = 0; i < table_pages->nr_pages; i++)
0375         dma_sync_single_for_device(real_dev, table_pages->daddrs[i],
0376                        PAGE_SIZE, DMA_TO_DEVICE);
0377 }
0378 EXPORT_SYMBOL_GPL(tmc_sg_table_sync_table);
0379 
0380 /*
0381  * tmc_sg_table_get_data: Get the buffer pointer for data @offset
0382  * in the SG buffer. The @bufpp is updated to point to the buffer.
0383  * Returns :
0384  *  the length of linear data available at @offset.
0385  *  or
0386  *  <= 0 if no data is available.
0387  */
0388 ssize_t tmc_sg_table_get_data(struct tmc_sg_table *sg_table,
0389                   u64 offset, size_t len, char **bufpp)
0390 {
0391     size_t size;
0392     int pg_idx = offset >> PAGE_SHIFT;
0393     int pg_offset = offset & (PAGE_SIZE - 1);
0394     struct tmc_pages *data_pages = &sg_table->data_pages;
0395 
0396     size = tmc_sg_table_buf_size(sg_table);
0397     if (offset >= size)
0398         return -EINVAL;
0399 
0400     /* Make sure we don't go beyond the end */
0401     len = (len < (size - offset)) ? len : size - offset;
0402     /* Respect the page boundaries */
0403     len = (len < (PAGE_SIZE - pg_offset)) ? len : (PAGE_SIZE - pg_offset);
0404     if (len > 0)
0405         *bufpp = page_address(data_pages->pages[pg_idx]) + pg_offset;
0406     return len;
0407 }
0408 EXPORT_SYMBOL_GPL(tmc_sg_table_get_data);
0409 
0410 #ifdef ETR_SG_DEBUG
0411 /* Map a dma address to virtual address */
0412 static unsigned long
0413 tmc_sg_daddr_to_vaddr(struct tmc_sg_table *sg_table,
0414               dma_addr_t addr, bool table)
0415 {
0416     long offset;
0417     unsigned long base;
0418     struct tmc_pages *tmc_pages;
0419 
0420     if (table) {
0421         tmc_pages = &sg_table->table_pages;
0422         base = (unsigned long)sg_table->table_vaddr;
0423     } else {
0424         tmc_pages = &sg_table->data_pages;
0425         base = (unsigned long)sg_table->data_vaddr;
0426     }
0427 
0428     offset = tmc_pages_get_offset(tmc_pages, addr);
0429     if (offset < 0)
0430         return 0;
0431     return base + offset;
0432 }
0433 
0434 /* Dump the given sg_table */
0435 static void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table)
0436 {
0437     sgte_t *ptr;
0438     int i = 0;
0439     dma_addr_t addr;
0440     struct tmc_sg_table *sg_table = etr_table->sg_table;
0441 
0442     ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
0443                           etr_table->hwaddr, true);
0444     while (ptr) {
0445         addr = ETR_SG_ADDR(*ptr);
0446         switch (ETR_SG_ET(*ptr)) {
0447         case ETR_SG_ET_NORMAL:
0448             dev_dbg(sg_table->dev,
0449                 "%05d: %p\t:[N] 0x%llx\n", i, ptr, addr);
0450             ptr++;
0451             break;
0452         case ETR_SG_ET_LINK:
0453             dev_dbg(sg_table->dev,
0454                 "%05d: *** %p\t:{L} 0x%llx ***\n",
0455                  i, ptr, addr);
0456             ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
0457                                   addr, true);
0458             break;
0459         case ETR_SG_ET_LAST:
0460             dev_dbg(sg_table->dev,
0461                 "%05d: ### %p\t:[L] 0x%llx ###\n",
0462                  i, ptr, addr);
0463             return;
0464         default:
0465             dev_dbg(sg_table->dev,
0466                 "%05d: xxx %p\t:[INVALID] 0x%llx xxx\n",
0467                  i, ptr, addr);
0468             return;
0469         }
0470         i++;
0471     }
0472     dev_dbg(sg_table->dev, "******* End of Table *****\n");
0473 }
0474 #else
0475 static inline void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table) {}
0476 #endif
0477 
0478 /*
0479  * Populate the SG Table page table entries from table/data
0480  * pages allocated. Each Data page has ETR_SG_PAGES_PER_SYSPAGE SG pages.
0481  * So does a Table page. So we keep track of indices of the tables
0482  * in each system page and move the pointers accordingly.
0483  */
0484 #define INC_IDX_ROUND(idx, size) ((idx) = ((idx) + 1) % (size))
0485 static void tmc_etr_sg_table_populate(struct etr_sg_table *etr_table)
0486 {
0487     dma_addr_t paddr;
0488     int i, type, nr_entries;
0489     int tpidx = 0; /* index to the current system table_page */
0490     int sgtidx = 0; /* index to the sg_table within the current syspage */
0491     int sgtentry = 0; /* the entry within the sg_table */
0492     int dpidx = 0; /* index to the current system data_page */
0493     int spidx = 0; /* index to the SG page within the current data page */
0494     sgte_t *ptr; /* pointer to the table entry to fill */
0495     struct tmc_sg_table *sg_table = etr_table->sg_table;
0496     dma_addr_t *table_daddrs = sg_table->table_pages.daddrs;
0497     dma_addr_t *data_daddrs = sg_table->data_pages.daddrs;
0498 
0499     nr_entries = tmc_etr_sg_table_entries(sg_table->data_pages.nr_pages);
0500     /*
0501      * Use the contiguous virtual address of the table to update entries.
0502      */
0503     ptr = sg_table->table_vaddr;
0504     /*
0505      * Fill all the entries, except the last entry to avoid special
0506      * checks within the loop.
0507      */
0508     for (i = 0; i < nr_entries - 1; i++) {
0509         if (sgtentry == ETR_SG_PTRS_PER_PAGE - 1) {
0510             /*
0511              * Last entry in a sg_table page is a link address to
0512              * the next table page. If this sg_table is the last
0513              * one in the system page, it links to the first
0514              * sg_table in the next system page. Otherwise, it
0515              * links to the next sg_table page within the system
0516              * page.
0517              */
0518             if (sgtidx == ETR_SG_PAGES_PER_SYSPAGE - 1) {
0519                 paddr = table_daddrs[tpidx + 1];
0520             } else {
0521                 paddr = table_daddrs[tpidx] +
0522                     (ETR_SG_PAGE_SIZE * (sgtidx + 1));
0523             }
0524             type = ETR_SG_ET_LINK;
0525         } else {
0526             /*
0527              * Update the indices to the data_pages to point to the
0528              * next sg_page in the data buffer.
0529              */
0530             type = ETR_SG_ET_NORMAL;
0531             paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
0532             if (!INC_IDX_ROUND(spidx, ETR_SG_PAGES_PER_SYSPAGE))
0533                 dpidx++;
0534         }
0535         *ptr++ = ETR_SG_ENTRY(paddr, type);
0536         /*
0537          * Move to the next table pointer, moving the table page index
0538          * if necessary
0539          */
0540         if (!INC_IDX_ROUND(sgtentry, ETR_SG_PTRS_PER_PAGE)) {
0541             if (!INC_IDX_ROUND(sgtidx, ETR_SG_PAGES_PER_SYSPAGE))
0542                 tpidx++;
0543         }
0544     }
0545 
0546     /* Set up the last entry, which is always a data pointer */
0547     paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
0548     *ptr++ = ETR_SG_ENTRY(paddr, ETR_SG_ET_LAST);
0549 }
0550 
0551 /*
0552  * tmc_init_etr_sg_table: Allocate a TMC ETR SG table, data buffer of @size and
0553  * populate the table.
0554  *
0555  * @dev     - Device pointer for the TMC
0556  * @node    - NUMA node where the memory should be allocated
0557  * @size    - Total size of the data buffer
0558  * @pages   - Optional list of page virtual address
0559  */
0560 static struct etr_sg_table *
0561 tmc_init_etr_sg_table(struct device *dev, int node,
0562               unsigned long size, void **pages)
0563 {
0564     int nr_entries, nr_tpages;
0565     int nr_dpages = size >> PAGE_SHIFT;
0566     struct tmc_sg_table *sg_table;
0567     struct etr_sg_table *etr_table;
0568 
0569     etr_table = kzalloc(sizeof(*etr_table), GFP_KERNEL);
0570     if (!etr_table)
0571         return ERR_PTR(-ENOMEM);
0572     nr_entries = tmc_etr_sg_table_entries(nr_dpages);
0573     nr_tpages = DIV_ROUND_UP(nr_entries, ETR_SG_PTRS_PER_SYSPAGE);
0574 
0575     sg_table = tmc_alloc_sg_table(dev, node, nr_tpages, nr_dpages, pages);
0576     if (IS_ERR(sg_table)) {
0577         kfree(etr_table);
0578         return ERR_CAST(sg_table);
0579     }
0580 
0581     etr_table->sg_table = sg_table;
0582     /* TMC should use table base address for DBA */
0583     etr_table->hwaddr = sg_table->table_daddr;
0584     tmc_etr_sg_table_populate(etr_table);
0585     /* Sync the table pages for the HW */
0586     tmc_sg_table_sync_table(sg_table);
0587     tmc_etr_sg_table_dump(etr_table);
0588 
0589     return etr_table;
0590 }
0591 
0592 /*
0593  * tmc_etr_alloc_flat_buf: Allocate a contiguous DMA buffer.
0594  */
0595 static int tmc_etr_alloc_flat_buf(struct tmc_drvdata *drvdata,
0596                   struct etr_buf *etr_buf, int node,
0597                   void **pages)
0598 {
0599     struct etr_flat_buf *flat_buf;
0600     struct device *real_dev = drvdata->csdev->dev.parent;
0601 
0602     /* We cannot reuse existing pages for flat buf */
0603     if (pages)
0604         return -EINVAL;
0605 
0606     flat_buf = kzalloc(sizeof(*flat_buf), GFP_KERNEL);
0607     if (!flat_buf)
0608         return -ENOMEM;
0609 
0610     flat_buf->vaddr = dma_alloc_noncoherent(real_dev, etr_buf->size,
0611                         &flat_buf->daddr,
0612                         DMA_FROM_DEVICE, GFP_KERNEL);
0613     if (!flat_buf->vaddr) {
0614         kfree(flat_buf);
0615         return -ENOMEM;
0616     }
0617 
0618     flat_buf->size = etr_buf->size;
0619     flat_buf->dev = &drvdata->csdev->dev;
0620     etr_buf->hwaddr = flat_buf->daddr;
0621     etr_buf->mode = ETR_MODE_FLAT;
0622     etr_buf->private = flat_buf;
0623     return 0;
0624 }
0625 
0626 static void tmc_etr_free_flat_buf(struct etr_buf *etr_buf)
0627 {
0628     struct etr_flat_buf *flat_buf = etr_buf->private;
0629 
0630     if (flat_buf && flat_buf->daddr) {
0631         struct device *real_dev = flat_buf->dev->parent;
0632 
0633         dma_free_noncoherent(real_dev, etr_buf->size,
0634                      flat_buf->vaddr, flat_buf->daddr,
0635                      DMA_FROM_DEVICE);
0636     }
0637     kfree(flat_buf);
0638 }
0639 
0640 static void tmc_etr_sync_flat_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
0641 {
0642     struct etr_flat_buf *flat_buf = etr_buf->private;
0643     struct device *real_dev = flat_buf->dev->parent;
0644 
0645     /*
0646      * Adjust the buffer to point to the beginning of the trace data
0647      * and update the available trace data.
0648      */
0649     etr_buf->offset = rrp - etr_buf->hwaddr;
0650     if (etr_buf->full)
0651         etr_buf->len = etr_buf->size;
0652     else
0653         etr_buf->len = rwp - rrp;
0654 
0655     /*
0656      * The driver always starts tracing at the beginning of the buffer,
0657      * the only reason why we would get a wrap around is when the buffer
0658      * is full.  Sync the entire buffer in one go for this case.
0659      */
0660     if (etr_buf->offset + etr_buf->len > etr_buf->size)
0661         dma_sync_single_for_cpu(real_dev, flat_buf->daddr,
0662                     etr_buf->size, DMA_FROM_DEVICE);
0663     else
0664         dma_sync_single_for_cpu(real_dev,
0665                     flat_buf->daddr + etr_buf->offset,
0666                     etr_buf->len, DMA_FROM_DEVICE);
0667 }
0668 
0669 static ssize_t tmc_etr_get_data_flat_buf(struct etr_buf *etr_buf,
0670                      u64 offset, size_t len, char **bufpp)
0671 {
0672     struct etr_flat_buf *flat_buf = etr_buf->private;
0673 
0674     *bufpp = (char *)flat_buf->vaddr + offset;
0675     /*
0676      * tmc_etr_buf_get_data already adjusts the length to handle
0677      * buffer wrapping around.
0678      */
0679     return len;
0680 }
0681 
0682 static const struct etr_buf_operations etr_flat_buf_ops = {
0683     .alloc = tmc_etr_alloc_flat_buf,
0684     .free = tmc_etr_free_flat_buf,
0685     .sync = tmc_etr_sync_flat_buf,
0686     .get_data = tmc_etr_get_data_flat_buf,
0687 };
0688 
0689 /*
0690  * tmc_etr_alloc_sg_buf: Allocate an SG buf @etr_buf. Setup the parameters
0691  * appropriately.
0692  */
0693 static int tmc_etr_alloc_sg_buf(struct tmc_drvdata *drvdata,
0694                 struct etr_buf *etr_buf, int node,
0695                 void **pages)
0696 {
0697     struct etr_sg_table *etr_table;
0698     struct device *dev = &drvdata->csdev->dev;
0699 
0700     etr_table = tmc_init_etr_sg_table(dev, node,
0701                       etr_buf->size, pages);
0702     if (IS_ERR(etr_table))
0703         return -ENOMEM;
0704     etr_buf->hwaddr = etr_table->hwaddr;
0705     etr_buf->mode = ETR_MODE_ETR_SG;
0706     etr_buf->private = etr_table;
0707     return 0;
0708 }
0709 
0710 static void tmc_etr_free_sg_buf(struct etr_buf *etr_buf)
0711 {
0712     struct etr_sg_table *etr_table = etr_buf->private;
0713 
0714     if (etr_table) {
0715         tmc_free_sg_table(etr_table->sg_table);
0716         kfree(etr_table);
0717     }
0718 }
0719 
0720 static ssize_t tmc_etr_get_data_sg_buf(struct etr_buf *etr_buf, u64 offset,
0721                        size_t len, char **bufpp)
0722 {
0723     struct etr_sg_table *etr_table = etr_buf->private;
0724 
0725     return tmc_sg_table_get_data(etr_table->sg_table, offset, len, bufpp);
0726 }
0727 
0728 static void tmc_etr_sync_sg_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
0729 {
0730     long r_offset, w_offset;
0731     struct etr_sg_table *etr_table = etr_buf->private;
0732     struct tmc_sg_table *table = etr_table->sg_table;
0733 
0734     /* Convert hw address to offset in the buffer */
0735     r_offset = tmc_sg_get_data_page_offset(table, rrp);
0736     if (r_offset < 0) {
0737         dev_warn(table->dev,
0738              "Unable to map RRP %llx to offset\n", rrp);
0739         etr_buf->len = 0;
0740         return;
0741     }
0742 
0743     w_offset = tmc_sg_get_data_page_offset(table, rwp);
0744     if (w_offset < 0) {
0745         dev_warn(table->dev,
0746              "Unable to map RWP %llx to offset\n", rwp);
0747         etr_buf->len = 0;
0748         return;
0749     }
0750 
0751     etr_buf->offset = r_offset;
0752     if (etr_buf->full)
0753         etr_buf->len = etr_buf->size;
0754     else
0755         etr_buf->len = ((w_offset < r_offset) ? etr_buf->size : 0) +
0756                 w_offset - r_offset;
0757     tmc_sg_table_sync_data_range(table, r_offset, etr_buf->len);
0758 }
0759 
0760 static const struct etr_buf_operations etr_sg_buf_ops = {
0761     .alloc = tmc_etr_alloc_sg_buf,
0762     .free = tmc_etr_free_sg_buf,
0763     .sync = tmc_etr_sync_sg_buf,
0764     .get_data = tmc_etr_get_data_sg_buf,
0765 };
0766 
0767 /*
0768  * TMC ETR could be connected to a CATU device, which can provide address
0769  * translation service. This is represented by the Output port of the TMC
0770  * (ETR) connected to the input port of the CATU.
0771  *
0772  * Returns  : coresight_device ptr for the CATU device if a CATU is found.
0773  *      : NULL otherwise.
0774  */
0775 struct coresight_device *
0776 tmc_etr_get_catu_device(struct tmc_drvdata *drvdata)
0777 {
0778     int i;
0779     struct coresight_device *tmp, *etr = drvdata->csdev;
0780 
0781     if (!IS_ENABLED(CONFIG_CORESIGHT_CATU))
0782         return NULL;
0783 
0784     for (i = 0; i < etr->pdata->nr_outport; i++) {
0785         tmp = etr->pdata->conns[i].child_dev;
0786         if (tmp && coresight_is_catu_device(tmp))
0787             return tmp;
0788     }
0789 
0790     return NULL;
0791 }
0792 EXPORT_SYMBOL_GPL(tmc_etr_get_catu_device);
0793 
0794 static inline int tmc_etr_enable_catu(struct tmc_drvdata *drvdata,
0795                       struct etr_buf *etr_buf)
0796 {
0797     struct coresight_device *catu = tmc_etr_get_catu_device(drvdata);
0798 
0799     if (catu && helper_ops(catu)->enable)
0800         return helper_ops(catu)->enable(catu, etr_buf);
0801     return 0;
0802 }
0803 
0804 static inline void tmc_etr_disable_catu(struct tmc_drvdata *drvdata)
0805 {
0806     struct coresight_device *catu = tmc_etr_get_catu_device(drvdata);
0807 
0808     if (catu && helper_ops(catu)->disable)
0809         helper_ops(catu)->disable(catu, drvdata->etr_buf);
0810 }
0811 
0812 static const struct etr_buf_operations *etr_buf_ops[] = {
0813     [ETR_MODE_FLAT] = &etr_flat_buf_ops,
0814     [ETR_MODE_ETR_SG] = &etr_sg_buf_ops,
0815     [ETR_MODE_CATU] = NULL,
0816 };
0817 
0818 void tmc_etr_set_catu_ops(const struct etr_buf_operations *catu)
0819 {
0820     etr_buf_ops[ETR_MODE_CATU] = catu;
0821 }
0822 EXPORT_SYMBOL_GPL(tmc_etr_set_catu_ops);
0823 
0824 void tmc_etr_remove_catu_ops(void)
0825 {
0826     etr_buf_ops[ETR_MODE_CATU] = NULL;
0827 }
0828 EXPORT_SYMBOL_GPL(tmc_etr_remove_catu_ops);
0829 
0830 static inline int tmc_etr_mode_alloc_buf(int mode,
0831                      struct tmc_drvdata *drvdata,
0832                      struct etr_buf *etr_buf, int node,
0833                      void **pages)
0834 {
0835     int rc = -EINVAL;
0836 
0837     switch (mode) {
0838     case ETR_MODE_FLAT:
0839     case ETR_MODE_ETR_SG:
0840     case ETR_MODE_CATU:
0841         if (etr_buf_ops[mode] && etr_buf_ops[mode]->alloc)
0842             rc = etr_buf_ops[mode]->alloc(drvdata, etr_buf,
0843                               node, pages);
0844         if (!rc)
0845             etr_buf->ops = etr_buf_ops[mode];
0846         return rc;
0847     default:
0848         return -EINVAL;
0849     }
0850 }
0851 
0852 /*
0853  * tmc_alloc_etr_buf: Allocate a buffer use by ETR.
0854  * @drvdata : ETR device details.
0855  * @size    : size of the requested buffer.
0856  * @flags   : Required properties for the buffer.
0857  * @node    : Node for memory allocations.
0858  * @pages   : An optional list of pages.
0859  */
0860 static struct etr_buf *tmc_alloc_etr_buf(struct tmc_drvdata *drvdata,
0861                      ssize_t size, int flags,
0862                      int node, void **pages)
0863 {
0864     int rc = -ENOMEM;
0865     bool has_etr_sg, has_iommu;
0866     bool has_sg, has_catu;
0867     struct etr_buf *etr_buf;
0868     struct device *dev = &drvdata->csdev->dev;
0869 
0870     has_etr_sg = tmc_etr_has_cap(drvdata, TMC_ETR_SG);
0871     has_iommu = iommu_get_domain_for_dev(dev->parent);
0872     has_catu = !!tmc_etr_get_catu_device(drvdata);
0873 
0874     has_sg = has_catu || has_etr_sg;
0875 
0876     etr_buf = kzalloc(sizeof(*etr_buf), GFP_KERNEL);
0877     if (!etr_buf)
0878         return ERR_PTR(-ENOMEM);
0879 
0880     etr_buf->size = size;
0881 
0882     /*
0883      * If we have to use an existing list of pages, we cannot reliably
0884      * use a contiguous DMA memory (even if we have an IOMMU). Otherwise,
0885      * we use the contiguous DMA memory if at least one of the following
0886      * conditions is true:
0887      *  a) The ETR cannot use Scatter-Gather.
0888      *  b) we have a backing IOMMU
0889      *  c) The requested memory size is smaller (< 1M).
0890      *
0891      * Fallback to available mechanisms.
0892      *
0893      */
0894     if (!pages &&
0895         (!has_sg || has_iommu || size < SZ_1M))
0896         rc = tmc_etr_mode_alloc_buf(ETR_MODE_FLAT, drvdata,
0897                         etr_buf, node, pages);
0898     if (rc && has_etr_sg)
0899         rc = tmc_etr_mode_alloc_buf(ETR_MODE_ETR_SG, drvdata,
0900                         etr_buf, node, pages);
0901     if (rc && has_catu)
0902         rc = tmc_etr_mode_alloc_buf(ETR_MODE_CATU, drvdata,
0903                         etr_buf, node, pages);
0904     if (rc) {
0905         kfree(etr_buf);
0906         return ERR_PTR(rc);
0907     }
0908 
0909     refcount_set(&etr_buf->refcount, 1);
0910     dev_dbg(dev, "allocated buffer of size %ldKB in mode %d\n",
0911         (unsigned long)size >> 10, etr_buf->mode);
0912     return etr_buf;
0913 }
0914 
0915 static void tmc_free_etr_buf(struct etr_buf *etr_buf)
0916 {
0917     WARN_ON(!etr_buf->ops || !etr_buf->ops->free);
0918     etr_buf->ops->free(etr_buf);
0919     kfree(etr_buf);
0920 }
0921 
0922 /*
0923  * tmc_etr_buf_get_data: Get the pointer the trace data at @offset
0924  * with a maximum of @len bytes.
0925  * Returns: The size of the linear data available @pos, with *bufpp
0926  * updated to point to the buffer.
0927  */
0928 static ssize_t tmc_etr_buf_get_data(struct etr_buf *etr_buf,
0929                     u64 offset, size_t len, char **bufpp)
0930 {
0931     /* Adjust the length to limit this transaction to end of buffer */
0932     len = (len < (etr_buf->size - offset)) ? len : etr_buf->size - offset;
0933 
0934     return etr_buf->ops->get_data(etr_buf, (u64)offset, len, bufpp);
0935 }
0936 
0937 static inline s64
0938 tmc_etr_buf_insert_barrier_packet(struct etr_buf *etr_buf, u64 offset)
0939 {
0940     ssize_t len;
0941     char *bufp;
0942 
0943     len = tmc_etr_buf_get_data(etr_buf, offset,
0944                    CORESIGHT_BARRIER_PKT_SIZE, &bufp);
0945     if (WARN_ON(len < CORESIGHT_BARRIER_PKT_SIZE))
0946         return -EINVAL;
0947     coresight_insert_barrier_packet(bufp);
0948     return offset + CORESIGHT_BARRIER_PKT_SIZE;
0949 }
0950 
0951 /*
0952  * tmc_sync_etr_buf: Sync the trace buffer availability with drvdata.
0953  * Makes sure the trace data is synced to the memory for consumption.
0954  * @etr_buf->offset will hold the offset to the beginning of the trace data
0955  * within the buffer, with @etr_buf->len bytes to consume.
0956  */
0957 static void tmc_sync_etr_buf(struct tmc_drvdata *drvdata)
0958 {
0959     struct etr_buf *etr_buf = drvdata->etr_buf;
0960     u64 rrp, rwp;
0961     u32 status;
0962 
0963     rrp = tmc_read_rrp(drvdata);
0964     rwp = tmc_read_rwp(drvdata);
0965     status = readl_relaxed(drvdata->base + TMC_STS);
0966 
0967     /*
0968      * If there were memory errors in the session, truncate the
0969      * buffer.
0970      */
0971     if (WARN_ON_ONCE(status & TMC_STS_MEMERR)) {
0972         dev_dbg(&drvdata->csdev->dev,
0973             "tmc memory error detected, truncating buffer\n");
0974         etr_buf->len = 0;
0975         etr_buf->full = false;
0976         return;
0977     }
0978 
0979     etr_buf->full = !!(status & TMC_STS_FULL);
0980 
0981     WARN_ON(!etr_buf->ops || !etr_buf->ops->sync);
0982 
0983     etr_buf->ops->sync(etr_buf, rrp, rwp);
0984 }
0985 
0986 static void __tmc_etr_enable_hw(struct tmc_drvdata *drvdata)
0987 {
0988     u32 axictl, sts;
0989     struct etr_buf *etr_buf = drvdata->etr_buf;
0990 
0991     CS_UNLOCK(drvdata->base);
0992 
0993     /* Wait for TMCSReady bit to be set */
0994     tmc_wait_for_tmcready(drvdata);
0995 
0996     writel_relaxed(etr_buf->size / 4, drvdata->base + TMC_RSZ);
0997     writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
0998 
0999     axictl = readl_relaxed(drvdata->base + TMC_AXICTL);
1000     axictl &= ~TMC_AXICTL_CLEAR_MASK;
1001     axictl |= TMC_AXICTL_PROT_CTL_B1;
1002     axictl |= TMC_AXICTL_WR_BURST(drvdata->max_burst_size);
1003     axictl |= TMC_AXICTL_AXCACHE_OS;
1004 
1005     if (tmc_etr_has_cap(drvdata, TMC_ETR_AXI_ARCACHE)) {
1006         axictl &= ~TMC_AXICTL_ARCACHE_MASK;
1007         axictl |= TMC_AXICTL_ARCACHE_OS;
1008     }
1009 
1010     if (etr_buf->mode == ETR_MODE_ETR_SG)
1011         axictl |= TMC_AXICTL_SCT_GAT_MODE;
1012 
1013     writel_relaxed(axictl, drvdata->base + TMC_AXICTL);
1014     tmc_write_dba(drvdata, etr_buf->hwaddr);
1015     /*
1016      * If the TMC pointers must be programmed before the session,
1017      * we have to set it properly (i.e, RRP/RWP to base address and
1018      * STS to "not full").
1019      */
1020     if (tmc_etr_has_cap(drvdata, TMC_ETR_SAVE_RESTORE)) {
1021         tmc_write_rrp(drvdata, etr_buf->hwaddr);
1022         tmc_write_rwp(drvdata, etr_buf->hwaddr);
1023         sts = readl_relaxed(drvdata->base + TMC_STS) & ~TMC_STS_FULL;
1024         writel_relaxed(sts, drvdata->base + TMC_STS);
1025     }
1026 
1027     writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |
1028                TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |
1029                TMC_FFCR_TRIGON_TRIGIN,
1030                drvdata->base + TMC_FFCR);
1031     writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
1032     tmc_enable_hw(drvdata);
1033 
1034     CS_LOCK(drvdata->base);
1035 }
1036 
1037 static int tmc_etr_enable_hw(struct tmc_drvdata *drvdata,
1038                  struct etr_buf *etr_buf)
1039 {
1040     int rc;
1041 
1042     /* Callers should provide an appropriate buffer for use */
1043     if (WARN_ON(!etr_buf))
1044         return -EINVAL;
1045 
1046     if ((etr_buf->mode == ETR_MODE_ETR_SG) &&
1047         WARN_ON(!tmc_etr_has_cap(drvdata, TMC_ETR_SG)))
1048         return -EINVAL;
1049 
1050     if (WARN_ON(drvdata->etr_buf))
1051         return -EBUSY;
1052 
1053     /*
1054      * If this ETR is connected to a CATU, enable it before we turn
1055      * this on.
1056      */
1057     rc = tmc_etr_enable_catu(drvdata, etr_buf);
1058     if (rc)
1059         return rc;
1060     rc = coresight_claim_device(drvdata->csdev);
1061     if (!rc) {
1062         drvdata->etr_buf = etr_buf;
1063         __tmc_etr_enable_hw(drvdata);
1064     }
1065 
1066     return rc;
1067 }
1068 
1069 /*
1070  * Return the available trace data in the buffer (starts at etr_buf->offset,
1071  * limited by etr_buf->len) from @pos, with a maximum limit of @len,
1072  * also updating the @bufpp on where to find it. Since the trace data
1073  * starts at anywhere in the buffer, depending on the RRP, we adjust the
1074  * @len returned to handle buffer wrapping around.
1075  *
1076  * We are protected here by drvdata->reading != 0, which ensures the
1077  * sysfs_buf stays alive.
1078  */
1079 ssize_t tmc_etr_get_sysfs_trace(struct tmc_drvdata *drvdata,
1080                 loff_t pos, size_t len, char **bufpp)
1081 {
1082     s64 offset;
1083     ssize_t actual = len;
1084     struct etr_buf *etr_buf = drvdata->sysfs_buf;
1085 
1086     if (pos + actual > etr_buf->len)
1087         actual = etr_buf->len - pos;
1088     if (actual <= 0)
1089         return actual;
1090 
1091     /* Compute the offset from which we read the data */
1092     offset = etr_buf->offset + pos;
1093     if (offset >= etr_buf->size)
1094         offset -= etr_buf->size;
1095     return tmc_etr_buf_get_data(etr_buf, offset, actual, bufpp);
1096 }
1097 
1098 static struct etr_buf *
1099 tmc_etr_setup_sysfs_buf(struct tmc_drvdata *drvdata)
1100 {
1101     return tmc_alloc_etr_buf(drvdata, drvdata->size,
1102                  0, cpu_to_node(0), NULL);
1103 }
1104 
1105 static void
1106 tmc_etr_free_sysfs_buf(struct etr_buf *buf)
1107 {
1108     if (buf)
1109         tmc_free_etr_buf(buf);
1110 }
1111 
1112 static void tmc_etr_sync_sysfs_buf(struct tmc_drvdata *drvdata)
1113 {
1114     struct etr_buf *etr_buf = drvdata->etr_buf;
1115 
1116     if (WARN_ON(drvdata->sysfs_buf != etr_buf)) {
1117         tmc_etr_free_sysfs_buf(drvdata->sysfs_buf);
1118         drvdata->sysfs_buf = NULL;
1119     } else {
1120         tmc_sync_etr_buf(drvdata);
1121         /*
1122          * Insert barrier packets at the beginning, if there was
1123          * an overflow.
1124          */
1125         if (etr_buf->full)
1126             tmc_etr_buf_insert_barrier_packet(etr_buf,
1127                               etr_buf->offset);
1128     }
1129 }
1130 
1131 static void __tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1132 {
1133     CS_UNLOCK(drvdata->base);
1134 
1135     tmc_flush_and_stop(drvdata);
1136     /*
1137      * When operating in sysFS mode the content of the buffer needs to be
1138      * read before the TMC is disabled.
1139      */
1140     if (drvdata->mode == CS_MODE_SYSFS)
1141         tmc_etr_sync_sysfs_buf(drvdata);
1142 
1143     tmc_disable_hw(drvdata);
1144 
1145     CS_LOCK(drvdata->base);
1146 
1147 }
1148 
1149 void tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1150 {
1151     __tmc_etr_disable_hw(drvdata);
1152     /* Disable CATU device if this ETR is connected to one */
1153     tmc_etr_disable_catu(drvdata);
1154     coresight_disclaim_device(drvdata->csdev);
1155     /* Reset the ETR buf used by hardware */
1156     drvdata->etr_buf = NULL;
1157 }
1158 
1159 static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev)
1160 {
1161     int ret = 0;
1162     unsigned long flags;
1163     struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1164     struct etr_buf *sysfs_buf = NULL, *new_buf = NULL, *free_buf = NULL;
1165 
1166     /*
1167      * If we are enabling the ETR from disabled state, we need to make
1168      * sure we have a buffer with the right size. The etr_buf is not reset
1169      * immediately after we stop the tracing in SYSFS mode as we wait for
1170      * the user to collect the data. We may be able to reuse the existing
1171      * buffer, provided the size matches. Any allocation has to be done
1172      * with the lock released.
1173      */
1174     spin_lock_irqsave(&drvdata->spinlock, flags);
1175     sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1176     if (!sysfs_buf || (sysfs_buf->size != drvdata->size)) {
1177         spin_unlock_irqrestore(&drvdata->spinlock, flags);
1178 
1179         /* Allocate memory with the locks released */
1180         free_buf = new_buf = tmc_etr_setup_sysfs_buf(drvdata);
1181         if (IS_ERR(new_buf))
1182             return PTR_ERR(new_buf);
1183 
1184         /* Let's try again */
1185         spin_lock_irqsave(&drvdata->spinlock, flags);
1186     }
1187 
1188     if (drvdata->reading || drvdata->mode == CS_MODE_PERF) {
1189         ret = -EBUSY;
1190         goto out;
1191     }
1192 
1193     /*
1194      * In sysFS mode we can have multiple writers per sink.  Since this
1195      * sink is already enabled no memory is needed and the HW need not be
1196      * touched, even if the buffer size has changed.
1197      */
1198     if (drvdata->mode == CS_MODE_SYSFS) {
1199         atomic_inc(csdev->refcnt);
1200         goto out;
1201     }
1202 
1203     /*
1204      * If we don't have a buffer or it doesn't match the requested size,
1205      * use the buffer allocated above. Otherwise reuse the existing buffer.
1206      */
1207     sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1208     if (!sysfs_buf || (new_buf && sysfs_buf->size != new_buf->size)) {
1209         free_buf = sysfs_buf;
1210         drvdata->sysfs_buf = new_buf;
1211     }
1212 
1213     ret = tmc_etr_enable_hw(drvdata, drvdata->sysfs_buf);
1214     if (!ret) {
1215         drvdata->mode = CS_MODE_SYSFS;
1216         atomic_inc(csdev->refcnt);
1217     }
1218 out:
1219     spin_unlock_irqrestore(&drvdata->spinlock, flags);
1220 
1221     /* Free memory outside the spinlock if need be */
1222     if (free_buf)
1223         tmc_etr_free_sysfs_buf(free_buf);
1224 
1225     if (!ret)
1226         dev_dbg(&csdev->dev, "TMC-ETR enabled\n");
1227 
1228     return ret;
1229 }
1230 
1231 /*
1232  * alloc_etr_buf: Allocate ETR buffer for use by perf.
1233  * The size of the hardware buffer is dependent on the size configured
1234  * via sysfs and the perf ring buffer size. We prefer to allocate the
1235  * largest possible size, scaling down the size by half until it
1236  * reaches a minimum limit (1M), beyond which we give up.
1237  */
1238 static struct etr_buf *
1239 alloc_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1240           int nr_pages, void **pages, bool snapshot)
1241 {
1242     int node;
1243     struct etr_buf *etr_buf;
1244     unsigned long size;
1245 
1246     node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
1247     /*
1248      * Try to match the perf ring buffer size if it is larger
1249      * than the size requested via sysfs.
1250      */
1251     if ((nr_pages << PAGE_SHIFT) > drvdata->size) {
1252         etr_buf = tmc_alloc_etr_buf(drvdata, (nr_pages << PAGE_SHIFT),
1253                         0, node, NULL);
1254         if (!IS_ERR(etr_buf))
1255             goto done;
1256     }
1257 
1258     /*
1259      * Else switch to configured size for this ETR
1260      * and scale down until we hit the minimum limit.
1261      */
1262     size = drvdata->size;
1263     do {
1264         etr_buf = tmc_alloc_etr_buf(drvdata, size, 0, node, NULL);
1265         if (!IS_ERR(etr_buf))
1266             goto done;
1267         size /= 2;
1268     } while (size >= TMC_ETR_PERF_MIN_BUF_SIZE);
1269 
1270     return ERR_PTR(-ENOMEM);
1271 
1272 done:
1273     return etr_buf;
1274 }
1275 
1276 static struct etr_buf *
1277 get_perf_etr_buf_cpu_wide(struct tmc_drvdata *drvdata,
1278               struct perf_event *event, int nr_pages,
1279               void **pages, bool snapshot)
1280 {
1281     int ret;
1282     pid_t pid = task_pid_nr(event->owner);
1283     struct etr_buf *etr_buf;
1284 
1285 retry:
1286     /*
1287      * An etr_perf_buffer is associated with an event and holds a reference
1288      * to the AUX ring buffer that was created for that event.  In CPU-wide
1289      * N:1 mode multiple events (one per CPU), each with its own AUX ring
1290      * buffer, share a sink.  As such an etr_perf_buffer is created for each
1291      * event but a single etr_buf associated with the ETR is shared between
1292      * them.  The last event in a trace session will copy the content of the
1293      * etr_buf to its AUX ring buffer.  Ring buffer associated to other
1294      * events are simply not used an freed as events are destoyed.  We still
1295      * need to allocate a ring buffer for each event since we don't know
1296      * which event will be last.
1297      */
1298 
1299     /*
1300      * The first thing to do here is check if an etr_buf has already been
1301      * allocated for this session.  If so it is shared with this event,
1302      * otherwise it is created.
1303      */
1304     mutex_lock(&drvdata->idr_mutex);
1305     etr_buf = idr_find(&drvdata->idr, pid);
1306     if (etr_buf) {
1307         refcount_inc(&etr_buf->refcount);
1308         mutex_unlock(&drvdata->idr_mutex);
1309         return etr_buf;
1310     }
1311 
1312     /* If we made it here no buffer has been allocated, do so now. */
1313     mutex_unlock(&drvdata->idr_mutex);
1314 
1315     etr_buf = alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1316     if (IS_ERR(etr_buf))
1317         return etr_buf;
1318 
1319     /* Now that we have a buffer, add it to the IDR. */
1320     mutex_lock(&drvdata->idr_mutex);
1321     ret = idr_alloc(&drvdata->idr, etr_buf, pid, pid + 1, GFP_KERNEL);
1322     mutex_unlock(&drvdata->idr_mutex);
1323 
1324     /* Another event with this session ID has allocated this buffer. */
1325     if (ret == -ENOSPC) {
1326         tmc_free_etr_buf(etr_buf);
1327         goto retry;
1328     }
1329 
1330     /* The IDR can't allocate room for a new session, abandon ship. */
1331     if (ret == -ENOMEM) {
1332         tmc_free_etr_buf(etr_buf);
1333         return ERR_PTR(ret);
1334     }
1335 
1336 
1337     return etr_buf;
1338 }
1339 
1340 static struct etr_buf *
1341 get_perf_etr_buf_per_thread(struct tmc_drvdata *drvdata,
1342                 struct perf_event *event, int nr_pages,
1343                 void **pages, bool snapshot)
1344 {
1345     /*
1346      * In per-thread mode the etr_buf isn't shared, so just go ahead
1347      * with memory allocation.
1348      */
1349     return alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1350 }
1351 
1352 static struct etr_buf *
1353 get_perf_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1354          int nr_pages, void **pages, bool snapshot)
1355 {
1356     if (event->cpu == -1)
1357         return get_perf_etr_buf_per_thread(drvdata, event, nr_pages,
1358                            pages, snapshot);
1359 
1360     return get_perf_etr_buf_cpu_wide(drvdata, event, nr_pages,
1361                      pages, snapshot);
1362 }
1363 
1364 static struct etr_perf_buffer *
1365 tmc_etr_setup_perf_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1366                int nr_pages, void **pages, bool snapshot)
1367 {
1368     int node;
1369     struct etr_buf *etr_buf;
1370     struct etr_perf_buffer *etr_perf;
1371 
1372     node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
1373 
1374     etr_perf = kzalloc_node(sizeof(*etr_perf), GFP_KERNEL, node);
1375     if (!etr_perf)
1376         return ERR_PTR(-ENOMEM);
1377 
1378     etr_buf = get_perf_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1379     if (!IS_ERR(etr_buf))
1380         goto done;
1381 
1382     kfree(etr_perf);
1383     return ERR_PTR(-ENOMEM);
1384 
1385 done:
1386     /*
1387      * Keep a reference to the ETR this buffer has been allocated for
1388      * in order to have access to the IDR in tmc_free_etr_buffer().
1389      */
1390     etr_perf->drvdata = drvdata;
1391     etr_perf->etr_buf = etr_buf;
1392 
1393     return etr_perf;
1394 }
1395 
1396 
1397 static void *tmc_alloc_etr_buffer(struct coresight_device *csdev,
1398                   struct perf_event *event, void **pages,
1399                   int nr_pages, bool snapshot)
1400 {
1401     struct etr_perf_buffer *etr_perf;
1402     struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1403 
1404     etr_perf = tmc_etr_setup_perf_buf(drvdata, event,
1405                       nr_pages, pages, snapshot);
1406     if (IS_ERR(etr_perf)) {
1407         dev_dbg(&csdev->dev, "Unable to allocate ETR buffer\n");
1408         return NULL;
1409     }
1410 
1411     etr_perf->pid = task_pid_nr(event->owner);
1412     etr_perf->snapshot = snapshot;
1413     etr_perf->nr_pages = nr_pages;
1414     etr_perf->pages = pages;
1415 
1416     return etr_perf;
1417 }
1418 
1419 static void tmc_free_etr_buffer(void *config)
1420 {
1421     struct etr_perf_buffer *etr_perf = config;
1422     struct tmc_drvdata *drvdata = etr_perf->drvdata;
1423     struct etr_buf *buf, *etr_buf = etr_perf->etr_buf;
1424 
1425     if (!etr_buf)
1426         goto free_etr_perf_buffer;
1427 
1428     mutex_lock(&drvdata->idr_mutex);
1429     /* If we are not the last one to use the buffer, don't touch it. */
1430     if (!refcount_dec_and_test(&etr_buf->refcount)) {
1431         mutex_unlock(&drvdata->idr_mutex);
1432         goto free_etr_perf_buffer;
1433     }
1434 
1435     /* We are the last one, remove from the IDR and free the buffer. */
1436     buf = idr_remove(&drvdata->idr, etr_perf->pid);
1437     mutex_unlock(&drvdata->idr_mutex);
1438 
1439     /*
1440      * Something went very wrong if the buffer associated with this ID
1441      * is not the same in the IDR.  Leak to avoid use after free.
1442      */
1443     if (buf && WARN_ON(buf != etr_buf))
1444         goto free_etr_perf_buffer;
1445 
1446     tmc_free_etr_buf(etr_perf->etr_buf);
1447 
1448 free_etr_perf_buffer:
1449     kfree(etr_perf);
1450 }
1451 
1452 /*
1453  * tmc_etr_sync_perf_buffer: Copy the actual trace data from the hardware
1454  * buffer to the perf ring buffer.
1455  */
1456 static void tmc_etr_sync_perf_buffer(struct etr_perf_buffer *etr_perf,
1457                      unsigned long head,
1458                      unsigned long src_offset,
1459                      unsigned long to_copy)
1460 {
1461     long bytes;
1462     long pg_idx, pg_offset;
1463     char **dst_pages, *src_buf;
1464     struct etr_buf *etr_buf = etr_perf->etr_buf;
1465 
1466     head = PERF_IDX2OFF(head, etr_perf);
1467     pg_idx = head >> PAGE_SHIFT;
1468     pg_offset = head & (PAGE_SIZE - 1);
1469     dst_pages = (char **)etr_perf->pages;
1470 
1471     while (to_copy > 0) {
1472         /*
1473          * In one iteration, we can copy minimum of :
1474          *  1) what is available in the source buffer,
1475          *  2) what is available in the source buffer, before it
1476          *     wraps around.
1477          *  3) what is available in the destination page.
1478          * in one iteration.
1479          */
1480         if (src_offset >= etr_buf->size)
1481             src_offset -= etr_buf->size;
1482         bytes = tmc_etr_buf_get_data(etr_buf, src_offset, to_copy,
1483                          &src_buf);
1484         if (WARN_ON_ONCE(bytes <= 0))
1485             break;
1486         bytes = min(bytes, (long)(PAGE_SIZE - pg_offset));
1487 
1488         memcpy(dst_pages[pg_idx] + pg_offset, src_buf, bytes);
1489 
1490         to_copy -= bytes;
1491 
1492         /* Move destination pointers */
1493         pg_offset += bytes;
1494         if (pg_offset == PAGE_SIZE) {
1495             pg_offset = 0;
1496             if (++pg_idx == etr_perf->nr_pages)
1497                 pg_idx = 0;
1498         }
1499 
1500         /* Move source pointers */
1501         src_offset += bytes;
1502     }
1503 }
1504 
1505 /*
1506  * tmc_update_etr_buffer : Update the perf ring buffer with the
1507  * available trace data. We use software double buffering at the moment.
1508  *
1509  * TODO: Add support for reusing the perf ring buffer.
1510  */
1511 static unsigned long
1512 tmc_update_etr_buffer(struct coresight_device *csdev,
1513               struct perf_output_handle *handle,
1514               void *config)
1515 {
1516     bool lost = false;
1517     unsigned long flags, offset, size = 0;
1518     struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1519     struct etr_perf_buffer *etr_perf = config;
1520     struct etr_buf *etr_buf = etr_perf->etr_buf;
1521 
1522     spin_lock_irqsave(&drvdata->spinlock, flags);
1523 
1524     /* Don't do anything if another tracer is using this sink */
1525     if (atomic_read(csdev->refcnt) != 1) {
1526         spin_unlock_irqrestore(&drvdata->spinlock, flags);
1527         goto out;
1528     }
1529 
1530     if (WARN_ON(drvdata->perf_buf != etr_buf)) {
1531         lost = true;
1532         spin_unlock_irqrestore(&drvdata->spinlock, flags);
1533         goto out;
1534     }
1535 
1536     CS_UNLOCK(drvdata->base);
1537 
1538     tmc_flush_and_stop(drvdata);
1539     tmc_sync_etr_buf(drvdata);
1540 
1541     CS_LOCK(drvdata->base);
1542     spin_unlock_irqrestore(&drvdata->spinlock, flags);
1543 
1544     lost = etr_buf->full;
1545     offset = etr_buf->offset;
1546     size = etr_buf->len;
1547 
1548     /*
1549      * The ETR buffer may be bigger than the space available in the
1550      * perf ring buffer (handle->size).  If so advance the offset so that we
1551      * get the latest trace data.  In snapshot mode none of that matters
1552      * since we are expected to clobber stale data in favour of the latest
1553      * traces.
1554      */
1555     if (!etr_perf->snapshot && size > handle->size) {
1556         u32 mask = tmc_get_memwidth_mask(drvdata);
1557 
1558         /*
1559          * Make sure the new size is aligned in accordance with the
1560          * requirement explained in function tmc_get_memwidth_mask().
1561          */
1562         size = handle->size & mask;
1563         offset = etr_buf->offset + etr_buf->len - size;
1564 
1565         if (offset >= etr_buf->size)
1566             offset -= etr_buf->size;
1567         lost = true;
1568     }
1569 
1570     /* Insert barrier packets at the beginning, if there was an overflow */
1571     if (lost)
1572         tmc_etr_buf_insert_barrier_packet(etr_buf, offset);
1573     tmc_etr_sync_perf_buffer(etr_perf, handle->head, offset, size);
1574 
1575     /*
1576      * In snapshot mode we simply increment the head by the number of byte
1577      * that were written.  User space will figure out how many bytes to get
1578      * from the AUX buffer based on the position of the head.
1579      */
1580     if (etr_perf->snapshot)
1581         handle->head += size;
1582 
1583     /*
1584      * Ensure that the AUX trace data is visible before the aux_head
1585      * is updated via perf_aux_output_end(), as expected by the
1586      * perf ring buffer.
1587      */
1588     smp_wmb();
1589 
1590 out:
1591     /*
1592      * Don't set the TRUNCATED flag in snapshot mode because 1) the
1593      * captured buffer is expected to be truncated and 2) a full buffer
1594      * prevents the event from being re-enabled by the perf core,
1595      * resulting in stale data being send to user space.
1596      */
1597     if (!etr_perf->snapshot && lost)
1598         perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
1599     return size;
1600 }
1601 
1602 static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data)
1603 {
1604     int rc = 0;
1605     pid_t pid;
1606     unsigned long flags;
1607     struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1608     struct perf_output_handle *handle = data;
1609     struct etr_perf_buffer *etr_perf = etm_perf_sink_config(handle);
1610 
1611     spin_lock_irqsave(&drvdata->spinlock, flags);
1612      /* Don't use this sink if it is already claimed by sysFS */
1613     if (drvdata->mode == CS_MODE_SYSFS) {
1614         rc = -EBUSY;
1615         goto unlock_out;
1616     }
1617 
1618     if (WARN_ON(!etr_perf || !etr_perf->etr_buf)) {
1619         rc = -EINVAL;
1620         goto unlock_out;
1621     }
1622 
1623     /* Get a handle on the pid of the process to monitor */
1624     pid = etr_perf->pid;
1625 
1626     /* Do not proceed if this device is associated with another session */
1627     if (drvdata->pid != -1 && drvdata->pid != pid) {
1628         rc = -EBUSY;
1629         goto unlock_out;
1630     }
1631 
1632     /*
1633      * No HW configuration is needed if the sink is already in
1634      * use for this session.
1635      */
1636     if (drvdata->pid == pid) {
1637         atomic_inc(csdev->refcnt);
1638         goto unlock_out;
1639     }
1640 
1641     rc = tmc_etr_enable_hw(drvdata, etr_perf->etr_buf);
1642     if (!rc) {
1643         /* Associate with monitored process. */
1644         drvdata->pid = pid;
1645         drvdata->mode = CS_MODE_PERF;
1646         drvdata->perf_buf = etr_perf->etr_buf;
1647         atomic_inc(csdev->refcnt);
1648     }
1649 
1650 unlock_out:
1651     spin_unlock_irqrestore(&drvdata->spinlock, flags);
1652     return rc;
1653 }
1654 
1655 static int tmc_enable_etr_sink(struct coresight_device *csdev,
1656                    u32 mode, void *data)
1657 {
1658     switch (mode) {
1659     case CS_MODE_SYSFS:
1660         return tmc_enable_etr_sink_sysfs(csdev);
1661     case CS_MODE_PERF:
1662         return tmc_enable_etr_sink_perf(csdev, data);
1663     }
1664 
1665     /* We shouldn't be here */
1666     return -EINVAL;
1667 }
1668 
1669 static int tmc_disable_etr_sink(struct coresight_device *csdev)
1670 {
1671     unsigned long flags;
1672     struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1673 
1674     spin_lock_irqsave(&drvdata->spinlock, flags);
1675 
1676     if (drvdata->reading) {
1677         spin_unlock_irqrestore(&drvdata->spinlock, flags);
1678         return -EBUSY;
1679     }
1680 
1681     if (atomic_dec_return(csdev->refcnt)) {
1682         spin_unlock_irqrestore(&drvdata->spinlock, flags);
1683         return -EBUSY;
1684     }
1685 
1686     /* Complain if we (somehow) got out of sync */
1687     WARN_ON_ONCE(drvdata->mode == CS_MODE_DISABLED);
1688     tmc_etr_disable_hw(drvdata);
1689     /* Dissociate from monitored process. */
1690     drvdata->pid = -1;
1691     drvdata->mode = CS_MODE_DISABLED;
1692     /* Reset perf specific data */
1693     drvdata->perf_buf = NULL;
1694 
1695     spin_unlock_irqrestore(&drvdata->spinlock, flags);
1696 
1697     dev_dbg(&csdev->dev, "TMC-ETR disabled\n");
1698     return 0;
1699 }
1700 
1701 static const struct coresight_ops_sink tmc_etr_sink_ops = {
1702     .enable     = tmc_enable_etr_sink,
1703     .disable    = tmc_disable_etr_sink,
1704     .alloc_buffer   = tmc_alloc_etr_buffer,
1705     .update_buffer  = tmc_update_etr_buffer,
1706     .free_buffer    = tmc_free_etr_buffer,
1707 };
1708 
1709 const struct coresight_ops tmc_etr_cs_ops = {
1710     .sink_ops   = &tmc_etr_sink_ops,
1711 };
1712 
1713 int tmc_read_prepare_etr(struct tmc_drvdata *drvdata)
1714 {
1715     int ret = 0;
1716     unsigned long flags;
1717 
1718     /* config types are set a boot time and never change */
1719     if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1720         return -EINVAL;
1721 
1722     spin_lock_irqsave(&drvdata->spinlock, flags);
1723     if (drvdata->reading) {
1724         ret = -EBUSY;
1725         goto out;
1726     }
1727 
1728     /*
1729      * We can safely allow reads even if the ETR is operating in PERF mode,
1730      * since the sysfs session is captured in mode specific data.
1731      * If drvdata::sysfs_data is NULL the trace data has been read already.
1732      */
1733     if (!drvdata->sysfs_buf) {
1734         ret = -EINVAL;
1735         goto out;
1736     }
1737 
1738     /* Disable the TMC if we are trying to read from a running session. */
1739     if (drvdata->mode == CS_MODE_SYSFS)
1740         __tmc_etr_disable_hw(drvdata);
1741 
1742     drvdata->reading = true;
1743 out:
1744     spin_unlock_irqrestore(&drvdata->spinlock, flags);
1745 
1746     return ret;
1747 }
1748 
1749 int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata)
1750 {
1751     unsigned long flags;
1752     struct etr_buf *sysfs_buf = NULL;
1753 
1754     /* config types are set a boot time and never change */
1755     if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1756         return -EINVAL;
1757 
1758     spin_lock_irqsave(&drvdata->spinlock, flags);
1759 
1760     /* RE-enable the TMC if need be */
1761     if (drvdata->mode == CS_MODE_SYSFS) {
1762         /*
1763          * The trace run will continue with the same allocated trace
1764          * buffer. Since the tracer is still enabled drvdata::buf can't
1765          * be NULL.
1766          */
1767         __tmc_etr_enable_hw(drvdata);
1768     } else {
1769         /*
1770          * The ETR is not tracing and the buffer was just read.
1771          * As such prepare to free the trace buffer.
1772          */
1773         sysfs_buf = drvdata->sysfs_buf;
1774         drvdata->sysfs_buf = NULL;
1775     }
1776 
1777     drvdata->reading = false;
1778     spin_unlock_irqrestore(&drvdata->spinlock, flags);
1779 
1780     /* Free allocated memory out side of the spinlock */
1781     if (sysfs_buf)
1782         tmc_etr_free_sysfs_buf(sysfs_buf);
1783 
1784     return 0;
1785 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!