Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
0002 /* Copyright (C) 2015-2018 Netronome Systems, Inc. */
0003 
0004 /*
0005  * nfp6000_pcie.c
0006  * Authors: Jakub Kicinski <jakub.kicinski@netronome.com>
0007  *          Jason McMullan <jason.mcmullan@netronome.com>
0008  *          Rolf Neugebauer <rolf.neugebauer@netronome.com>
0009  *
0010  * Multiplexes the NFP BARs between NFP internal resources and
0011  * implements the PCIe specific interface for generic CPP bus access.
0012  *
0013  * The BARs are managed with refcounts and are allocated/acquired
0014  * using target, token and offset/size matching.  The generic CPP bus
0015  * abstraction builds upon this BAR interface.
0016  */
0017 
0018 #include <asm/unaligned.h>
0019 #include <linux/kernel.h>
0020 #include <linux/module.h>
0021 #include <linux/kref.h>
0022 #include <linux/io.h>
0023 #include <linux/delay.h>
0024 #include <linux/interrupt.h>
0025 #include <linux/sort.h>
0026 #include <linux/sched.h>
0027 #include <linux/types.h>
0028 #include <linux/pci.h>
0029 
0030 #include "nfp_cpp.h"
0031 #include "nfp_dev.h"
0032 
0033 #include "nfp6000/nfp6000.h"
0034 
0035 #include "nfp6000_pcie.h"
0036 
0037 #define NFP_PCIE_BAR(_pf)   (0x30000 + ((_pf) & 7) * 0xc0)
0038 #define NFP_PCIE_BAR_EXPLICIT_BAR0(_x, _y) \
0039     (0x00000080 + (0x40 * ((_x) & 0x3)) + (0x10 * ((_y) & 0x3)))
0040 #define   NFP_PCIE_BAR_EXPLICIT_BAR0_SignalType(_x)     (((_x) & 0x3) << 30)
0041 #define   NFP_PCIE_BAR_EXPLICIT_BAR0_SignalType_of(_x)  (((_x) >> 30) & 0x3)
0042 #define   NFP_PCIE_BAR_EXPLICIT_BAR0_Token(_x)          (((_x) & 0x3) << 28)
0043 #define   NFP_PCIE_BAR_EXPLICIT_BAR0_Token_of(_x)       (((_x) >> 28) & 0x3)
0044 #define   NFP_PCIE_BAR_EXPLICIT_BAR0_Address(_x)        (((_x) & 0xffffff) << 0)
0045 #define   NFP_PCIE_BAR_EXPLICIT_BAR0_Address_of(_x)     (((_x) >> 0) & 0xffffff)
0046 #define NFP_PCIE_BAR_EXPLICIT_BAR1(_x, _y) \
0047     (0x00000084 + (0x40 * ((_x) & 0x3)) + (0x10 * ((_y) & 0x3)))
0048 #define   NFP_PCIE_BAR_EXPLICIT_BAR1_SignalRef(_x)      (((_x) & 0x7f) << 24)
0049 #define   NFP_PCIE_BAR_EXPLICIT_BAR1_SignalRef_of(_x)   (((_x) >> 24) & 0x7f)
0050 #define   NFP_PCIE_BAR_EXPLICIT_BAR1_DataMaster(_x)     (((_x) & 0x3ff) << 14)
0051 #define   NFP_PCIE_BAR_EXPLICIT_BAR1_DataMaster_of(_x)  (((_x) >> 14) & 0x3ff)
0052 #define   NFP_PCIE_BAR_EXPLICIT_BAR1_DataRef(_x)        (((_x) & 0x3fff) << 0)
0053 #define   NFP_PCIE_BAR_EXPLICIT_BAR1_DataRef_of(_x)     (((_x) >> 0) & 0x3fff)
0054 #define NFP_PCIE_BAR_EXPLICIT_BAR2(_x, _y) \
0055     (0x00000088 + (0x40 * ((_x) & 0x3)) + (0x10 * ((_y) & 0x3)))
0056 #define   NFP_PCIE_BAR_EXPLICIT_BAR2_Target(_x)         (((_x) & 0xf) << 28)
0057 #define   NFP_PCIE_BAR_EXPLICIT_BAR2_Target_of(_x)      (((_x) >> 28) & 0xf)
0058 #define   NFP_PCIE_BAR_EXPLICIT_BAR2_Action(_x)         (((_x) & 0x1f) << 23)
0059 #define   NFP_PCIE_BAR_EXPLICIT_BAR2_Action_of(_x)      (((_x) >> 23) & 0x1f)
0060 #define   NFP_PCIE_BAR_EXPLICIT_BAR2_Length(_x)         (((_x) & 0x1f) << 18)
0061 #define   NFP_PCIE_BAR_EXPLICIT_BAR2_Length_of(_x)      (((_x) >> 18) & 0x1f)
0062 #define   NFP_PCIE_BAR_EXPLICIT_BAR2_ByteMask(_x)       (((_x) & 0xff) << 10)
0063 #define   NFP_PCIE_BAR_EXPLICIT_BAR2_ByteMask_of(_x)    (((_x) >> 10) & 0xff)
0064 #define   NFP_PCIE_BAR_EXPLICIT_BAR2_SignalMaster(_x)   (((_x) & 0x3ff) << 0)
0065 #define   NFP_PCIE_BAR_EXPLICIT_BAR2_SignalMaster_of(_x) (((_x) >> 0) & 0x3ff)
0066 
0067 #define   NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress(_x)  (((_x) & 0x1f) << 16)
0068 #define   NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress_of(_x) (((_x) >> 16) & 0x1f)
0069 #define   NFP_PCIE_BAR_PCIE2CPP_BaseAddress(_x)         (((_x) & 0xffff) << 0)
0070 #define   NFP_PCIE_BAR_PCIE2CPP_BaseAddress_of(_x)      (((_x) >> 0) & 0xffff)
0071 #define   NFP_PCIE_BAR_PCIE2CPP_LengthSelect(_x)        (((_x) & 0x3) << 27)
0072 #define   NFP_PCIE_BAR_PCIE2CPP_LengthSelect_of(_x)     (((_x) >> 27) & 0x3)
0073 #define     NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT    0
0074 #define     NFP_PCIE_BAR_PCIE2CPP_LengthSelect_64BIT    1
0075 #define     NFP_PCIE_BAR_PCIE2CPP_LengthSelect_0BYTE    3
0076 #define   NFP_PCIE_BAR_PCIE2CPP_MapType(_x)             (((_x) & 0x7) << 29)
0077 #define   NFP_PCIE_BAR_PCIE2CPP_MapType_of(_x)          (((_x) >> 29) & 0x7)
0078 #define     NFP_PCIE_BAR_PCIE2CPP_MapType_FIXED         0
0079 #define     NFP_PCIE_BAR_PCIE2CPP_MapType_BULK          1
0080 #define     NFP_PCIE_BAR_PCIE2CPP_MapType_TARGET        2
0081 #define     NFP_PCIE_BAR_PCIE2CPP_MapType_GENERAL       3
0082 #define     NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT0     4
0083 #define     NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT1     5
0084 #define     NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT2     6
0085 #define     NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT3     7
0086 #define   NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(_x)  (((_x) & 0xf) << 23)
0087 #define   NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress_of(_x) (((_x) >> 23) & 0xf)
0088 #define   NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress(_x)   (((_x) & 0x3) << 21)
0089 #define   NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress_of(_x) (((_x) >> 21) & 0x3)
0090 #define NFP_PCIE_EM                                     0x020000
0091 #define NFP_PCIE_SRAM                                   0x000000
0092 
0093 /* Minimal size of the PCIe cfg memory we depend on being mapped,
0094  * queue controller and DMA controller don't have to be covered.
0095  */
0096 #define NFP_PCI_MIN_MAP_SIZE                0x080000
0097 
0098 #define NFP_PCIE_P2C_FIXED_SIZE(bar)               (1 << (bar)->bitsize)
0099 #define NFP_PCIE_P2C_BULK_SIZE(bar)                (1 << (bar)->bitsize)
0100 #define NFP_PCIE_P2C_GENERAL_TARGET_OFFSET(bar, x) ((x) << ((bar)->bitsize - 2))
0101 #define NFP_PCIE_P2C_GENERAL_TOKEN_OFFSET(bar, x) ((x) << ((bar)->bitsize - 4))
0102 #define NFP_PCIE_P2C_GENERAL_SIZE(bar)             (1 << ((bar)->bitsize - 4))
0103 
0104 #define NFP_PCIE_P2C_EXPBAR_OFFSET(bar_index)       ((bar_index) * 4)
0105 
0106 /* The number of explicit BARs to reserve.
0107  * Minimum is 0, maximum is 4 on the NFP6000.
0108  * The NFP3800 can have only one per PF.
0109  */
0110 #define NFP_PCIE_EXPLICIT_BARS      2
0111 
0112 struct nfp6000_pcie;
0113 struct nfp6000_area_priv;
0114 
0115 /**
0116  * struct nfp_bar - describes BAR configuration and usage
0117  * @nfp:    backlink to owner
0118  * @barcfg: cached contents of BAR config CSR
0119  * @base:   the BAR's base CPP offset
0120  * @mask:       mask for the BAR aperture (read only)
0121  * @bitsize:    bitsize of BAR aperture (read only)
0122  * @index:  index of the BAR
0123  * @refcnt: number of current users
0124  * @iomem:  mapped IO memory
0125  * @resource:   iomem resource window
0126  */
0127 struct nfp_bar {
0128     struct nfp6000_pcie *nfp;
0129     u32 barcfg;
0130     u64 base;          /* CPP address base */
0131     u64 mask;          /* Bit mask of the bar */
0132     u32 bitsize;       /* Bit size of the bar */
0133     int index;
0134     atomic_t refcnt;
0135 
0136     void __iomem *iomem;
0137     struct resource *resource;
0138 };
0139 
0140 #define NFP_PCI_BAR_MAX    (PCI_64BIT_BAR_COUNT * 8)
0141 
0142 struct nfp6000_pcie {
0143     struct pci_dev *pdev;
0144     struct device *dev;
0145     const struct nfp_dev_info *dev_info;
0146 
0147     /* PCI BAR management */
0148     spinlock_t bar_lock;        /* Protect the PCI2CPP BAR cache */
0149     int bars;
0150     struct nfp_bar bar[NFP_PCI_BAR_MAX];
0151     wait_queue_head_t bar_waiters;
0152 
0153     /* Reserved BAR access */
0154     struct {
0155         void __iomem *csr;
0156         void __iomem *em;
0157         void __iomem *expl[4];
0158     } iomem;
0159 
0160     /* Explicit IO access */
0161     struct {
0162         struct mutex mutex; /* Lock access to this explicit group */
0163         u8 master_id;
0164         u8 signal_ref;
0165         void __iomem *data;
0166         struct {
0167             void __iomem *addr;
0168             int bitsize;
0169             int free[4];
0170         } group[4];
0171     } expl;
0172 };
0173 
0174 static u32 nfp_bar_maptype(struct nfp_bar *bar)
0175 {
0176     return NFP_PCIE_BAR_PCIE2CPP_MapType_of(bar->barcfg);
0177 }
0178 
0179 static resource_size_t nfp_bar_resource_len(struct nfp_bar *bar)
0180 {
0181     return pci_resource_len(bar->nfp->pdev, (bar->index / 8) * 2) / 8;
0182 }
0183 
0184 static resource_size_t nfp_bar_resource_start(struct nfp_bar *bar)
0185 {
0186     return pci_resource_start(bar->nfp->pdev, (bar->index / 8) * 2)
0187         + nfp_bar_resource_len(bar) * (bar->index & 7);
0188 }
0189 
0190 #define TARGET_WIDTH_32    4
0191 #define TARGET_WIDTH_64    8
0192 
0193 static int
0194 compute_bar(const struct nfp6000_pcie *nfp, const struct nfp_bar *bar,
0195         u32 *bar_config, u64 *bar_base,
0196         int tgt, int act, int tok, u64 offset, size_t size, int width)
0197 {
0198     int bitsize;
0199     u32 newcfg;
0200 
0201     if (tgt >= NFP_CPP_NUM_TARGETS)
0202         return -EINVAL;
0203 
0204     switch (width) {
0205     case 8:
0206         newcfg = NFP_PCIE_BAR_PCIE2CPP_LengthSelect(
0207             NFP_PCIE_BAR_PCIE2CPP_LengthSelect_64BIT);
0208         break;
0209     case 4:
0210         newcfg = NFP_PCIE_BAR_PCIE2CPP_LengthSelect(
0211             NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT);
0212         break;
0213     case 0:
0214         newcfg = NFP_PCIE_BAR_PCIE2CPP_LengthSelect(
0215             NFP_PCIE_BAR_PCIE2CPP_LengthSelect_0BYTE);
0216         break;
0217     default:
0218         return -EINVAL;
0219     }
0220 
0221     if (act != NFP_CPP_ACTION_RW && act != 0) {
0222         /* Fixed CPP mapping with specific action */
0223         u64 mask = ~(NFP_PCIE_P2C_FIXED_SIZE(bar) - 1);
0224 
0225         newcfg |= NFP_PCIE_BAR_PCIE2CPP_MapType(
0226               NFP_PCIE_BAR_PCIE2CPP_MapType_FIXED);
0227         newcfg |= NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(tgt);
0228         newcfg |= NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress(act);
0229         newcfg |= NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress(tok);
0230 
0231         if ((offset & mask) != ((offset + size - 1) & mask))
0232             return -EINVAL;
0233         offset &= mask;
0234 
0235         bitsize = 40 - 16;
0236     } else {
0237         u64 mask = ~(NFP_PCIE_P2C_BULK_SIZE(bar) - 1);
0238 
0239         /* Bulk mapping */
0240         newcfg |= NFP_PCIE_BAR_PCIE2CPP_MapType(
0241             NFP_PCIE_BAR_PCIE2CPP_MapType_BULK);
0242         newcfg |= NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(tgt);
0243         newcfg |= NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress(tok);
0244 
0245         if ((offset & mask) != ((offset + size - 1) & mask))
0246             return -EINVAL;
0247 
0248         offset &= mask;
0249 
0250         bitsize = 40 - 21;
0251     }
0252 
0253     if (bar->bitsize < bitsize)
0254         return -EINVAL;
0255 
0256     newcfg |= offset >> bitsize;
0257 
0258     if (bar_base)
0259         *bar_base = offset;
0260 
0261     if (bar_config)
0262         *bar_config = newcfg;
0263 
0264     return 0;
0265 }
0266 
0267 static int
0268 nfp6000_bar_write(struct nfp6000_pcie *nfp, struct nfp_bar *bar, u32 newcfg)
0269 {
0270     unsigned int xbar;
0271 
0272     xbar = NFP_PCIE_P2C_EXPBAR_OFFSET(bar->index);
0273 
0274     if (nfp->iomem.csr) {
0275         writel(newcfg, nfp->iomem.csr + xbar);
0276         /* Readback to ensure BAR is flushed */
0277         readl(nfp->iomem.csr + xbar);
0278     } else {
0279         xbar += nfp->dev_info->pcie_cfg_expbar_offset;
0280         pci_write_config_dword(nfp->pdev, xbar, newcfg);
0281     }
0282 
0283     bar->barcfg = newcfg;
0284 
0285     return 0;
0286 }
0287 
0288 static int
0289 reconfigure_bar(struct nfp6000_pcie *nfp, struct nfp_bar *bar,
0290         int tgt, int act, int tok, u64 offset, size_t size, int width)
0291 {
0292     u64 newbase;
0293     u32 newcfg;
0294     int err;
0295 
0296     err = compute_bar(nfp, bar, &newcfg, &newbase,
0297               tgt, act, tok, offset, size, width);
0298     if (err)
0299         return err;
0300 
0301     bar->base = newbase;
0302 
0303     return nfp6000_bar_write(nfp, bar, newcfg);
0304 }
0305 
0306 /* Check if BAR can be used with the given parameters. */
0307 static int matching_bar(struct nfp_bar *bar, u32 tgt, u32 act, u32 tok,
0308             u64 offset, size_t size, int width)
0309 {
0310     int bartgt, baract, bartok;
0311     int barwidth;
0312     u32 maptype;
0313 
0314     maptype = NFP_PCIE_BAR_PCIE2CPP_MapType_of(bar->barcfg);
0315     bartgt = NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress_of(bar->barcfg);
0316     bartok = NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress_of(bar->barcfg);
0317     baract = NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress_of(bar->barcfg);
0318 
0319     barwidth = NFP_PCIE_BAR_PCIE2CPP_LengthSelect_of(bar->barcfg);
0320     switch (barwidth) {
0321     case NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT:
0322         barwidth = 4;
0323         break;
0324     case NFP_PCIE_BAR_PCIE2CPP_LengthSelect_64BIT:
0325         barwidth = 8;
0326         break;
0327     case NFP_PCIE_BAR_PCIE2CPP_LengthSelect_0BYTE:
0328         barwidth = 0;
0329         break;
0330     default:
0331         barwidth = -1;
0332         break;
0333     }
0334 
0335     switch (maptype) {
0336     case NFP_PCIE_BAR_PCIE2CPP_MapType_TARGET:
0337         bartok = -1;
0338         fallthrough;
0339     case NFP_PCIE_BAR_PCIE2CPP_MapType_BULK:
0340         baract = NFP_CPP_ACTION_RW;
0341         if (act == 0)
0342             act = NFP_CPP_ACTION_RW;
0343         fallthrough;
0344     case NFP_PCIE_BAR_PCIE2CPP_MapType_FIXED:
0345         break;
0346     default:
0347         /* We don't match explicit bars through the area interface */
0348         return 0;
0349     }
0350 
0351     /* Make sure to match up the width */
0352     if (barwidth != width)
0353         return 0;
0354 
0355     if ((bartgt < 0 || bartgt == tgt) &&
0356         (bartok < 0 || bartok == tok) &&
0357         (baract == act) &&
0358         bar->base <= offset &&
0359         (bar->base + (1 << bar->bitsize)) >= (offset + size))
0360         return 1;
0361 
0362     /* No match */
0363     return 0;
0364 }
0365 
0366 static int
0367 find_matching_bar(struct nfp6000_pcie *nfp,
0368           u32 tgt, u32 act, u32 tok, u64 offset, size_t size, int width)
0369 {
0370     int n;
0371 
0372     for (n = 0; n < nfp->bars; n++) {
0373         struct nfp_bar *bar = &nfp->bar[n];
0374 
0375         if (matching_bar(bar, tgt, act, tok, offset, size, width))
0376             return n;
0377     }
0378 
0379     return -1;
0380 }
0381 
0382 /* Return EAGAIN if no resource is available */
0383 static int
0384 find_unused_bar_noblock(const struct nfp6000_pcie *nfp,
0385             int tgt, int act, int tok,
0386             u64 offset, size_t size, int width)
0387 {
0388     int n, busy = 0;
0389 
0390     for (n = 0; n < nfp->bars; n++) {
0391         const struct nfp_bar *bar = &nfp->bar[n];
0392         int err;
0393 
0394         if (!bar->bitsize)
0395             continue;
0396 
0397         /* Just check to see if we can make it fit... */
0398         err = compute_bar(nfp, bar, NULL, NULL,
0399                   tgt, act, tok, offset, size, width);
0400         if (err)
0401             continue;
0402 
0403         if (!atomic_read(&bar->refcnt))
0404             return n;
0405 
0406         busy++;
0407     }
0408 
0409     if (WARN(!busy, "No suitable BAR found for request tgt:0x%x act:0x%x tok:0x%x off:0x%llx size:%zd width:%d\n",
0410          tgt, act, tok, offset, size, width))
0411         return -EINVAL;
0412 
0413     return -EAGAIN;
0414 }
0415 
0416 static int
0417 find_unused_bar_and_lock(struct nfp6000_pcie *nfp,
0418              int tgt, int act, int tok,
0419              u64 offset, size_t size, int width)
0420 {
0421     unsigned long flags;
0422     int n;
0423 
0424     spin_lock_irqsave(&nfp->bar_lock, flags);
0425 
0426     n = find_unused_bar_noblock(nfp, tgt, act, tok, offset, size, width);
0427     if (n < 0)
0428         spin_unlock_irqrestore(&nfp->bar_lock, flags);
0429     else
0430         __release(&nfp->bar_lock);
0431 
0432     return n;
0433 }
0434 
0435 static void nfp_bar_get(struct nfp6000_pcie *nfp, struct nfp_bar *bar)
0436 {
0437     atomic_inc(&bar->refcnt);
0438 }
0439 
0440 static void nfp_bar_put(struct nfp6000_pcie *nfp, struct nfp_bar *bar)
0441 {
0442     if (atomic_dec_and_test(&bar->refcnt))
0443         wake_up_interruptible(&nfp->bar_waiters);
0444 }
0445 
0446 static int
0447 nfp_wait_for_bar(struct nfp6000_pcie *nfp, int *barnum,
0448          u32 tgt, u32 act, u32 tok, u64 offset, size_t size, int width)
0449 {
0450     return wait_event_interruptible(nfp->bar_waiters,
0451         (*barnum = find_unused_bar_and_lock(nfp, tgt, act, tok,
0452                             offset, size, width))
0453                     != -EAGAIN);
0454 }
0455 
0456 static int
0457 nfp_alloc_bar(struct nfp6000_pcie *nfp,
0458           u32 tgt, u32 act, u32 tok,
0459           u64 offset, size_t size, int width, int nonblocking)
0460 {
0461     unsigned long irqflags;
0462     int barnum, retval;
0463 
0464     if (size > (1 << 24))
0465         return -EINVAL;
0466 
0467     spin_lock_irqsave(&nfp->bar_lock, irqflags);
0468     barnum = find_matching_bar(nfp, tgt, act, tok, offset, size, width);
0469     if (barnum >= 0) {
0470         /* Found a perfect match. */
0471         nfp_bar_get(nfp, &nfp->bar[barnum]);
0472         spin_unlock_irqrestore(&nfp->bar_lock, irqflags);
0473         return barnum;
0474     }
0475 
0476     barnum = find_unused_bar_noblock(nfp, tgt, act, tok,
0477                      offset, size, width);
0478     if (barnum < 0) {
0479         if (nonblocking)
0480             goto err_nobar;
0481 
0482         /* Wait until a BAR becomes available.  The
0483          * find_unused_bar function will reclaim the bar_lock
0484          * if a free BAR is found.
0485          */
0486         spin_unlock_irqrestore(&nfp->bar_lock, irqflags);
0487         retval = nfp_wait_for_bar(nfp, &barnum, tgt, act, tok,
0488                       offset, size, width);
0489         if (retval)
0490             return retval;
0491         __acquire(&nfp->bar_lock);
0492     }
0493 
0494     nfp_bar_get(nfp, &nfp->bar[barnum]);
0495     retval = reconfigure_bar(nfp, &nfp->bar[barnum],
0496                  tgt, act, tok, offset, size, width);
0497     if (retval < 0) {
0498         nfp_bar_put(nfp, &nfp->bar[barnum]);
0499         barnum = retval;
0500     }
0501 
0502 err_nobar:
0503     spin_unlock_irqrestore(&nfp->bar_lock, irqflags);
0504     return barnum;
0505 }
0506 
0507 static void disable_bars(struct nfp6000_pcie *nfp);
0508 
0509 static int bar_cmp(const void *aptr, const void *bptr)
0510 {
0511     const struct nfp_bar *a = aptr, *b = bptr;
0512 
0513     if (a->bitsize == b->bitsize)
0514         return a->index - b->index;
0515     else
0516         return a->bitsize - b->bitsize;
0517 }
0518 
0519 /* Map all PCI bars and fetch the actual BAR configurations from the
0520  * board.  We assume that the BAR with the PCIe config block is
0521  * already mapped.
0522  *
0523  * BAR0.0: Reserved for General Mapping (for MSI-X access to PCIe SRAM)
0524  * BAR0.1: Reserved for XPB access (for MSI-X access to PCIe PBA)
0525  * BAR0.2: --
0526  * BAR0.3: --
0527  * BAR0.4: Reserved for Explicit 0.0-0.3 access
0528  * BAR0.5: Reserved for Explicit 1.0-1.3 access
0529  * BAR0.6: Reserved for Explicit 2.0-2.3 access
0530  * BAR0.7: Reserved for Explicit 3.0-3.3 access
0531  *
0532  * BAR1.0-BAR1.7: --
0533  * BAR2.0-BAR2.7: --
0534  */
0535 static int enable_bars(struct nfp6000_pcie *nfp, u16 interface)
0536 {
0537     const u32 barcfg_msix_general =
0538         NFP_PCIE_BAR_PCIE2CPP_MapType(
0539             NFP_PCIE_BAR_PCIE2CPP_MapType_GENERAL) |
0540         NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT;
0541     const u32 barcfg_msix_xpb =
0542         NFP_PCIE_BAR_PCIE2CPP_MapType(
0543             NFP_PCIE_BAR_PCIE2CPP_MapType_BULK) |
0544         NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT |
0545         NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(
0546             NFP_CPP_TARGET_ISLAND_XPB);
0547     const u32 barcfg_explicit[4] = {
0548         NFP_PCIE_BAR_PCIE2CPP_MapType(
0549             NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT0),
0550         NFP_PCIE_BAR_PCIE2CPP_MapType(
0551             NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT1),
0552         NFP_PCIE_BAR_PCIE2CPP_MapType(
0553             NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT2),
0554         NFP_PCIE_BAR_PCIE2CPP_MapType(
0555             NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT3),
0556     };
0557     char status_msg[196] = {};
0558     int i, err, bars_free;
0559     struct nfp_bar *bar;
0560     int expl_groups;
0561     char *msg, *end;
0562 
0563     msg = status_msg +
0564         snprintf(status_msg, sizeof(status_msg) - 1, "RESERVED BARs: ");
0565     end = status_msg + sizeof(status_msg) - 1;
0566 
0567     bar = &nfp->bar[0];
0568     for (i = 0; i < ARRAY_SIZE(nfp->bar); i++, bar++) {
0569         struct resource *res;
0570 
0571         res = &nfp->pdev->resource[(i >> 3) * 2];
0572 
0573         /* Skip over BARs that are not IORESOURCE_MEM */
0574         if (!(resource_type(res) & IORESOURCE_MEM)) {
0575             bar--;
0576             continue;
0577         }
0578 
0579         bar->resource = res;
0580         bar->barcfg = 0;
0581 
0582         bar->nfp = nfp;
0583         bar->index = i;
0584         bar->mask = nfp_bar_resource_len(bar) - 1;
0585         bar->bitsize = fls(bar->mask);
0586         bar->base = 0;
0587         bar->iomem = NULL;
0588     }
0589 
0590     nfp->bars = bar - &nfp->bar[0];
0591     if (nfp->bars < 8) {
0592         dev_err(nfp->dev, "No usable BARs found!\n");
0593         return -EINVAL;
0594     }
0595 
0596     bars_free = nfp->bars;
0597 
0598     /* Convert unit ID (0..3) to signal master/data master ID (0x40..0x70)
0599      */
0600     mutex_init(&nfp->expl.mutex);
0601 
0602     nfp->expl.master_id = ((NFP_CPP_INTERFACE_UNIT_of(interface) & 3) + 4)
0603         << 4;
0604     nfp->expl.signal_ref = 0x10;
0605 
0606     /* Configure, and lock, BAR0.0 for General Target use (MSI-X SRAM) */
0607     bar = &nfp->bar[0];
0608     if (nfp_bar_resource_len(bar) >= NFP_PCI_MIN_MAP_SIZE)
0609         bar->iomem = ioremap(nfp_bar_resource_start(bar),
0610                          nfp_bar_resource_len(bar));
0611     if (bar->iomem) {
0612         int pf;
0613 
0614         msg += scnprintf(msg, end - msg, "0.0: General/MSI-X SRAM, ");
0615         atomic_inc(&bar->refcnt);
0616         bars_free--;
0617 
0618         nfp6000_bar_write(nfp, bar, barcfg_msix_general);
0619 
0620         nfp->expl.data = bar->iomem + NFP_PCIE_SRAM +
0621             nfp->dev_info->pcie_expl_offset;
0622 
0623         switch (nfp->pdev->device) {
0624         case PCI_DEVICE_ID_NFP3800:
0625             pf = nfp->pdev->devfn & 7;
0626             nfp->iomem.csr = bar->iomem + NFP_PCIE_BAR(pf);
0627             break;
0628         case PCI_DEVICE_ID_NFP4000:
0629         case PCI_DEVICE_ID_NFP5000:
0630         case PCI_DEVICE_ID_NFP6000:
0631             nfp->iomem.csr = bar->iomem + NFP_PCIE_BAR(0);
0632             break;
0633         default:
0634             dev_err(nfp->dev, "Unsupported device ID: %04hx!\n",
0635                 nfp->pdev->device);
0636             err = -EINVAL;
0637             goto err_unmap_bar0;
0638         }
0639         nfp->iomem.em = bar->iomem + NFP_PCIE_EM;
0640     }
0641 
0642     switch (nfp->pdev->device) {
0643     case PCI_DEVICE_ID_NFP3800:
0644         expl_groups = 1;
0645         break;
0646     case PCI_DEVICE_ID_NFP4000:
0647     case PCI_DEVICE_ID_NFP5000:
0648     case PCI_DEVICE_ID_NFP6000:
0649         expl_groups = 4;
0650         break;
0651     default:
0652         dev_err(nfp->dev, "Unsupported device ID: %04hx!\n",
0653             nfp->pdev->device);
0654         err = -EINVAL;
0655         goto err_unmap_bar0;
0656     }
0657 
0658     /* Configure, and lock, BAR0.1 for PCIe XPB (MSI-X PBA) */
0659     bar = &nfp->bar[1];
0660     msg += scnprintf(msg, end - msg, "0.1: PCIe XPB/MSI-X PBA, ");
0661     atomic_inc(&bar->refcnt);
0662     bars_free--;
0663 
0664     nfp6000_bar_write(nfp, bar, barcfg_msix_xpb);
0665 
0666     /* Use BAR0.4..BAR0.7 for EXPL IO */
0667     for (i = 0; i < 4; i++) {
0668         int j;
0669 
0670         if (i >= NFP_PCIE_EXPLICIT_BARS || i >= expl_groups) {
0671             nfp->expl.group[i].bitsize = 0;
0672             continue;
0673         }
0674 
0675         bar = &nfp->bar[4 + i];
0676         bar->iomem = ioremap(nfp_bar_resource_start(bar),
0677                          nfp_bar_resource_len(bar));
0678         if (bar->iomem) {
0679             msg += scnprintf(msg, end - msg,
0680                      "0.%d: Explicit%d, ", 4 + i, i);
0681             atomic_inc(&bar->refcnt);
0682             bars_free--;
0683 
0684             nfp->expl.group[i].bitsize = bar->bitsize;
0685             nfp->expl.group[i].addr = bar->iomem;
0686             nfp6000_bar_write(nfp, bar, barcfg_explicit[i]);
0687 
0688             for (j = 0; j < 4; j++)
0689                 nfp->expl.group[i].free[j] = true;
0690         }
0691         nfp->iomem.expl[i] = bar->iomem;
0692     }
0693 
0694     /* Sort bars by bit size - use the smallest possible first. */
0695     sort(&nfp->bar[0], nfp->bars, sizeof(nfp->bar[0]),
0696          bar_cmp, NULL);
0697 
0698     dev_info(nfp->dev, "%sfree: %d/%d\n", status_msg, bars_free, nfp->bars);
0699 
0700     return 0;
0701 
0702 err_unmap_bar0:
0703     if (nfp->bar[0].iomem)
0704         iounmap(nfp->bar[0].iomem);
0705     return err;
0706 }
0707 
0708 static void disable_bars(struct nfp6000_pcie *nfp)
0709 {
0710     struct nfp_bar *bar = &nfp->bar[0];
0711     int n;
0712 
0713     for (n = 0; n < nfp->bars; n++, bar++) {
0714         if (bar->iomem) {
0715             iounmap(bar->iomem);
0716             bar->iomem = NULL;
0717         }
0718     }
0719 }
0720 
0721 /*
0722  * Generic CPP bus access interface.
0723  */
0724 
0725 struct nfp6000_area_priv {
0726     atomic_t refcnt;
0727 
0728     struct nfp_bar *bar;
0729     u32 bar_offset;
0730 
0731     u32 target;
0732     u32 action;
0733     u32 token;
0734     u64 offset;
0735     struct {
0736         int read;
0737         int write;
0738         int bar;
0739     } width;
0740     size_t size;
0741 
0742     void __iomem *iomem;
0743     phys_addr_t phys;
0744     struct resource resource;
0745 };
0746 
0747 static int nfp6000_area_init(struct nfp_cpp_area *area, u32 dest,
0748                  unsigned long long address, unsigned long size)
0749 {
0750     struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
0751     u32 target = NFP_CPP_ID_TARGET_of(dest);
0752     u32 action = NFP_CPP_ID_ACTION_of(dest);
0753     u32 token = NFP_CPP_ID_TOKEN_of(dest);
0754     int pp;
0755 
0756     pp = nfp_target_pushpull(NFP_CPP_ID(target, action, token), address);
0757     if (pp < 0)
0758         return pp;
0759 
0760     priv->width.read = PUSH_WIDTH(pp);
0761     priv->width.write = PULL_WIDTH(pp);
0762     if (priv->width.read > 0 &&
0763         priv->width.write > 0 &&
0764         priv->width.read != priv->width.write) {
0765         return -EINVAL;
0766     }
0767 
0768     if (priv->width.read > 0)
0769         priv->width.bar = priv->width.read;
0770     else
0771         priv->width.bar = priv->width.write;
0772 
0773     atomic_set(&priv->refcnt, 0);
0774     priv->bar = NULL;
0775 
0776     priv->target = target;
0777     priv->action = action;
0778     priv->token = token;
0779     priv->offset = address;
0780     priv->size = size;
0781     memset(&priv->resource, 0, sizeof(priv->resource));
0782 
0783     return 0;
0784 }
0785 
0786 static void nfp6000_area_cleanup(struct nfp_cpp_area *area)
0787 {
0788 }
0789 
0790 static void priv_area_get(struct nfp_cpp_area *area)
0791 {
0792     struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
0793 
0794     atomic_inc(&priv->refcnt);
0795 }
0796 
0797 static int priv_area_put(struct nfp_cpp_area *area)
0798 {
0799     struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
0800 
0801     if (WARN_ON(!atomic_read(&priv->refcnt)))
0802         return 0;
0803 
0804     return atomic_dec_and_test(&priv->refcnt);
0805 }
0806 
0807 static int nfp6000_area_acquire(struct nfp_cpp_area *area)
0808 {
0809     struct nfp6000_pcie *nfp = nfp_cpp_priv(nfp_cpp_area_cpp(area));
0810     struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
0811     int barnum, err;
0812 
0813     if (priv->bar) {
0814         /* Already allocated. */
0815         priv_area_get(area);
0816         return 0;
0817     }
0818 
0819     barnum = nfp_alloc_bar(nfp, priv->target, priv->action, priv->token,
0820                    priv->offset, priv->size, priv->width.bar, 1);
0821 
0822     if (barnum < 0) {
0823         err = barnum;
0824         goto err_alloc_bar;
0825     }
0826     priv->bar = &nfp->bar[barnum];
0827 
0828     /* Calculate offset into BAR. */
0829     if (nfp_bar_maptype(priv->bar) ==
0830         NFP_PCIE_BAR_PCIE2CPP_MapType_GENERAL) {
0831         priv->bar_offset = priv->offset &
0832             (NFP_PCIE_P2C_GENERAL_SIZE(priv->bar) - 1);
0833         priv->bar_offset += NFP_PCIE_P2C_GENERAL_TARGET_OFFSET(
0834             priv->bar, priv->target);
0835         priv->bar_offset += NFP_PCIE_P2C_GENERAL_TOKEN_OFFSET(
0836             priv->bar, priv->token);
0837     } else {
0838         priv->bar_offset = priv->offset & priv->bar->mask;
0839     }
0840 
0841     /* We don't actually try to acquire the resource area using
0842      * request_resource.  This would prevent sharing the mapped
0843      * BAR between multiple CPP areas and prevent us from
0844      * effectively utilizing the limited amount of BAR resources.
0845      */
0846     priv->phys = nfp_bar_resource_start(priv->bar) + priv->bar_offset;
0847     priv->resource.name = nfp_cpp_area_name(area);
0848     priv->resource.start = priv->phys;
0849     priv->resource.end = priv->resource.start + priv->size - 1;
0850     priv->resource.flags = IORESOURCE_MEM;
0851 
0852     /* If the bar is already mapped in, use its mapping */
0853     if (priv->bar->iomem)
0854         priv->iomem = priv->bar->iomem + priv->bar_offset;
0855     else
0856         /* Must have been too big. Sub-allocate. */
0857         priv->iomem = ioremap(priv->phys, priv->size);
0858 
0859     if (IS_ERR_OR_NULL(priv->iomem)) {
0860         dev_err(nfp->dev, "Can't ioremap() a %d byte region of BAR %d\n",
0861             (int)priv->size, priv->bar->index);
0862         err = !priv->iomem ? -ENOMEM : PTR_ERR(priv->iomem);
0863         priv->iomem = NULL;
0864         goto err_iomem_remap;
0865     }
0866 
0867     priv_area_get(area);
0868     return 0;
0869 
0870 err_iomem_remap:
0871     nfp_bar_put(nfp, priv->bar);
0872     priv->bar = NULL;
0873 err_alloc_bar:
0874     return err;
0875 }
0876 
0877 static void nfp6000_area_release(struct nfp_cpp_area *area)
0878 {
0879     struct nfp6000_pcie *nfp = nfp_cpp_priv(nfp_cpp_area_cpp(area));
0880     struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
0881 
0882     if (!priv_area_put(area))
0883         return;
0884 
0885     if (!priv->bar->iomem)
0886         iounmap(priv->iomem);
0887 
0888     nfp_bar_put(nfp, priv->bar);
0889 
0890     priv->bar = NULL;
0891     priv->iomem = NULL;
0892 }
0893 
0894 static phys_addr_t nfp6000_area_phys(struct nfp_cpp_area *area)
0895 {
0896     struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
0897 
0898     return priv->phys;
0899 }
0900 
0901 static void __iomem *nfp6000_area_iomem(struct nfp_cpp_area *area)
0902 {
0903     struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
0904 
0905     return priv->iomem;
0906 }
0907 
0908 static struct resource *nfp6000_area_resource(struct nfp_cpp_area *area)
0909 {
0910     /* Use the BAR resource as the resource for the CPP area.
0911      * This enables us to share the BAR among multiple CPP areas
0912      * without resource conflicts.
0913      */
0914     struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
0915 
0916     return priv->bar->resource;
0917 }
0918 
0919 static int nfp6000_area_read(struct nfp_cpp_area *area, void *kernel_vaddr,
0920                  unsigned long offset, unsigned int length)
0921 {
0922     u64 __maybe_unused *wrptr64 = kernel_vaddr;
0923     const u64 __iomem __maybe_unused *rdptr64;
0924     struct nfp6000_area_priv *priv;
0925     u32 *wrptr32 = kernel_vaddr;
0926     const u32 __iomem *rdptr32;
0927     int n, width;
0928 
0929     priv = nfp_cpp_area_priv(area);
0930     rdptr64 = priv->iomem + offset;
0931     rdptr32 = priv->iomem + offset;
0932 
0933     if (offset + length > priv->size)
0934         return -EFAULT;
0935 
0936     width = priv->width.read;
0937     if (width <= 0)
0938         return -EINVAL;
0939 
0940     /* MU reads via a PCIe2CPP BAR support 32bit (and other) lengths */
0941     if (priv->target == (NFP_CPP_TARGET_MU & NFP_CPP_TARGET_ID_MASK) &&
0942         priv->action == NFP_CPP_ACTION_RW &&
0943         (offset % sizeof(u64) == 4 || length % sizeof(u64) == 4))
0944         width = TARGET_WIDTH_32;
0945 
0946     /* Unaligned? Translate to an explicit access */
0947     if ((priv->offset + offset) & (width - 1))
0948         return nfp_cpp_explicit_read(nfp_cpp_area_cpp(area),
0949                          NFP_CPP_ID(priv->target,
0950                             priv->action,
0951                             priv->token),
0952                          priv->offset + offset,
0953                          kernel_vaddr, length, width);
0954 
0955     if (WARN_ON(!priv->bar))
0956         return -EFAULT;
0957 
0958     switch (width) {
0959     case TARGET_WIDTH_32:
0960         if (offset % sizeof(u32) != 0 || length % sizeof(u32) != 0)
0961             return -EINVAL;
0962 
0963         for (n = 0; n < length; n += sizeof(u32))
0964             *wrptr32++ = __raw_readl(rdptr32++);
0965         return n;
0966 #ifdef __raw_readq
0967     case TARGET_WIDTH_64:
0968         if (offset % sizeof(u64) != 0 || length % sizeof(u64) != 0)
0969             return -EINVAL;
0970 
0971         for (n = 0; n < length; n += sizeof(u64))
0972             *wrptr64++ = __raw_readq(rdptr64++);
0973         return n;
0974 #endif
0975     default:
0976         return -EINVAL;
0977     }
0978 }
0979 
0980 static int
0981 nfp6000_area_write(struct nfp_cpp_area *area,
0982            const void *kernel_vaddr,
0983            unsigned long offset, unsigned int length)
0984 {
0985     const u64 __maybe_unused *rdptr64 = kernel_vaddr;
0986     u64 __iomem __maybe_unused *wrptr64;
0987     const u32 *rdptr32 = kernel_vaddr;
0988     struct nfp6000_area_priv *priv;
0989     u32 __iomem *wrptr32;
0990     int n, width;
0991 
0992     priv = nfp_cpp_area_priv(area);
0993     wrptr64 = priv->iomem + offset;
0994     wrptr32 = priv->iomem + offset;
0995 
0996     if (offset + length > priv->size)
0997         return -EFAULT;
0998 
0999     width = priv->width.write;
1000     if (width <= 0)
1001         return -EINVAL;
1002 
1003     /* MU writes via a PCIe2CPP BAR support 32bit (and other) lengths */
1004     if (priv->target == (NFP_CPP_TARGET_ID_MASK & NFP_CPP_TARGET_MU) &&
1005         priv->action == NFP_CPP_ACTION_RW &&
1006         (offset % sizeof(u64) == 4 || length % sizeof(u64) == 4))
1007         width = TARGET_WIDTH_32;
1008 
1009     /* Unaligned? Translate to an explicit access */
1010     if ((priv->offset + offset) & (width - 1))
1011         return nfp_cpp_explicit_write(nfp_cpp_area_cpp(area),
1012                           NFP_CPP_ID(priv->target,
1013                              priv->action,
1014                              priv->token),
1015                           priv->offset + offset,
1016                           kernel_vaddr, length, width);
1017 
1018     if (WARN_ON(!priv->bar))
1019         return -EFAULT;
1020 
1021     switch (width) {
1022     case TARGET_WIDTH_32:
1023         if (offset % sizeof(u32) != 0 || length % sizeof(u32) != 0)
1024             return -EINVAL;
1025 
1026         for (n = 0; n < length; n += sizeof(u32)) {
1027             __raw_writel(*rdptr32++, wrptr32++);
1028             wmb();
1029         }
1030         return n;
1031 #ifdef __raw_writeq
1032     case TARGET_WIDTH_64:
1033         if (offset % sizeof(u64) != 0 || length % sizeof(u64) != 0)
1034             return -EINVAL;
1035 
1036         for (n = 0; n < length; n += sizeof(u64)) {
1037             __raw_writeq(*rdptr64++, wrptr64++);
1038             wmb();
1039         }
1040         return n;
1041 #endif
1042     default:
1043         return -EINVAL;
1044     }
1045 }
1046 
1047 struct nfp6000_explicit_priv {
1048     struct nfp6000_pcie *nfp;
1049     struct {
1050         int group;
1051         int area;
1052     } bar;
1053     int bitsize;
1054     void __iomem *data;
1055     void __iomem *addr;
1056 };
1057 
1058 static int nfp6000_explicit_acquire(struct nfp_cpp_explicit *expl)
1059 {
1060     struct nfp6000_pcie *nfp = nfp_cpp_priv(nfp_cpp_explicit_cpp(expl));
1061     struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl);
1062     int i, j;
1063 
1064     mutex_lock(&nfp->expl.mutex);
1065     for (i = 0; i < ARRAY_SIZE(nfp->expl.group); i++) {
1066         if (!nfp->expl.group[i].bitsize)
1067             continue;
1068 
1069         for (j = 0; j < ARRAY_SIZE(nfp->expl.group[i].free); j++) {
1070             u16 data_offset;
1071 
1072             if (!nfp->expl.group[i].free[j])
1073                 continue;
1074 
1075             priv->nfp = nfp;
1076             priv->bar.group = i;
1077             priv->bar.area = j;
1078             priv->bitsize = nfp->expl.group[i].bitsize - 2;
1079 
1080             data_offset = (priv->bar.group << 9) +
1081                 (priv->bar.area << 7);
1082             priv->data = nfp->expl.data + data_offset;
1083             priv->addr = nfp->expl.group[i].addr +
1084                 (priv->bar.area << priv->bitsize);
1085             nfp->expl.group[i].free[j] = false;
1086 
1087             mutex_unlock(&nfp->expl.mutex);
1088             return 0;
1089         }
1090     }
1091     mutex_unlock(&nfp->expl.mutex);
1092 
1093     return -EAGAIN;
1094 }
1095 
1096 static void nfp6000_explicit_release(struct nfp_cpp_explicit *expl)
1097 {
1098     struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl);
1099     struct nfp6000_pcie *nfp = priv->nfp;
1100 
1101     mutex_lock(&nfp->expl.mutex);
1102     nfp->expl.group[priv->bar.group].free[priv->bar.area] = true;
1103     mutex_unlock(&nfp->expl.mutex);
1104 }
1105 
1106 static int nfp6000_explicit_put(struct nfp_cpp_explicit *expl,
1107                 const void *buff, size_t len)
1108 {
1109     struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl);
1110     const u32 *src = buff;
1111     size_t i;
1112 
1113     for (i = 0; i < len; i += sizeof(u32))
1114         writel(*(src++), priv->data + i);
1115 
1116     return i;
1117 }
1118 
1119 static int
1120 nfp6000_explicit_do(struct nfp_cpp_explicit *expl,
1121             const struct nfp_cpp_explicit_command *cmd, u64 address)
1122 {
1123     struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl);
1124     u8 signal_master, signal_ref, data_master;
1125     struct nfp6000_pcie *nfp = priv->nfp;
1126     int sigmask = 0;
1127     u16 data_ref;
1128     u32 csr[3];
1129 
1130     if (cmd->siga_mode)
1131         sigmask |= 1 << cmd->siga;
1132     if (cmd->sigb_mode)
1133         sigmask |= 1 << cmd->sigb;
1134 
1135     signal_master = cmd->signal_master;
1136     if (!signal_master)
1137         signal_master = nfp->expl.master_id;
1138 
1139     signal_ref = cmd->signal_ref;
1140     if (signal_master == nfp->expl.master_id)
1141         signal_ref = nfp->expl.signal_ref +
1142             ((priv->bar.group * 4 + priv->bar.area) << 1);
1143 
1144     data_master = cmd->data_master;
1145     if (!data_master)
1146         data_master = nfp->expl.master_id;
1147 
1148     data_ref = cmd->data_ref;
1149     if (data_master == nfp->expl.master_id)
1150         data_ref = 0x1000 +
1151             (priv->bar.group << 9) + (priv->bar.area << 7);
1152 
1153     csr[0] = NFP_PCIE_BAR_EXPLICIT_BAR0_SignalType(sigmask) |
1154         NFP_PCIE_BAR_EXPLICIT_BAR0_Token(
1155             NFP_CPP_ID_TOKEN_of(cmd->cpp_id)) |
1156         NFP_PCIE_BAR_EXPLICIT_BAR0_Address(address >> 16);
1157 
1158     csr[1] = NFP_PCIE_BAR_EXPLICIT_BAR1_SignalRef(signal_ref) |
1159         NFP_PCIE_BAR_EXPLICIT_BAR1_DataMaster(data_master) |
1160         NFP_PCIE_BAR_EXPLICIT_BAR1_DataRef(data_ref);
1161 
1162     csr[2] = NFP_PCIE_BAR_EXPLICIT_BAR2_Target(
1163             NFP_CPP_ID_TARGET_of(cmd->cpp_id)) |
1164         NFP_PCIE_BAR_EXPLICIT_BAR2_Action(
1165             NFP_CPP_ID_ACTION_of(cmd->cpp_id)) |
1166         NFP_PCIE_BAR_EXPLICIT_BAR2_Length(cmd->len) |
1167         NFP_PCIE_BAR_EXPLICIT_BAR2_ByteMask(cmd->byte_mask) |
1168         NFP_PCIE_BAR_EXPLICIT_BAR2_SignalMaster(signal_master);
1169 
1170     if (nfp->iomem.csr) {
1171         writel(csr[0], nfp->iomem.csr +
1172                NFP_PCIE_BAR_EXPLICIT_BAR0(priv->bar.group,
1173                           priv->bar.area));
1174         writel(csr[1], nfp->iomem.csr +
1175                NFP_PCIE_BAR_EXPLICIT_BAR1(priv->bar.group,
1176                           priv->bar.area));
1177         writel(csr[2], nfp->iomem.csr +
1178                NFP_PCIE_BAR_EXPLICIT_BAR2(priv->bar.group,
1179                           priv->bar.area));
1180         /* Readback to ensure BAR is flushed */
1181         readl(nfp->iomem.csr +
1182               NFP_PCIE_BAR_EXPLICIT_BAR0(priv->bar.group,
1183                          priv->bar.area));
1184         readl(nfp->iomem.csr +
1185               NFP_PCIE_BAR_EXPLICIT_BAR1(priv->bar.group,
1186                          priv->bar.area));
1187         readl(nfp->iomem.csr +
1188               NFP_PCIE_BAR_EXPLICIT_BAR2(priv->bar.group,
1189                          priv->bar.area));
1190     } else {
1191         pci_write_config_dword(nfp->pdev, 0x400 +
1192                        NFP_PCIE_BAR_EXPLICIT_BAR0(
1193                            priv->bar.group, priv->bar.area),
1194                        csr[0]);
1195 
1196         pci_write_config_dword(nfp->pdev, 0x400 +
1197                        NFP_PCIE_BAR_EXPLICIT_BAR1(
1198                            priv->bar.group, priv->bar.area),
1199                        csr[1]);
1200 
1201         pci_write_config_dword(nfp->pdev, 0x400 +
1202                        NFP_PCIE_BAR_EXPLICIT_BAR2(
1203                            priv->bar.group, priv->bar.area),
1204                        csr[2]);
1205     }
1206 
1207     /* Issue the 'kickoff' transaction */
1208     readb(priv->addr + (address & ((1 << priv->bitsize) - 1)));
1209 
1210     return sigmask;
1211 }
1212 
1213 static int nfp6000_explicit_get(struct nfp_cpp_explicit *expl,
1214                 void *buff, size_t len)
1215 {
1216     struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl);
1217     u32 *dst = buff;
1218     size_t i;
1219 
1220     for (i = 0; i < len; i += sizeof(u32))
1221         *(dst++) = readl(priv->data + i);
1222 
1223     return i;
1224 }
1225 
1226 static int nfp6000_init(struct nfp_cpp *cpp)
1227 {
1228     nfp_cpp_area_cache_add(cpp, SZ_64K);
1229     nfp_cpp_area_cache_add(cpp, SZ_64K);
1230     nfp_cpp_area_cache_add(cpp, SZ_256K);
1231 
1232     return 0;
1233 }
1234 
1235 static void nfp6000_free(struct nfp_cpp *cpp)
1236 {
1237     struct nfp6000_pcie *nfp = nfp_cpp_priv(cpp);
1238 
1239     disable_bars(nfp);
1240     kfree(nfp);
1241 }
1242 
1243 static int nfp6000_read_serial(struct device *dev, u8 *serial)
1244 {
1245     struct pci_dev *pdev = to_pci_dev(dev);
1246     u64 dsn;
1247 
1248     dsn = pci_get_dsn(pdev);
1249     if (!dsn) {
1250         dev_err(dev, "can't find PCIe Serial Number Capability\n");
1251         return -EINVAL;
1252     }
1253 
1254     put_unaligned_be32((u32)(dsn >> 32), serial);
1255     put_unaligned_be16((u16)(dsn >> 16), serial + 4);
1256 
1257     return 0;
1258 }
1259 
1260 static int nfp6000_get_interface(struct device *dev)
1261 {
1262     struct pci_dev *pdev = to_pci_dev(dev);
1263     u64 dsn;
1264 
1265     dsn = pci_get_dsn(pdev);
1266     if (!dsn) {
1267         dev_err(dev, "can't find PCIe Serial Number Capability\n");
1268         return -EINVAL;
1269     }
1270 
1271     return dsn & 0xffff;
1272 }
1273 
1274 static const struct nfp_cpp_operations nfp6000_pcie_ops = {
1275     .owner          = THIS_MODULE,
1276 
1277     .init           = nfp6000_init,
1278     .free           = nfp6000_free,
1279 
1280     .read_serial        = nfp6000_read_serial,
1281     .get_interface      = nfp6000_get_interface,
1282 
1283     .area_priv_size     = sizeof(struct nfp6000_area_priv),
1284     .area_init      = nfp6000_area_init,
1285     .area_cleanup       = nfp6000_area_cleanup,
1286     .area_acquire       = nfp6000_area_acquire,
1287     .area_release       = nfp6000_area_release,
1288     .area_phys      = nfp6000_area_phys,
1289     .area_iomem     = nfp6000_area_iomem,
1290     .area_resource      = nfp6000_area_resource,
1291     .area_read      = nfp6000_area_read,
1292     .area_write     = nfp6000_area_write,
1293 
1294     .explicit_priv_size = sizeof(struct nfp6000_explicit_priv),
1295     .explicit_acquire   = nfp6000_explicit_acquire,
1296     .explicit_release   = nfp6000_explicit_release,
1297     .explicit_put       = nfp6000_explicit_put,
1298     .explicit_do        = nfp6000_explicit_do,
1299     .explicit_get       = nfp6000_explicit_get,
1300 };
1301 
1302 /**
1303  * nfp_cpp_from_nfp6000_pcie() - Build a NFP CPP bus from a NFP6000 PCI device
1304  * @pdev:   NFP6000 PCI device
1305  * @dev_info:   NFP ASIC params
1306  *
1307  * Return: NFP CPP handle
1308  */
1309 struct nfp_cpp *
1310 nfp_cpp_from_nfp6000_pcie(struct pci_dev *pdev, const struct nfp_dev_info *dev_info)
1311 {
1312     struct nfp6000_pcie *nfp;
1313     u16 interface;
1314     int err;
1315 
1316     /*  Finished with card initialization. */
1317     dev_info(&pdev->dev, "Network Flow Processor %s PCIe Card Probe\n",
1318          dev_info->chip_names);
1319     pcie_print_link_status(pdev);
1320 
1321     nfp = kzalloc(sizeof(*nfp), GFP_KERNEL);
1322     if (!nfp) {
1323         err = -ENOMEM;
1324         goto err_ret;
1325     }
1326 
1327     nfp->dev = &pdev->dev;
1328     nfp->pdev = pdev;
1329     nfp->dev_info = dev_info;
1330     init_waitqueue_head(&nfp->bar_waiters);
1331     spin_lock_init(&nfp->bar_lock);
1332 
1333     interface = nfp6000_get_interface(&pdev->dev);
1334 
1335     if (NFP_CPP_INTERFACE_TYPE_of(interface) !=
1336         NFP_CPP_INTERFACE_TYPE_PCI) {
1337         dev_err(&pdev->dev,
1338             "Interface type %d is not the expected %d\n",
1339             NFP_CPP_INTERFACE_TYPE_of(interface),
1340             NFP_CPP_INTERFACE_TYPE_PCI);
1341         err = -ENODEV;
1342         goto err_free_nfp;
1343     }
1344 
1345     if (NFP_CPP_INTERFACE_CHANNEL_of(interface) !=
1346         NFP_CPP_INTERFACE_CHANNEL_PEROPENER) {
1347         dev_err(&pdev->dev, "Interface channel %d is not the expected %d\n",
1348             NFP_CPP_INTERFACE_CHANNEL_of(interface),
1349             NFP_CPP_INTERFACE_CHANNEL_PEROPENER);
1350         err = -ENODEV;
1351         goto err_free_nfp;
1352     }
1353 
1354     err = enable_bars(nfp, interface);
1355     if (err)
1356         goto err_free_nfp;
1357 
1358     /* Probe for all the common NFP devices */
1359     return nfp_cpp_from_operations(&nfp6000_pcie_ops, &pdev->dev, nfp);
1360 
1361 err_free_nfp:
1362     kfree(nfp);
1363 err_ret:
1364     dev_err(&pdev->dev, "NFP6000 PCI setup failed\n");
1365     return ERR_PTR(err);
1366 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!