endpoint/functions/pci-epf-ntb.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Endpoint Function Driver to implement Non-Transparent Bridge functionality
0004  *
0005  * Copyright (C) 2020 Texas Instruments
0006  * Author: Kishon Vijay Abraham I <kishon@ti.com>
0007  */
0008
0009 /*
0010  * The PCI NTB function driver configures the SoC with multiple PCIe Endpoint
0011  * (EP) controller instances (see diagram below) in such a way that
0012  * transactions from one EP controller are routed to the other EP controller.
0013  * Once PCI NTB function driver configures the SoC with multiple EP instances,
0014  * HOST1 and HOST2 can communicate with each other using SoC as a bridge.
0015  *
0016  *    +-------------+                                   +-------------+
0017  *    |             |                                   |             |
0018  *    |    HOST1    |                                   |    HOST2    |
0019  *    |             |                                   |             |
0020  *    +------^------+                                   +------^------+
0021  *           |                                                 |
0022  *           |                                                 |
0023  * +---------|-------------------------------------------------|---------+
0024  * |  +------v------+                                   +------v------+  |
0025  * |  |             |                                   |             |  |
0026  * |  |     EP      |                                   |     EP      |  |
0027  * |  | CONTROLLER1 |                                   | CONTROLLER2 |  |
0028  * |  |             <----------------------------------->             |  |
0029  * |  |             |                                   |             |  |
0030  * |  |             |                                   |             |  |
0031  * |  |             |  SoC With Multiple EP Instances   |             |  |
0032  * |  |             |  (Configured using NTB Function)  |             |  |
0033  * |  +-------------+                                   +-------------+  |
0034  * +---------------------------------------------------------------------+
0035  */
0036
0037 #include <linux/delay.h>
0038 #include <linux/io.h>
0039 #include <linux/module.h>
0040 #include <linux/slab.h>
0041
0042 #include <linux/pci-epc.h>
0043 #include <linux/pci-epf.h>
0044
0045 static struct workqueue_struct *kpcintb_workqueue;
0046
0047 #define COMMAND_CONFIGURE_DOORBELL  1
0048 #define COMMAND_TEARDOWN_DOORBELL   2
0049 #define COMMAND_CONFIGURE_MW        3
0050 #define COMMAND_TEARDOWN_MW     4
0051 #define COMMAND_LINK_UP         5
0052 #define COMMAND_LINK_DOWN       6
0053
0054 #define COMMAND_STATUS_OK       1
0055 #define COMMAND_STATUS_ERROR        2
0056
0057 #define LINK_STATUS_UP          BIT(0)
0058
0059 #define SPAD_COUNT          64
0060 #define DB_COUNT            4
0061 #define NTB_MW_OFFSET           2
0062 #define DB_COUNT_MASK           GENMASK(15, 0)
0063 #define MSIX_ENABLE         BIT(16)
0064 #define MAX_DB_COUNT            32
0065 #define MAX_MW              4
0066
0067 enum epf_ntb_bar {
0068     BAR_CONFIG,
0069     BAR_PEER_SPAD,
0070     BAR_DB_MW1,
0071     BAR_MW2,
0072     BAR_MW3,
0073     BAR_MW4,
0074 };
0075
0076 struct epf_ntb {
0077     u32 num_mws;
0078     u32 db_count;
0079     u32 spad_count;
0080     struct pci_epf *epf;
0081     u64 mws_size[MAX_MW];
0082     struct config_group group;
0083     struct epf_ntb_epc *epc[2];
0084 };
0085
0086 #define to_epf_ntb(epf_group) container_of((epf_group), struct epf_ntb, group)
0087
0088 struct epf_ntb_epc {
0089     u8 func_no;
0090     u8 vfunc_no;
0091     bool linkup;
0092     bool is_msix;
0093     int msix_bar;
0094     u32 spad_size;
0095     struct pci_epc *epc;
0096     struct epf_ntb *epf_ntb;
0097     void __iomem *mw_addr[6];
0098     size_t msix_table_offset;
0099     struct epf_ntb_ctrl *reg;
0100     struct pci_epf_bar *epf_bar;
0101     enum pci_barno epf_ntb_bar[6];
0102     struct delayed_work cmd_handler;
0103     enum pci_epc_interface_type type;
0104     const struct pci_epc_features *epc_features;
0105 };
0106
0107 struct epf_ntb_ctrl {
0108     u32 command;
0109     u32 argument;
0110     u16 command_status;
0111     u16 link_status;
0112     u32 topology;
0113     u64 addr;
0114     u64 size;
0115     u32 num_mws;
0116     u32 mw1_offset;
0117     u32 spad_offset;
0118     u32 spad_count;
0119     u32 db_entry_size;
0120     u32 db_data[MAX_DB_COUNT];
0121     u32 db_offset[MAX_DB_COUNT];
0122 } __packed;
0123
0124 static struct pci_epf_header epf_ntb_header = {
0125     .vendorid   = PCI_ANY_ID,
0126     .deviceid   = PCI_ANY_ID,
0127     .baseclass_code = PCI_BASE_CLASS_MEMORY,
0128     .interrupt_pin  = PCI_INTERRUPT_INTA,
0129 };
0130
0131 /**
0132  * epf_ntb_link_up() - Raise link_up interrupt to both the hosts
0133  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
0134  * @link_up: true or false indicating Link is UP or Down
0135  *
0136  * Once NTB function in HOST1 and the NTB function in HOST2 invoke
0137  * ntb_link_enable(), this NTB function driver will trigger a link event to
0138  * the NTB client in both the hosts.
0139  */
0140 static int epf_ntb_link_up(struct epf_ntb *ntb, bool link_up)
0141 {
0142     enum pci_epc_interface_type type;
0143     enum pci_epc_irq_type irq_type;
0144     struct epf_ntb_epc *ntb_epc;
0145     struct epf_ntb_ctrl *ctrl;
0146     struct pci_epc *epc;
0147     u8 func_no, vfunc_no;
0148     bool is_msix;
0149     int ret;
0150
0151     for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {
0152         ntb_epc = ntb->epc[type];
0153         epc = ntb_epc->epc;
0154         func_no = ntb_epc->func_no;
0155         vfunc_no = ntb_epc->vfunc_no;
0156         is_msix = ntb_epc->is_msix;
0157         ctrl = ntb_epc->reg;
0158         if (link_up)
0159             ctrl->link_status |= LINK_STATUS_UP;
0160         else
0161             ctrl->link_status &= ~LINK_STATUS_UP;
0162         irq_type = is_msix ? PCI_EPC_IRQ_MSIX : PCI_EPC_IRQ_MSI;
0163         ret = pci_epc_raise_irq(epc, func_no, vfunc_no, irq_type, 1);
0164         if (ret) {
0165             dev_err(&epc->dev,
0166                 "%s intf: Failed to raise Link Up IRQ\n",
0167                 pci_epc_interface_string(type));
0168             return ret;
0169         }
0170     }
0171
0172     return 0;
0173 }
0174
0175 /**
0176  * epf_ntb_configure_mw() - Configure the Outbound Address Space for one host
0177  *   to access the memory window of other host
0178  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
0179  * @type: PRIMARY interface or SECONDARY interface
0180  * @mw: Index of the memory window (either 0, 1, 2 or 3)
0181  *
0182  * +-----------------+    +---->+----------------+-----------+-----------------+
0183  * |       BAR0      |    |     |   Doorbell 1   +-----------> MSI|X ADDRESS 1 |
0184  * +-----------------+    |     +----------------+           +-----------------+
0185  * |       BAR1      |    |     |   Doorbell 2   +---------+ |                 |
0186  * +-----------------+----+     +----------------+         | |                 |
0187  * |       BAR2      |          |   Doorbell 3   +-------+ | +-----------------+
0188  * +-----------------+----+     +----------------+       | +-> MSI|X ADDRESS 2 |
0189  * |       BAR3      |    |     |   Doorbell 4   +-----+ |   +-----------------+
0190  * +-----------------+    |     |----------------+     | |   |                 |
0191  * |       BAR4      |    |     |                |     | |   +-----------------+
0192  * +-----------------+    |     |      MW1       +---+ | +-->+ MSI|X ADDRESS 3||
0193  * |       BAR5      |    |     |                |   | |     +-----------------+
0194  * +-----------------+    +---->-----------------+   | |     |                 |
0195  *   EP CONTROLLER 1            |                |   | |     +-----------------+
0196  *                              |                |   | +---->+ MSI|X ADDRESS 4 |
0197  *                              +----------------+   |       +-----------------+
0198  *                      (A)      EP CONTROLLER 2     |       |                 |
0199  *                                 (OB SPACE)        |       |                 |
0200  *                                                   +------->      MW1        |
0201  *                                                           |                 |
0202  *                                                           |                 |
0203  *                                                   (B)     +-----------------+
0204  *                                                           |                 |
0205  *                                                           |                 |
0206  *                                                           |                 |
0207  *                                                           |                 |
0208  *                                                           |                 |
0209  *                                                           +-----------------+
0210  *                                                           PCI Address Space
0211  *                                                           (Managed by HOST2)
0212  *
0213  * This function performs stage (B) in the above diagram (see MW1) i.e., map OB
0214  * address space of memory window to PCI address space.
0215  *
0216  * This operation requires 3 parameters
0217  *  1) Address in the outbound address space
0218  *  2) Address in the PCI Address space
0219  *  3) Size of the address region to be mapped
0220  *
0221  * The address in the outbound address space (for MW1, MW2, MW3 and MW4) is
0222  * stored in epf_bar corresponding to BAR_DB_MW1 for MW1 and BAR_MW2, BAR_MW3
0223  * BAR_MW4 for rest of the BARs of epf_ntb_epc that is connected to HOST1. This
0224  * is populated in epf_ntb_alloc_peer_mem() in this driver.
0225  *
0226  * The address and size of the PCI address region that has to be mapped would
0227  * be provided by HOST2 in ctrl->addr and ctrl->size of epf_ntb_epc that is
0228  * connected to HOST2.
0229  *
0230  * Please note Memory window1 (MW1) and Doorbell registers together will be
0231  * mapped to a single BAR (BAR2) above for 32-bit BARs. The exact BAR that's
0232  * used for Memory window (MW) can be obtained from epf_ntb_bar[BAR_DB_MW1],
0233  * epf_ntb_bar[BAR_MW2], epf_ntb_bar[BAR_MW2], epf_ntb_bar[BAR_MW2].
0234  */
0235 static int epf_ntb_configure_mw(struct epf_ntb *ntb,
0236                 enum pci_epc_interface_type type, u32 mw)
0237 {
0238     struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
0239     struct pci_epf_bar *peer_epf_bar;
0240     enum pci_barno peer_barno;
0241     struct epf_ntb_ctrl *ctrl;
0242     phys_addr_t phys_addr;
0243     u8 func_no, vfunc_no;
0244     struct pci_epc *epc;
0245     u64 addr, size;
0246     int ret = 0;
0247
0248     ntb_epc = ntb->epc[type];
0249     epc = ntb_epc->epc;
0250
0251     peer_ntb_epc = ntb->epc[!type];
0252     peer_barno = peer_ntb_epc->epf_ntb_bar[mw + NTB_MW_OFFSET];
0253     peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];
0254
0255     phys_addr = peer_epf_bar->phys_addr;
0256     ctrl = ntb_epc->reg;
0257     addr = ctrl->addr;
0258     size = ctrl->size;
0259     if (mw + NTB_MW_OFFSET == BAR_DB_MW1)
0260         phys_addr += ctrl->mw1_offset;
0261
0262     if (size > ntb->mws_size[mw]) {
0263         dev_err(&epc->dev,
0264             "%s intf: MW: %d Req Sz:%llxx > Supported Sz:%llx\n",
0265             pci_epc_interface_string(type), mw, size,
0266             ntb->mws_size[mw]);
0267         ret = -EINVAL;
0268         goto err_invalid_size;
0269     }
0270
0271     func_no = ntb_epc->func_no;
0272     vfunc_no = ntb_epc->vfunc_no;
0273
0274     ret = pci_epc_map_addr(epc, func_no, vfunc_no, phys_addr, addr, size);
0275     if (ret)
0276         dev_err(&epc->dev,
0277             "%s intf: Failed to map memory window %d address\n",
0278             pci_epc_interface_string(type), mw);
0279
0280 err_invalid_size:
0281
0282     return ret;
0283 }
0284
0285 /**
0286  * epf_ntb_teardown_mw() - Teardown the configured OB ATU
0287  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
0288  * @type: PRIMARY interface or SECONDARY interface
0289  * @mw: Index of the memory window (either 0, 1, 2 or 3)
0290  *
0291  * Teardown the configured OB ATU configured in epf_ntb_configure_mw() using
0292  * pci_epc_unmap_addr()
0293  */
0294 static void epf_ntb_teardown_mw(struct epf_ntb *ntb,
0295                 enum pci_epc_interface_type type, u32 mw)
0296 {
0297     struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
0298     struct pci_epf_bar *peer_epf_bar;
0299     enum pci_barno peer_barno;
0300     struct epf_ntb_ctrl *ctrl;
0301     phys_addr_t phys_addr;
0302     u8 func_no, vfunc_no;
0303     struct pci_epc *epc;
0304
0305     ntb_epc = ntb->epc[type];
0306     epc = ntb_epc->epc;
0307
0308     peer_ntb_epc = ntb->epc[!type];
0309     peer_barno = peer_ntb_epc->epf_ntb_bar[mw + NTB_MW_OFFSET];
0310     peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];
0311
0312     phys_addr = peer_epf_bar->phys_addr;
0313     ctrl = ntb_epc->reg;
0314     if (mw + NTB_MW_OFFSET == BAR_DB_MW1)
0315         phys_addr += ctrl->mw1_offset;
0316     func_no = ntb_epc->func_no;
0317     vfunc_no = ntb_epc->vfunc_no;
0318
0319     pci_epc_unmap_addr(epc, func_no, vfunc_no, phys_addr);
0320 }
0321
0322 /**
0323  * epf_ntb_configure_msi() - Map OB address space to MSI address
0324  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
0325  * @type: PRIMARY interface or SECONDARY interface
0326  * @db_count: Number of doorbell interrupts to map
0327  *
0328  *+-----------------+    +----->+----------------+-----------+-----------------+
0329  *|       BAR0      |    |      |   Doorbell 1   +---+------->   MSI ADDRESS   |
0330  *+-----------------+    |      +----------------+   |       +-----------------+
0331  *|       BAR1      |    |      |   Doorbell 2   +---+       |                 |
0332  *+-----------------+----+      +----------------+   |       |                 |
0333  *|       BAR2      |           |   Doorbell 3   +---+       |                 |
0334  *+-----------------+----+      +----------------+   |       |                 |
0335  *|       BAR3      |    |      |   Doorbell 4   +---+       |                 |
0336  *+-----------------+    |      |----------------+           |                 |
0337  *|       BAR4      |    |      |                |           |                 |
0338  *+-----------------+    |      |      MW1       |           |                 |
0339  *|       BAR5      |    |      |                |           |                 |
0340  *+-----------------+    +----->-----------------+           |                 |
0341  *  EP CONTROLLER 1             |                |           |                 |
0342  *                              |                |           |                 |
0343  *                              +----------------+           +-----------------+
0344  *                     (A)       EP CONTROLLER 2             |                 |
0345  *                                 (OB SPACE)                |                 |
0346  *                                                           |      MW1        |
0347  *                                                           |                 |
0348  *                                                           |                 |
0349  *                                                   (B)     +-----------------+
0350  *                                                           |                 |
0351  *                                                           |                 |
0352  *                                                           |                 |
0353  *                                                           |                 |
0354  *                                                           |                 |
0355  *                                                           +-----------------+
0356  *                                                           PCI Address Space
0357  *                                                           (Managed by HOST2)
0358  *
0359  *
0360  * This function performs stage (B) in the above diagram (see Doorbell 1,
0361  * Doorbell 2, Doorbell 3, Doorbell 4) i.e map OB address space corresponding to
0362  * doorbell to MSI address in PCI address space.
0363  *
0364  * This operation requires 3 parameters
0365  *  1) Address reserved for doorbell in the outbound address space
0366  *  2) MSI-X address in the PCIe Address space
0367  *  3) Number of MSI-X interrupts that has to be configured
0368  *
0369  * The address in the outbound address space (for the Doorbell) is stored in
0370  * epf_bar corresponding to BAR_DB_MW1 of epf_ntb_epc that is connected to
0371  * HOST1. This is populated in epf_ntb_alloc_peer_mem() in this driver along
0372  * with address for MW1.
0373  *
0374  * pci_epc_map_msi_irq() takes the MSI address from MSI capability register
0375  * and maps the OB address (obtained in epf_ntb_alloc_peer_mem()) to the MSI
0376  * address.
0377  *
0378  * epf_ntb_configure_msi() also stores the MSI data to raise each interrupt
0379  * in db_data of the peer's control region. This helps the peer to raise
0380  * doorbell of the other host by writing db_data to the BAR corresponding to
0381  * BAR_DB_MW1.
0382  */
0383 static int epf_ntb_configure_msi(struct epf_ntb *ntb,
0384                  enum pci_epc_interface_type type, u16 db_count)
0385 {
0386     struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
0387     u32 db_entry_size, db_data, db_offset;
0388     struct pci_epf_bar *peer_epf_bar;
0389     struct epf_ntb_ctrl *peer_ctrl;
0390     enum pci_barno peer_barno;
0391     phys_addr_t phys_addr;
0392     u8 func_no, vfunc_no;
0393     struct pci_epc *epc;
0394     int ret, i;
0395
0396     ntb_epc = ntb->epc[type];
0397     epc = ntb_epc->epc;
0398
0399     peer_ntb_epc = ntb->epc[!type];
0400     peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1];
0401     peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];
0402     peer_ctrl = peer_ntb_epc->reg;
0403     db_entry_size = peer_ctrl->db_entry_size;
0404
0405     phys_addr = peer_epf_bar->phys_addr;
0406     func_no = ntb_epc->func_no;
0407     vfunc_no = ntb_epc->vfunc_no;
0408
0409     ret = pci_epc_map_msi_irq(epc, func_no, vfunc_no, phys_addr, db_count,
0410                   db_entry_size, &db_data, &db_offset);
0411     if (ret) {
0412         dev_err(&epc->dev, "%s intf: Failed to map MSI IRQ\n",
0413             pci_epc_interface_string(type));
0414         return ret;
0415     }
0416
0417     for (i = 0; i < db_count; i++) {
0418         peer_ctrl->db_data[i] = db_data | i;
0419         peer_ctrl->db_offset[i] = db_offset;
0420     }
0421
0422     return 0;
0423 }
0424
0425 /**
0426  * epf_ntb_configure_msix() - Map OB address space to MSI-X address
0427  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
0428  * @type: PRIMARY interface or SECONDARY interface
0429  * @db_count: Number of doorbell interrupts to map
0430  *
0431  *+-----------------+    +----->+----------------+-----------+-----------------+
0432  *|       BAR0      |    |      |   Doorbell 1   +-----------> MSI-X ADDRESS 1 |
0433  *+-----------------+    |      +----------------+           +-----------------+
0434  *|       BAR1      |    |      |   Doorbell 2   +---------+ |                 |
0435  *+-----------------+----+      +----------------+         | |                 |
0436  *|       BAR2      |           |   Doorbell 3   +-------+ | +-----------------+
0437  *+-----------------+----+      +----------------+       | +-> MSI-X ADDRESS 2 |
0438  *|       BAR3      |    |      |   Doorbell 4   +-----+ |   +-----------------+
0439  *+-----------------+    |      |----------------+     | |   |                 |
0440  *|       BAR4      |    |      |                |     | |   +-----------------+
0441  *+-----------------+    |      |      MW1       +     | +-->+ MSI-X ADDRESS 3||
0442  *|       BAR5      |    |      |                |     |     +-----------------+
0443  *+-----------------+    +----->-----------------+     |     |                 |
0444  *  EP CONTROLLER 1             |                |     |     +-----------------+
0445  *                              |                |     +---->+ MSI-X ADDRESS 4 |
0446  *                              +----------------+           +-----------------+
0447  *                     (A)       EP CONTROLLER 2             |                 |
0448  *                                 (OB SPACE)                |                 |
0449  *                                                           |      MW1        |
0450  *                                                           |                 |
0451  *                                                           |                 |
0452  *                                                   (B)     +-----------------+
0453  *                                                           |                 |
0454  *                                                           |                 |
0455  *                                                           |                 |
0456  *                                                           |                 |
0457  *                                                           |                 |
0458  *                                                           +-----------------+
0459  *                                                           PCI Address Space
0460  *                                                           (Managed by HOST2)
0461  *
0462  * This function performs stage (B) in the above diagram (see Doorbell 1,
0463  * Doorbell 2, Doorbell 3, Doorbell 4) i.e map OB address space corresponding to
0464  * doorbell to MSI-X address in PCI address space.
0465  *
0466  * This operation requires 3 parameters
0467  *  1) Address reserved for doorbell in the outbound address space
0468  *  2) MSI-X address in the PCIe Address space
0469  *  3) Number of MSI-X interrupts that has to be configured
0470  *
0471  * The address in the outbound address space (for the Doorbell) is stored in
0472  * epf_bar corresponding to BAR_DB_MW1 of epf_ntb_epc that is connected to
0473  * HOST1. This is populated in epf_ntb_alloc_peer_mem() in this driver along
0474  * with address for MW1.
0475  *
0476  * The MSI-X address is in the MSI-X table of EP CONTROLLER 2 and
0477  * the count of doorbell is in ctrl->argument of epf_ntb_epc that is connected
0478  * to HOST2. MSI-X table is stored memory mapped to ntb_epc->msix_bar and the
0479  * offset is in ntb_epc->msix_table_offset. From this epf_ntb_configure_msix()
0480  * gets the MSI-X address and data.
0481  *
0482  * epf_ntb_configure_msix() also stores the MSI-X data to raise each interrupt
0483  * in db_data of the peer's control region. This helps the peer to raise
0484  * doorbell of the other host by writing db_data to the BAR corresponding to
0485  * BAR_DB_MW1.
0486  */
0487 static int epf_ntb_configure_msix(struct epf_ntb *ntb,
0488                   enum pci_epc_interface_type type,
0489                   u16 db_count)
0490 {
0491     const struct pci_epc_features *epc_features;
0492     struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
0493     struct pci_epf_bar *peer_epf_bar, *epf_bar;
0494     struct pci_epf_msix_tbl *msix_tbl;
0495     struct epf_ntb_ctrl *peer_ctrl;
0496     u32 db_entry_size, msg_data;
0497     enum pci_barno peer_barno;
0498     phys_addr_t phys_addr;
0499     u8 func_no, vfunc_no;
0500     struct pci_epc *epc;
0501     size_t align;
0502     u64 msg_addr;
0503     int ret, i;
0504
0505     ntb_epc = ntb->epc[type];
0506     epc = ntb_epc->epc;
0507
0508     epf_bar = &ntb_epc->epf_bar[ntb_epc->msix_bar];
0509     msix_tbl = epf_bar->addr + ntb_epc->msix_table_offset;
0510
0511     peer_ntb_epc = ntb->epc[!type];
0512     peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1];
0513     peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];
0514     phys_addr = peer_epf_bar->phys_addr;
0515     peer_ctrl = peer_ntb_epc->reg;
0516     epc_features = ntb_epc->epc_features;
0517     align = epc_features->align;
0518
0519     func_no = ntb_epc->func_no;
0520     vfunc_no = ntb_epc->vfunc_no;
0521     db_entry_size = peer_ctrl->db_entry_size;
0522
0523     for (i = 0; i < db_count; i++) {
0524         msg_addr = ALIGN_DOWN(msix_tbl[i].msg_addr, align);
0525         msg_data = msix_tbl[i].msg_data;
0526         ret = pci_epc_map_addr(epc, func_no, vfunc_no, phys_addr, msg_addr,
0527                        db_entry_size);
0528         if (ret) {
0529             dev_err(&epc->dev,
0530                 "%s intf: Failed to configure MSI-X IRQ\n",
0531                 pci_epc_interface_string(type));
0532             return ret;
0533         }
0534         phys_addr = phys_addr + db_entry_size;
0535         peer_ctrl->db_data[i] = msg_data;
0536         peer_ctrl->db_offset[i] = msix_tbl[i].msg_addr & (align - 1);
0537     }
0538     ntb_epc->is_msix = true;
0539
0540     return 0;
0541 }
0542
0543 /**
0544  * epf_ntb_configure_db() - Configure the Outbound Address Space for one host
0545  *   to ring the doorbell of other host
0546  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
0547  * @type: PRIMARY interface or SECONDARY interface
0548  * @db_count: Count of the number of doorbells that has to be configured
0549  * @msix: Indicates whether MSI-X or MSI should be used
0550  *
0551  * Invokes epf_ntb_configure_msix() or epf_ntb_configure_msi() required for
0552  * one HOST to ring the doorbell of other HOST.
0553  */
0554 static int epf_ntb_configure_db(struct epf_ntb *ntb,
0555                 enum pci_epc_interface_type type,
0556                 u16 db_count, bool msix)
0557 {
0558     struct epf_ntb_epc *ntb_epc;
0559     struct pci_epc *epc;
0560     int ret;
0561
0562     if (db_count > MAX_DB_COUNT)
0563         return -EINVAL;
0564
0565     ntb_epc = ntb->epc[type];
0566     epc = ntb_epc->epc;
0567
0568     if (msix)
0569         ret = epf_ntb_configure_msix(ntb, type, db_count);
0570     else
0571         ret = epf_ntb_configure_msi(ntb, type, db_count);
0572
0573     if (ret)
0574         dev_err(&epc->dev, "%s intf: Failed to configure DB\n",
0575             pci_epc_interface_string(type));
0576
0577     return ret;
0578 }
0579
0580 /**
0581  * epf_ntb_teardown_db() - Unmap address in OB address space to MSI/MSI-X
0582  *   address
0583  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
0584  * @type: PRIMARY interface or SECONDARY interface
0585  *
0586  * Invoke pci_epc_unmap_addr() to unmap OB address to MSI/MSI-X address.
0587  */
0588 static void
0589 epf_ntb_teardown_db(struct epf_ntb *ntb, enum pci_epc_interface_type type)
0590 {
0591     struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
0592     struct pci_epf_bar *peer_epf_bar;
0593     enum pci_barno peer_barno;
0594     phys_addr_t phys_addr;
0595     u8 func_no, vfunc_no;
0596     struct pci_epc *epc;
0597
0598     ntb_epc = ntb->epc[type];
0599     epc = ntb_epc->epc;
0600
0601     peer_ntb_epc = ntb->epc[!type];
0602     peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1];
0603     peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];
0604     phys_addr = peer_epf_bar->phys_addr;
0605     func_no = ntb_epc->func_no;
0606     vfunc_no = ntb_epc->vfunc_no;
0607
0608     pci_epc_unmap_addr(epc, func_no, vfunc_no, phys_addr);
0609 }
0610
0611 /**
0612  * epf_ntb_cmd_handler() - Handle commands provided by the NTB Host
0613  * @work: work_struct for the two epf_ntb_epc (PRIMARY and SECONDARY)
0614  *
0615  * Workqueue function that gets invoked for the two epf_ntb_epc
0616  * periodically (once every 5ms) to see if it has received any commands
0617  * from NTB host. The host can send commands to configure doorbell or
0618  * configure memory window or to update link status.
0619  */
0620 static void epf_ntb_cmd_handler(struct work_struct *work)
0621 {
0622     enum pci_epc_interface_type type;
0623     struct epf_ntb_epc *ntb_epc;
0624     struct epf_ntb_ctrl *ctrl;
0625     u32 command, argument;
0626     struct epf_ntb *ntb;
0627     struct device *dev;
0628     u16 db_count;
0629     bool is_msix;
0630     int ret;
0631
0632     ntb_epc = container_of(work, struct epf_ntb_epc, cmd_handler.work);
0633     ctrl = ntb_epc->reg;
0634     command = ctrl->command;
0635     if (!command)
0636         goto reset_handler;
0637     argument = ctrl->argument;
0638
0639     ctrl->command = 0;
0640     ctrl->argument = 0;
0641
0642     ctrl = ntb_epc->reg;
0643     type = ntb_epc->type;
0644     ntb = ntb_epc->epf_ntb;
0645     dev = &ntb->epf->dev;
0646
0647     switch (command) {
0648     case COMMAND_CONFIGURE_DOORBELL:
0649         db_count = argument & DB_COUNT_MASK;
0650         is_msix = argument & MSIX_ENABLE;
0651         ret = epf_ntb_configure_db(ntb, type, db_count, is_msix);
0652         if (ret < 0)
0653             ctrl->command_status = COMMAND_STATUS_ERROR;
0654         else
0655             ctrl->command_status = COMMAND_STATUS_OK;
0656         break;
0657     case COMMAND_TEARDOWN_DOORBELL:
0658         epf_ntb_teardown_db(ntb, type);
0659         ctrl->command_status = COMMAND_STATUS_OK;
0660         break;
0661     case COMMAND_CONFIGURE_MW:
0662         ret = epf_ntb_configure_mw(ntb, type, argument);
0663         if (ret < 0)
0664             ctrl->command_status = COMMAND_STATUS_ERROR;
0665         else
0666             ctrl->command_status = COMMAND_STATUS_OK;
0667         break;
0668     case COMMAND_TEARDOWN_MW:
0669         epf_ntb_teardown_mw(ntb, type, argument);
0670         ctrl->command_status = COMMAND_STATUS_OK;
0671         break;
0672     case COMMAND_LINK_UP:
0673         ntb_epc->linkup = true;
0674         if (ntb->epc[PRIMARY_INTERFACE]->linkup &&
0675             ntb->epc[SECONDARY_INTERFACE]->linkup) {
0676             ret = epf_ntb_link_up(ntb, true);
0677             if (ret < 0)
0678                 ctrl->command_status = COMMAND_STATUS_ERROR;
0679             else
0680                 ctrl->command_status = COMMAND_STATUS_OK;
0681             goto reset_handler;
0682         }
0683         ctrl->command_status = COMMAND_STATUS_OK;
0684         break;
0685     case COMMAND_LINK_DOWN:
0686         ntb_epc->linkup = false;
0687         ret = epf_ntb_link_up(ntb, false);
0688         if (ret < 0)
0689             ctrl->command_status = COMMAND_STATUS_ERROR;
0690         else
0691             ctrl->command_status = COMMAND_STATUS_OK;
0692         break;
0693     default:
0694         dev_err(dev, "%s intf UNKNOWN command: %d\n",
0695             pci_epc_interface_string(type), command);
0696         break;
0697     }
0698
0699 reset_handler:
0700     queue_delayed_work(kpcintb_workqueue, &ntb_epc->cmd_handler,
0701                msecs_to_jiffies(5));
0702 }
0703
0704 /**
0705  * epf_ntb_peer_spad_bar_clear() - Clear Peer Scratchpad BAR
0706  * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound
0707  *       address.
0708  *
0709  *+-----------------+------->+------------------+        +-----------------+
0710  *|       BAR0      |        |  CONFIG REGION   |        |       BAR0      |
0711  *+-----------------+----+   +------------------+<-------+-----------------+
0712  *|       BAR1      |    |   |SCRATCHPAD REGION |        |       BAR1      |
0713  *+-----------------+    +-->+------------------+<-------+-----------------+
0714  *|       BAR2      |            Local Memory            |       BAR2      |
0715  *+-----------------+                                    +-----------------+
0716  *|       BAR3      |                                    |       BAR3      |
0717  *+-----------------+                                    +-----------------+
0718  *|       BAR4      |                                    |       BAR4      |
0719  *+-----------------+                                    +-----------------+
0720  *|       BAR5      |                                    |       BAR5      |
0721  *+-----------------+                                    +-----------------+
0722  *  EP CONTROLLER 1                                        EP CONTROLLER 2
0723  *
0724  * Clear BAR1 of EP CONTROLLER 2 which contains the HOST2's peer scratchpad
0725  * region. While BAR1 is the default peer scratchpad BAR, an NTB could have
0726  * other BARs for peer scratchpad (because of 64-bit BARs or reserved BARs).
0727  * This function can get the exact BAR used for peer scratchpad from
0728  * epf_ntb_bar[BAR_PEER_SPAD].
0729  *
0730  * Since HOST2's peer scratchpad is also HOST1's self scratchpad, this function
0731  * gets the address of peer scratchpad from
0732  * peer_ntb_epc->epf_ntb_bar[BAR_CONFIG].
0733  */
0734 static void epf_ntb_peer_spad_bar_clear(struct epf_ntb_epc *ntb_epc)
0735 {
0736     struct pci_epf_bar *epf_bar;
0737     enum pci_barno barno;
0738     u8 func_no, vfunc_no;
0739     struct pci_epc *epc;
0740
0741     epc = ntb_epc->epc;
0742     func_no = ntb_epc->func_no;
0743     vfunc_no = ntb_epc->vfunc_no;
0744     barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD];
0745     epf_bar = &ntb_epc->epf_bar[barno];
0746     pci_epc_clear_bar(epc, func_no, vfunc_no, epf_bar);
0747 }
0748
0749 /**
0750  * epf_ntb_peer_spad_bar_set() - Set peer scratchpad BAR
0751  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
0752  * @type: PRIMARY interface or SECONDARY interface
0753  *
0754  *+-----------------+------->+------------------+        +-----------------+
0755  *|       BAR0      |        |  CONFIG REGION   |        |       BAR0      |
0756  *+-----------------+----+   +------------------+<-------+-----------------+
0757  *|       BAR1      |    |   |SCRATCHPAD REGION |        |       BAR1      |
0758  *+-----------------+    +-->+------------------+<-------+-----------------+
0759  *|       BAR2      |            Local Memory            |       BAR2      |
0760  *+-----------------+                                    +-----------------+
0761  *|       BAR3      |                                    |       BAR3      |
0762  *+-----------------+                                    +-----------------+
0763  *|       BAR4      |                                    |       BAR4      |
0764  *+-----------------+                                    +-----------------+
0765  *|       BAR5      |                                    |       BAR5      |
0766  *+-----------------+                                    +-----------------+
0767  *  EP CONTROLLER 1                                        EP CONTROLLER 2
0768  *
0769  * Set BAR1 of EP CONTROLLER 2 which contains the HOST2's peer scratchpad
0770  * region. While BAR1 is the default peer scratchpad BAR, an NTB could have
0771  * other BARs for peer scratchpad (because of 64-bit BARs or reserved BARs).
0772  * This function can get the exact BAR used for peer scratchpad from
0773  * epf_ntb_bar[BAR_PEER_SPAD].
0774  *
0775  * Since HOST2's peer scratchpad is also HOST1's self scratchpad, this function
0776  * gets the address of peer scratchpad from
0777  * peer_ntb_epc->epf_ntb_bar[BAR_CONFIG].
0778  */
0779 static int epf_ntb_peer_spad_bar_set(struct epf_ntb *ntb,
0780                      enum pci_epc_interface_type type)
0781 {
0782     struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
0783     struct pci_epf_bar *peer_epf_bar, *epf_bar;
0784     enum pci_barno peer_barno, barno;
0785     u32 peer_spad_offset;
0786     u8 func_no, vfunc_no;
0787     struct pci_epc *epc;
0788     struct device *dev;
0789     int ret;
0790
0791     dev = &ntb->epf->dev;
0792
0793     peer_ntb_epc = ntb->epc[!type];
0794     peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_CONFIG];
0795     peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];
0796
0797     ntb_epc = ntb->epc[type];
0798     barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD];
0799     epf_bar = &ntb_epc->epf_bar[barno];
0800     func_no = ntb_epc->func_no;
0801     vfunc_no = ntb_epc->vfunc_no;
0802     epc = ntb_epc->epc;
0803
0804     peer_spad_offset = peer_ntb_epc->reg->spad_offset;
0805     epf_bar->phys_addr = peer_epf_bar->phys_addr + peer_spad_offset;
0806     epf_bar->size = peer_ntb_epc->spad_size;
0807     epf_bar->barno = barno;
0808     epf_bar->flags = PCI_BASE_ADDRESS_MEM_TYPE_32;
0809
0810     ret = pci_epc_set_bar(epc, func_no, vfunc_no, epf_bar);
0811     if (ret) {
0812         dev_err(dev, "%s intf: peer SPAD BAR set failed\n",
0813             pci_epc_interface_string(type));
0814         return ret;
0815     }
0816
0817     return 0;
0818 }
0819
0820 /**
0821  * epf_ntb_config_sspad_bar_clear() - Clear Config + Self scratchpad BAR
0822  * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound
0823  *       address.
0824  *
0825  * +-----------------+------->+------------------+        +-----------------+
0826  * |       BAR0      |        |  CONFIG REGION   |        |       BAR0      |
0827  * +-----------------+----+   +------------------+<-------+-----------------+
0828  * |       BAR1      |    |   |SCRATCHPAD REGION |        |       BAR1      |
0829  * +-----------------+    +-->+------------------+<-------+-----------------+
0830  * |       BAR2      |            Local Memory            |       BAR2      |
0831  * +-----------------+                                    +-----------------+
0832  * |       BAR3      |                                    |       BAR3      |
0833  * +-----------------+                                    +-----------------+
0834  * |       BAR4      |                                    |       BAR4      |
0835  * +-----------------+                                    +-----------------+
0836  * |       BAR5      |                                    |       BAR5      |
0837  * +-----------------+                                    +-----------------+
0838  *   EP CONTROLLER 1                                        EP CONTROLLER 2
0839  *
0840  * Clear BAR0 of EP CONTROLLER 1 which contains the HOST1's config and
0841  * self scratchpad region (removes inbound ATU configuration). While BAR0 is
0842  * the default self scratchpad BAR, an NTB could have other BARs for self
0843  * scratchpad (because of reserved BARs). This function can get the exact BAR
0844  * used for self scratchpad from epf_ntb_bar[BAR_CONFIG].
0845  *
0846  * Please note the self scratchpad region and config region is combined to
0847  * a single region and mapped using the same BAR. Also note HOST2's peer
0848  * scratchpad is HOST1's self scratchpad.
0849  */
0850 static void epf_ntb_config_sspad_bar_clear(struct epf_ntb_epc *ntb_epc)
0851 {
0852     struct pci_epf_bar *epf_bar;
0853     enum pci_barno barno;
0854     u8 func_no, vfunc_no;
0855     struct pci_epc *epc;
0856
0857     epc = ntb_epc->epc;
0858     func_no = ntb_epc->func_no;
0859     vfunc_no = ntb_epc->vfunc_no;
0860     barno = ntb_epc->epf_ntb_bar[BAR_CONFIG];
0861     epf_bar = &ntb_epc->epf_bar[barno];
0862     pci_epc_clear_bar(epc, func_no, vfunc_no, epf_bar);
0863 }
0864
0865 /**
0866  * epf_ntb_config_sspad_bar_set() - Set Config + Self scratchpad BAR
0867  * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound
0868  *       address.
0869  *
0870  * +-----------------+------->+------------------+        +-----------------+
0871  * |       BAR0      |        |  CONFIG REGION   |        |       BAR0      |
0872  * +-----------------+----+   +------------------+<-------+-----------------+
0873  * |       BAR1      |    |   |SCRATCHPAD REGION |        |       BAR1      |
0874  * +-----------------+    +-->+------------------+<-------+-----------------+
0875  * |       BAR2      |            Local Memory            |       BAR2      |
0876  * +-----------------+                                    +-----------------+
0877  * |       BAR3      |                                    |       BAR3      |
0878  * +-----------------+                                    +-----------------+
0879  * |       BAR4      |                                    |       BAR4      |
0880  * +-----------------+                                    +-----------------+
0881  * |       BAR5      |                                    |       BAR5      |
0882  * +-----------------+                                    +-----------------+
0883  *   EP CONTROLLER 1                                        EP CONTROLLER 2
0884  *
0885  * Map BAR0 of EP CONTROLLER 1 which contains the HOST1's config and
0886  * self scratchpad region. While BAR0 is the default self scratchpad BAR, an
0887  * NTB could have other BARs for self scratchpad (because of reserved BARs).
0888  * This function can get the exact BAR used for self scratchpad from
0889  * epf_ntb_bar[BAR_CONFIG].
0890  *
0891  * Please note the self scratchpad region and config region is combined to
0892  * a single region and mapped using the same BAR. Also note HOST2's peer
0893  * scratchpad is HOST1's self scratchpad.
0894  */
0895 static int epf_ntb_config_sspad_bar_set(struct epf_ntb_epc *ntb_epc)
0896 {
0897     struct pci_epf_bar *epf_bar;
0898     enum pci_barno barno;
0899     u8 func_no, vfunc_no;
0900     struct epf_ntb *ntb;
0901     struct pci_epc *epc;
0902     struct device *dev;
0903     int ret;
0904
0905     ntb = ntb_epc->epf_ntb;
0906     dev = &ntb->epf->dev;
0907
0908     epc = ntb_epc->epc;
0909     func_no = ntb_epc->func_no;
0910     vfunc_no = ntb_epc->vfunc_no;
0911     barno = ntb_epc->epf_ntb_bar[BAR_CONFIG];
0912     epf_bar = &ntb_epc->epf_bar[barno];
0913
0914     ret = pci_epc_set_bar(epc, func_no, vfunc_no, epf_bar);
0915     if (ret) {
0916         dev_err(dev, "%s inft: Config/Status/SPAD BAR set failed\n",
0917             pci_epc_interface_string(ntb_epc->type));
0918         return ret;
0919     }
0920
0921     return 0;
0922 }
0923
0924 /**
0925  * epf_ntb_config_spad_bar_free() - Free the physical memory associated with
0926  *   config + scratchpad region
0927  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
0928  *
0929  * +-----------------+------->+------------------+        +-----------------+
0930  * |       BAR0      |        |  CONFIG REGION   |        |       BAR0      |
0931  * +-----------------+----+   +------------------+<-------+-----------------+
0932  * |       BAR1      |    |   |SCRATCHPAD REGION |        |       BAR1      |
0933  * +-----------------+    +-->+------------------+<-------+-----------------+
0934  * |       BAR2      |            Local Memory            |       BAR2      |
0935  * +-----------------+                                    +-----------------+
0936  * |       BAR3      |                                    |       BAR3      |
0937  * +-----------------+                                    +-----------------+
0938  * |       BAR4      |                                    |       BAR4      |
0939  * +-----------------+                                    +-----------------+
0940  * |       BAR5      |                                    |       BAR5      |
0941  * +-----------------+                                    +-----------------+
0942  *   EP CONTROLLER 1                                        EP CONTROLLER 2
0943  *
0944  * Free the Local Memory mentioned in the above diagram. After invoking this
0945  * function, any of config + self scratchpad region of HOST1 or peer scratchpad
0946  * region of HOST2 should not be accessed.
0947  */
0948 static void epf_ntb_config_spad_bar_free(struct epf_ntb *ntb)
0949 {
0950     enum pci_epc_interface_type type;
0951     struct epf_ntb_epc *ntb_epc;
0952     enum pci_barno barno;
0953     struct pci_epf *epf;
0954
0955     epf = ntb->epf;
0956     for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {
0957         ntb_epc = ntb->epc[type];
0958         barno = ntb_epc->epf_ntb_bar[BAR_CONFIG];
0959         if (ntb_epc->reg)
0960             pci_epf_free_space(epf, ntb_epc->reg, barno, type);
0961     }
0962 }
0963
0964 /**
0965  * epf_ntb_config_spad_bar_alloc() - Allocate memory for config + scratchpad
0966  *   region
0967  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
0968  * @type: PRIMARY interface or SECONDARY interface
0969  *
0970  * +-----------------+------->+------------------+        +-----------------+
0971  * |       BAR0      |        |  CONFIG REGION   |        |       BAR0      |
0972  * +-----------------+----+   +------------------+<-------+-----------------+
0973  * |       BAR1      |    |   |SCRATCHPAD REGION |        |       BAR1      |
0974  * +-----------------+    +-->+------------------+<-------+-----------------+
0975  * |       BAR2      |            Local Memory            |       BAR2      |
0976  * +-----------------+                                    +-----------------+
0977  * |       BAR3      |                                    |       BAR3      |
0978  * +-----------------+                                    +-----------------+
0979  * |       BAR4      |                                    |       BAR4      |
0980  * +-----------------+                                    +-----------------+
0981  * |       BAR5      |                                    |       BAR5      |
0982  * +-----------------+                                    +-----------------+
0983  *   EP CONTROLLER 1                                        EP CONTROLLER 2
0984  *
0985  * Allocate the Local Memory mentioned in the above diagram. The size of
0986  * CONFIG REGION is sizeof(struct epf_ntb_ctrl) and size of SCRATCHPAD REGION
0987  * is obtained from "spad-count" configfs entry.
0988  *
0989  * The size of both config region and scratchpad region has to be aligned,
0990  * since the scratchpad region will also be mapped as PEER SCRATCHPAD of
0991  * other host using a separate BAR.
0992  */
0993 static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb,
0994                      enum pci_epc_interface_type type)
0995 {
0996     const struct pci_epc_features *peer_epc_features, *epc_features;
0997     struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
0998     size_t msix_table_size, pba_size, align;
0999     enum pci_barno peer_barno, barno;
1000     struct epf_ntb_ctrl *ctrl;
1001     u32 spad_size, ctrl_size;
1002     u64 size, peer_size;
1003     struct pci_epf *epf;
1004     struct device *dev;
1005     bool msix_capable;
1006     u32 spad_count;
1007     void *base;
1008
1009     epf = ntb->epf;
1010     dev = &epf->dev;
1011     ntb_epc = ntb->epc[type];
1012
1013     epc_features = ntb_epc->epc_features;
1014     barno = ntb_epc->epf_ntb_bar[BAR_CONFIG];
1015     size = epc_features->bar_fixed_size[barno];
1016     align = epc_features->align;
1017
1018     peer_ntb_epc = ntb->epc[!type];
1019     peer_epc_features = peer_ntb_epc->epc_features;
1020     peer_barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD];
1021     peer_size = peer_epc_features->bar_fixed_size[peer_barno];
1022
1023     /* Check if epc_features is populated incorrectly */
1024     if ((!IS_ALIGNED(size, align)))
1025         return -EINVAL;
1026
1027     spad_count = ntb->spad_count;
1028
1029     ctrl_size = sizeof(struct epf_ntb_ctrl);
1030     spad_size = spad_count * 4;
1031
1032     msix_capable = epc_features->msix_capable;
1033     if (msix_capable) {
1034         msix_table_size = PCI_MSIX_ENTRY_SIZE * ntb->db_count;
1035         ctrl_size = ALIGN(ctrl_size, 8);
1036         ntb_epc->msix_table_offset = ctrl_size;
1037         ntb_epc->msix_bar = barno;
1038         /* Align to QWORD or 8 Bytes */
1039         pba_size = ALIGN(DIV_ROUND_UP(ntb->db_count, 8), 8);
1040         ctrl_size = ctrl_size + msix_table_size + pba_size;
1041     }
1042
1043     if (!align) {
1044         ctrl_size = roundup_pow_of_two(ctrl_size);
1045         spad_size = roundup_pow_of_two(spad_size);
1046     } else {
1047         ctrl_size = ALIGN(ctrl_size, align);
1048         spad_size = ALIGN(spad_size, align);
1049     }
1050
1051     if (peer_size) {
1052         if (peer_size < spad_size)
1053             spad_count = peer_size / 4;
1054         spad_size = peer_size;
1055     }
1056
1057     /*
1058      * In order to make sure SPAD offset is aligned to its size,
1059      * expand control region size to the size of SPAD if SPAD size
1060      * is greater than control region size.
1061      */
1062     if (spad_size > ctrl_size)
1063         ctrl_size = spad_size;
1064
1065     if (!size)
1066         size = ctrl_size + spad_size;
1067     else if (size < ctrl_size + spad_size)
1068         return -EINVAL;
1069
1070     base = pci_epf_alloc_space(epf, size, barno, align, type);
1071     if (!base) {
1072         dev_err(dev, "%s intf: Config/Status/SPAD alloc region fail\n",
1073             pci_epc_interface_string(type));
1074         return -ENOMEM;
1075     }
1076
1077     ntb_epc->reg = base;
1078
1079     ctrl = ntb_epc->reg;
1080     ctrl->spad_offset = ctrl_size;
1081     ctrl->spad_count = spad_count;
1082     ctrl->num_mws = ntb->num_mws;
1083     ctrl->db_entry_size = align ? align : 4;
1084     ntb_epc->spad_size = spad_size;
1085
1086     return 0;
1087 }
1088
1089 /**
1090  * epf_ntb_config_spad_bar_alloc_interface() - Allocate memory for config +
1091  *   scratchpad region for each of PRIMARY and SECONDARY interface
1092  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1093  *
1094  * Wrapper for epf_ntb_config_spad_bar_alloc() which allocates memory for
1095  * config + scratchpad region for a specific interface
1096  */
1097 static int epf_ntb_config_spad_bar_alloc_interface(struct epf_ntb *ntb)
1098 {
1099     enum pci_epc_interface_type type;
1100     struct device *dev;
1101     int ret;
1102
1103     dev = &ntb->epf->dev;
1104
1105     for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {
1106         ret = epf_ntb_config_spad_bar_alloc(ntb, type);
1107         if (ret) {
1108             dev_err(dev, "%s intf: Config/SPAD BAR alloc failed\n",
1109                 pci_epc_interface_string(type));
1110             return ret;
1111         }
1112     }
1113
1114     return 0;
1115 }
1116
1117 /**
1118  * epf_ntb_free_peer_mem() - Free memory allocated in peers outbound address
1119  *   space
1120  * @ntb_epc: EPC associated with one of the HOST which holds peers outbound
1121  *   address regions
1122  *
1123  * +-----------------+    +---->+----------------+-----------+-----------------+
1124  * |       BAR0      |    |     |   Doorbell 1   +-----------> MSI|X ADDRESS 1 |
1125  * +-----------------+    |     +----------------+           +-----------------+
1126  * |       BAR1      |    |     |   Doorbell 2   +---------+ |                 |
1127  * +-----------------+----+     +----------------+         | |                 |
1128  * |       BAR2      |          |   Doorbell 3   +-------+ | +-----------------+
1129  * +-----------------+----+     +----------------+       | +-> MSI|X ADDRESS 2 |
1130  * |       BAR3      |    |     |   Doorbell 4   +-----+ |   +-----------------+
1131  * +-----------------+    |     |----------------+     | |   |                 |
1132  * |       BAR4      |    |     |                |     | |   +-----------------+
1133  * +-----------------+    |     |      MW1       +---+ | +-->+ MSI|X ADDRESS 3||
1134  * |       BAR5      |    |     |                |   | |     +-----------------+
1135  * +-----------------+    +---->-----------------+   | |     |                 |
1136  *   EP CONTROLLER 1            |                |   | |     +-----------------+
1137  *                              |                |   | +---->+ MSI|X ADDRESS 4 |
1138  *                              +----------------+   |       +-----------------+
1139  *                      (A)      EP CONTROLLER 2     |       |                 |
1140  *                                 (OB SPACE)        |       |                 |
1141  *                                                   +------->      MW1        |
1142  *                                                           |                 |
1143  *                                                           |                 |
1144  *                                                   (B)     +-----------------+
1145  *                                                           |                 |
1146  *                                                           |                 |
1147  *                                                           |                 |
1148  *                                                           |                 |
1149  *                                                           |                 |
1150  *                                                           +-----------------+
1151  *                                                           PCI Address Space
1152  *                                                           (Managed by HOST2)
1153  *
1154  * Free memory allocated in EP CONTROLLER 2 (OB SPACE) in the above diagram.
1155  * It'll free Doorbell 1, Doorbell 2, Doorbell 3, Doorbell 4, MW1 (and MW2, MW3,
1156  * MW4).
1157  */
1158 static void epf_ntb_free_peer_mem(struct epf_ntb_epc *ntb_epc)
1159 {
1160     struct pci_epf_bar *epf_bar;
1161     void __iomem *mw_addr;
1162     phys_addr_t phys_addr;
1163     enum epf_ntb_bar bar;
1164     enum pci_barno barno;
1165     struct pci_epc *epc;
1166     size_t size;
1167
1168     epc = ntb_epc->epc;
1169
1170     for (bar = BAR_DB_MW1; bar < BAR_MW4; bar++) {
1171         barno = ntb_epc->epf_ntb_bar[bar];
1172         mw_addr = ntb_epc->mw_addr[barno];
1173         epf_bar = &ntb_epc->epf_bar[barno];
1174         phys_addr = epf_bar->phys_addr;
1175         size = epf_bar->size;
1176         if (mw_addr) {
1177             pci_epc_mem_free_addr(epc, phys_addr, mw_addr, size);
1178             ntb_epc->mw_addr[barno] = NULL;
1179         }
1180     }
1181 }
1182
1183 /**
1184  * epf_ntb_db_mw_bar_clear() - Clear doorbell and memory BAR
1185  * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound
1186  *   address
1187  *
1188  * +-----------------+    +---->+----------------+-----------+-----------------+
1189  * |       BAR0      |    |     |   Doorbell 1   +-----------> MSI|X ADDRESS 1 |
1190  * +-----------------+    |     +----------------+           +-----------------+
1191  * |       BAR1      |    |     |   Doorbell 2   +---------+ |                 |
1192  * +-----------------+----+     +----------------+         | |                 |
1193  * |       BAR2      |          |   Doorbell 3   +-------+ | +-----------------+
1194  * +-----------------+----+     +----------------+       | +-> MSI|X ADDRESS 2 |
1195  * |       BAR3      |    |     |   Doorbell 4   +-----+ |   +-----------------+
1196  * +-----------------+    |     |----------------+     | |   |                 |
1197  * |       BAR4      |    |     |                |     | |   +-----------------+
1198  * +-----------------+    |     |      MW1       +---+ | +-->+ MSI|X ADDRESS 3||
1199  * |       BAR5      |    |     |                |   | |     +-----------------+
1200  * +-----------------+    +---->-----------------+   | |     |                 |
1201  *   EP CONTROLLER 1            |                |   | |     +-----------------+
1202  *                              |                |   | +---->+ MSI|X ADDRESS 4 |
1203  *                              +----------------+   |       +-----------------+
1204  *                      (A)      EP CONTROLLER 2     |       |                 |
1205  *                                 (OB SPACE)        |       |                 |
1206  *                                                   +------->      MW1        |
1207  *                                                           |                 |
1208  *                                                           |                 |
1209  *                                                   (B)     +-----------------+
1210  *                                                           |                 |
1211  *                                                           |                 |
1212  *                                                           |                 |
1213  *                                                           |                 |
1214  *                                                           |                 |
1215  *                                                           +-----------------+
1216  *                                                           PCI Address Space
1217  *                                                           (Managed by HOST2)
1218  *
1219  * Clear doorbell and memory BARs (remove inbound ATU configuration). In the above
1220  * diagram it clears BAR2 TO BAR5 of EP CONTROLLER 1 (Doorbell BAR, MW1 BAR, MW2
1221  * BAR, MW3 BAR and MW4 BAR).
1222  */
1223 static void epf_ntb_db_mw_bar_clear(struct epf_ntb_epc *ntb_epc)
1224 {
1225     struct pci_epf_bar *epf_bar;
1226     enum epf_ntb_bar bar;
1227     enum pci_barno barno;
1228     u8 func_no, vfunc_no;
1229     struct pci_epc *epc;
1230
1231     epc = ntb_epc->epc;
1232
1233     func_no = ntb_epc->func_no;
1234     vfunc_no = ntb_epc->vfunc_no;
1235
1236     for (bar = BAR_DB_MW1; bar < BAR_MW4; bar++) {
1237         barno = ntb_epc->epf_ntb_bar[bar];
1238         epf_bar = &ntb_epc->epf_bar[barno];
1239         pci_epc_clear_bar(epc, func_no, vfunc_no, epf_bar);
1240     }
1241 }
1242
1243 /**
1244  * epf_ntb_db_mw_bar_cleanup() - Clear doorbell/memory BAR and free memory
1245  *   allocated in peers outbound address space
1246  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1247  * @type: PRIMARY interface or SECONDARY interface
1248  *
1249  * Wrapper for epf_ntb_db_mw_bar_clear() to clear HOST1's BAR and
1250  * epf_ntb_free_peer_mem() which frees up HOST2 outbound memory.
1251  */
1252 static void epf_ntb_db_mw_bar_cleanup(struct epf_ntb *ntb,
1253                       enum pci_epc_interface_type type)
1254 {
1255     struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
1256
1257     ntb_epc = ntb->epc[type];
1258     peer_ntb_epc = ntb->epc[!type];
1259
1260     epf_ntb_db_mw_bar_clear(ntb_epc);
1261     epf_ntb_free_peer_mem(peer_ntb_epc);
1262 }
1263
1264 /**
1265  * epf_ntb_configure_interrupt() - Configure MSI/MSI-X capability
1266  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1267  * @type: PRIMARY interface or SECONDARY interface
1268  *
1269  * Configure MSI/MSI-X capability for each interface with number of
1270  * interrupts equal to "db_count" configfs entry.
1271  */
1272 static int epf_ntb_configure_interrupt(struct epf_ntb *ntb,
1273                        enum pci_epc_interface_type type)
1274 {
1275     const struct pci_epc_features *epc_features;
1276     bool msix_capable, msi_capable;
1277     struct epf_ntb_epc *ntb_epc;
1278     u8 func_no, vfunc_no;
1279     struct pci_epc *epc;
1280     struct device *dev;
1281     u32 db_count;
1282     int ret;
1283
1284     ntb_epc = ntb->epc[type];
1285     dev = &ntb->epf->dev;
1286
1287     epc_features = ntb_epc->epc_features;
1288     msix_capable = epc_features->msix_capable;
1289     msi_capable = epc_features->msi_capable;
1290
1291     if (!(msix_capable || msi_capable)) {
1292         dev_err(dev, "MSI or MSI-X is required for doorbell\n");
1293         return -EINVAL;
1294     }
1295
1296     func_no = ntb_epc->func_no;
1297     vfunc_no = ntb_epc->vfunc_no;
1298
1299     db_count = ntb->db_count;
1300     if (db_count > MAX_DB_COUNT) {
1301         dev_err(dev, "DB count cannot be more than %d\n", MAX_DB_COUNT);
1302         return -EINVAL;
1303     }
1304
1305     ntb->db_count = db_count;
1306     epc = ntb_epc->epc;
1307
1308     if (msi_capable) {
1309         ret = pci_epc_set_msi(epc, func_no, vfunc_no, db_count);
1310         if (ret) {
1311             dev_err(dev, "%s intf: MSI configuration failed\n",
1312                 pci_epc_interface_string(type));
1313             return ret;
1314         }
1315     }
1316
1317     if (msix_capable) {
1318         ret = pci_epc_set_msix(epc, func_no, vfunc_no, db_count,
1319                        ntb_epc->msix_bar,
1320                        ntb_epc->msix_table_offset);
1321         if (ret) {
1322             dev_err(dev, "MSI configuration failed\n");
1323             return ret;
1324         }
1325     }
1326
1327     return 0;
1328 }
1329
1330 /**
1331  * epf_ntb_alloc_peer_mem() - Allocate memory in peer's outbound address space
1332  * @dev: The PCI device.
1333  * @ntb_epc: EPC associated with one of the HOST whose BAR holds peer's outbound
1334  *   address
1335  * @bar: BAR of @ntb_epc in for which memory has to be allocated (could be
1336  *   BAR_DB_MW1, BAR_MW2, BAR_MW3, BAR_MW4)
1337  * @peer_ntb_epc: EPC associated with HOST whose outbound address space is
1338  *   used by @ntb_epc
1339  * @size: Size of the address region that has to be allocated in peers OB SPACE
1340  *
1341  *
1342  * +-----------------+    +---->+----------------+-----------+-----------------+
1343  * |       BAR0      |    |     |   Doorbell 1   +-----------> MSI|X ADDRESS 1 |
1344  * +-----------------+    |     +----------------+           +-----------------+
1345  * |       BAR1      |    |     |   Doorbell 2   +---------+ |                 |
1346  * +-----------------+----+     +----------------+         | |                 |
1347  * |       BAR2      |          |   Doorbell 3   +-------+ | +-----------------+
1348  * +-----------------+----+     +----------------+       | +-> MSI|X ADDRESS 2 |
1349  * |       BAR3      |    |     |   Doorbell 4   +-----+ |   +-----------------+
1350  * +-----------------+    |     |----------------+     | |   |                 |
1351  * |       BAR4      |    |     |                |     | |   +-----------------+
1352  * +-----------------+    |     |      MW1       +---+ | +-->+ MSI|X ADDRESS 3||
1353  * |       BAR5      |    |     |                |   | |     +-----------------+
1354  * +-----------------+    +---->-----------------+   | |     |                 |
1355  *   EP CONTROLLER 1            |                |   | |     +-----------------+
1356  *                              |                |   | +---->+ MSI|X ADDRESS 4 |
1357  *                              +----------------+   |       +-----------------+
1358  *                      (A)      EP CONTROLLER 2     |       |                 |
1359  *                                 (OB SPACE)        |       |                 |
1360  *                                                   +------->      MW1        |
1361  *                                                           |                 |
1362  *                                                           |                 |
1363  *                                                   (B)     +-----------------+
1364  *                                                           |                 |
1365  *                                                           |                 |
1366  *                                                           |                 |
1367  *                                                           |                 |
1368  *                                                           |                 |
1369  *                                                           +-----------------+
1370  *                                                           PCI Address Space
1371  *                                                           (Managed by HOST2)
1372  *
1373  * Allocate memory in OB space of EP CONTROLLER 2 in the above diagram. Allocate
1374  * for Doorbell 1, Doorbell 2, Doorbell 3, Doorbell 4, MW1 (and MW2, MW3, MW4).
1375  */
1376 static int epf_ntb_alloc_peer_mem(struct device *dev,
1377                   struct epf_ntb_epc *ntb_epc,
1378                   enum epf_ntb_bar bar,
1379                   struct epf_ntb_epc *peer_ntb_epc,
1380                   size_t size)
1381 {
1382     const struct pci_epc_features *epc_features;
1383     struct pci_epf_bar *epf_bar;
1384     struct pci_epc *peer_epc;
1385     phys_addr_t phys_addr;
1386     void __iomem *mw_addr;
1387     enum pci_barno barno;
1388     size_t align;
1389
1390     epc_features = ntb_epc->epc_features;
1391     align = epc_features->align;
1392
1393     if (size < 128)
1394         size = 128;
1395
1396     if (align)
1397         size = ALIGN(size, align);
1398     else
1399         size = roundup_pow_of_two(size);
1400
1401     peer_epc = peer_ntb_epc->epc;
1402     mw_addr = pci_epc_mem_alloc_addr(peer_epc, &phys_addr, size);
1403     if (!mw_addr) {
1404         dev_err(dev, "%s intf: Failed to allocate OB address\n",
1405             pci_epc_interface_string(peer_ntb_epc->type));
1406         return -ENOMEM;
1407     }
1408
1409     barno = ntb_epc->epf_ntb_bar[bar];
1410     epf_bar = &ntb_epc->epf_bar[barno];
1411     ntb_epc->mw_addr[barno] = mw_addr;
1412
1413     epf_bar->phys_addr = phys_addr;
1414     epf_bar->size = size;
1415     epf_bar->barno = barno;
1416     epf_bar->flags = PCI_BASE_ADDRESS_MEM_TYPE_32;
1417
1418     return 0;
1419 }
1420
1421 /**
1422  * epf_ntb_db_mw_bar_init() - Configure Doorbell and Memory window BARs
1423  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1424  * @type: PRIMARY interface or SECONDARY interface
1425  *
1426  * Wrapper for epf_ntb_alloc_peer_mem() and pci_epc_set_bar() that allocates
1427  * memory in OB address space of HOST2 and configures BAR of HOST1
1428  */
1429 static int epf_ntb_db_mw_bar_init(struct epf_ntb *ntb,
1430                   enum pci_epc_interface_type type)
1431 {
1432     const struct pci_epc_features *epc_features;
1433     struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
1434     struct pci_epf_bar *epf_bar;
1435     struct epf_ntb_ctrl *ctrl;
1436     u32 num_mws, db_count;
1437     enum epf_ntb_bar bar;
1438     enum pci_barno barno;
1439     u8 func_no, vfunc_no;
1440     struct pci_epc *epc;
1441     struct device *dev;
1442     size_t align;
1443     int ret, i;
1444     u64 size;
1445
1446     ntb_epc = ntb->epc[type];
1447     peer_ntb_epc = ntb->epc[!type];
1448
1449     dev = &ntb->epf->dev;
1450     epc_features = ntb_epc->epc_features;
1451     align = epc_features->align;
1452     func_no = ntb_epc->func_no;
1453     vfunc_no = ntb_epc->vfunc_no;
1454     epc = ntb_epc->epc;
1455     num_mws = ntb->num_mws;
1456     db_count = ntb->db_count;
1457
1458     for (bar = BAR_DB_MW1, i = 0; i < num_mws; bar++, i++) {
1459         if (bar == BAR_DB_MW1) {
1460             align = align ? align : 4;
1461             size = db_count * align;
1462             size = ALIGN(size, ntb->mws_size[i]);
1463             ctrl = ntb_epc->reg;
1464             ctrl->mw1_offset = size;
1465             size += ntb->mws_size[i];
1466         } else {
1467             size = ntb->mws_size[i];
1468         }
1469
1470         ret = epf_ntb_alloc_peer_mem(dev, ntb_epc, bar,
1471                          peer_ntb_epc, size);
1472         if (ret) {
1473             dev_err(dev, "%s intf: DoorBell mem alloc failed\n",
1474                 pci_epc_interface_string(type));
1475             goto err_alloc_peer_mem;
1476         }
1477
1478         barno = ntb_epc->epf_ntb_bar[bar];
1479         epf_bar = &ntb_epc->epf_bar[barno];
1480
1481         ret = pci_epc_set_bar(epc, func_no, vfunc_no, epf_bar);
1482         if (ret) {
1483             dev_err(dev, "%s intf: DoorBell BAR set failed\n",
1484                 pci_epc_interface_string(type));
1485             goto err_alloc_peer_mem;
1486         }
1487     }
1488
1489     return 0;
1490
1491 err_alloc_peer_mem:
1492     epf_ntb_db_mw_bar_cleanup(ntb, type);
1493
1494     return ret;
1495 }
1496
1497 /**
1498  * epf_ntb_epc_destroy_interface() - Cleanup NTB EPC interface
1499  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1500  * @type: PRIMARY interface or SECONDARY interface
1501  *
1502  * Unbind NTB function device from EPC and relinquish reference to pci_epc
1503  * for each of the interface.
1504  */
1505 static void epf_ntb_epc_destroy_interface(struct epf_ntb *ntb,
1506                       enum pci_epc_interface_type type)
1507 {
1508     struct epf_ntb_epc *ntb_epc;
1509     struct pci_epc *epc;
1510     struct pci_epf *epf;
1511
1512     if (type < 0)
1513         return;
1514
1515     epf = ntb->epf;
1516     ntb_epc = ntb->epc[type];
1517     if (!ntb_epc)
1518         return;
1519     epc = ntb_epc->epc;
1520     pci_epc_remove_epf(epc, epf, type);
1521     pci_epc_put(epc);
1522 }
1523
1524 /**
1525  * epf_ntb_epc_destroy() - Cleanup NTB EPC interface
1526  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1527  *
1528  * Wrapper for epf_ntb_epc_destroy_interface() to cleanup all the NTB interfaces
1529  */
1530 static void epf_ntb_epc_destroy(struct epf_ntb *ntb)
1531 {
1532     enum pci_epc_interface_type type;
1533
1534     for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++)
1535         epf_ntb_epc_destroy_interface(ntb, type);
1536 }
1537
1538 /**
1539  * epf_ntb_epc_create_interface() - Create and initialize NTB EPC interface
1540  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1541  * @epc: struct pci_epc to which a particular NTB interface should be associated
1542  * @type: PRIMARY interface or SECONDARY interface
1543  *
1544  * Allocate memory for NTB EPC interface and initialize it.
1545  */
1546 static int epf_ntb_epc_create_interface(struct epf_ntb *ntb,
1547                     struct pci_epc *epc,
1548                     enum pci_epc_interface_type type)
1549 {
1550     const struct pci_epc_features *epc_features;
1551     struct pci_epf_bar *epf_bar;
1552     struct epf_ntb_epc *ntb_epc;
1553     u8 func_no, vfunc_no;
1554     struct pci_epf *epf;
1555     struct device *dev;
1556
1557     dev = &ntb->epf->dev;
1558
1559     ntb_epc = devm_kzalloc(dev, sizeof(*ntb_epc), GFP_KERNEL);
1560     if (!ntb_epc)
1561         return -ENOMEM;
1562
1563     epf = ntb->epf;
1564     vfunc_no = epf->vfunc_no;
1565     if (type == PRIMARY_INTERFACE) {
1566         func_no = epf->func_no;
1567         epf_bar = epf->bar;
1568     } else {
1569         func_no = epf->sec_epc_func_no;
1570         epf_bar = epf->sec_epc_bar;
1571     }
1572
1573     ntb_epc->linkup = false;
1574     ntb_epc->epc = epc;
1575     ntb_epc->func_no = func_no;
1576     ntb_epc->vfunc_no = vfunc_no;
1577     ntb_epc->type = type;
1578     ntb_epc->epf_bar = epf_bar;
1579     ntb_epc->epf_ntb = ntb;
1580
1581     epc_features = pci_epc_get_features(epc, func_no, vfunc_no);
1582     if (!epc_features)
1583         return -EINVAL;
1584     ntb_epc->epc_features = epc_features;
1585
1586     ntb->epc[type] = ntb_epc;
1587
1588     return 0;
1589 }
1590
1591 /**
1592  * epf_ntb_epc_create() - Create and initialize NTB EPC interface
1593  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1594  *
1595  * Get a reference to EPC device and bind NTB function device to that EPC
1596  * for each of the interface. It is also a wrapper to
1597  * epf_ntb_epc_create_interface() to allocate memory for NTB EPC interface
1598  * and initialize it
1599  */
1600 static int epf_ntb_epc_create(struct epf_ntb *ntb)
1601 {
1602     struct pci_epf *epf;
1603     struct device *dev;
1604     int ret;
1605
1606     epf = ntb->epf;
1607     dev = &epf->dev;
1608
1609     ret = epf_ntb_epc_create_interface(ntb, epf->epc, PRIMARY_INTERFACE);
1610     if (ret) {
1611         dev_err(dev, "PRIMARY intf: Fail to create NTB EPC\n");
1612         return ret;
1613     }
1614
1615     ret = epf_ntb_epc_create_interface(ntb, epf->sec_epc,
1616                        SECONDARY_INTERFACE);
1617     if (ret) {
1618         dev_err(dev, "SECONDARY intf: Fail to create NTB EPC\n");
1619         goto err_epc_create;
1620     }
1621
1622     return 0;
1623
1624 err_epc_create:
1625     epf_ntb_epc_destroy_interface(ntb, PRIMARY_INTERFACE);
1626
1627     return ret;
1628 }
1629
1630 /**
1631  * epf_ntb_init_epc_bar_interface() - Identify BARs to be used for each of
1632  *   the NTB constructs (scratchpad region, doorbell, memorywindow)
1633  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1634  * @type: PRIMARY interface or SECONDARY interface
1635  *
1636  * Identify the free BARs to be used for each of BAR_CONFIG, BAR_PEER_SPAD,
1637  * BAR_DB_MW1, BAR_MW2, BAR_MW3 and BAR_MW4.
1638  */
1639 static int epf_ntb_init_epc_bar_interface(struct epf_ntb *ntb,
1640                       enum pci_epc_interface_type type)
1641 {
1642     const struct pci_epc_features *epc_features;
1643     struct epf_ntb_epc *ntb_epc;
1644     enum pci_barno barno;
1645     enum epf_ntb_bar bar;
1646     struct device *dev;
1647     u32 num_mws;
1648     int i;
1649
1650     barno = BAR_0;
1651     ntb_epc = ntb->epc[type];
1652     num_mws = ntb->num_mws;
1653     dev = &ntb->epf->dev;
1654     epc_features = ntb_epc->epc_features;
1655
1656     /* These are required BARs which are mandatory for NTB functionality */
1657     for (bar = BAR_CONFIG; bar <= BAR_DB_MW1; bar++, barno++) {
1658         barno = pci_epc_get_next_free_bar(epc_features, barno);
1659         if (barno < 0) {
1660             dev_err(dev, "%s intf: Fail to get NTB function BAR\n",
1661                 pci_epc_interface_string(type));
1662             return barno;
1663         }
1664         ntb_epc->epf_ntb_bar[bar] = barno;
1665     }
1666
1667     /* These are optional BARs which don't impact NTB functionality */
1668     for (bar = BAR_MW2, i = 1; i < num_mws; bar++, barno++, i++) {
1669         barno = pci_epc_get_next_free_bar(epc_features, barno);
1670         if (barno < 0) {
1671             ntb->num_mws = i;
1672             dev_dbg(dev, "BAR not available for > MW%d\n", i + 1);
1673         }
1674         ntb_epc->epf_ntb_bar[bar] = barno;
1675     }
1676
1677     return 0;
1678 }
1679
1680 /**
1681  * epf_ntb_init_epc_bar() - Identify BARs to be used for each of the NTB
1682  * constructs (scratchpad region, doorbell, memorywindow)
1683  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1684  *
1685  * Wrapper to epf_ntb_init_epc_bar_interface() to identify the free BARs
1686  * to be used for each of BAR_CONFIG, BAR_PEER_SPAD, BAR_DB_MW1, BAR_MW2,
1687  * BAR_MW3 and BAR_MW4 for all the interfaces.
1688  */
1689 static int epf_ntb_init_epc_bar(struct epf_ntb *ntb)
1690 {
1691     enum pci_epc_interface_type type;
1692     struct device *dev;
1693     int ret;
1694
1695     dev = &ntb->epf->dev;
1696     for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {
1697         ret = epf_ntb_init_epc_bar_interface(ntb, type);
1698         if (ret) {
1699             dev_err(dev, "Fail to init EPC bar for %s interface\n",
1700                 pci_epc_interface_string(type));
1701             return ret;
1702         }
1703     }
1704
1705     return 0;
1706 }
1707
1708 /**
1709  * epf_ntb_epc_init_interface() - Initialize NTB interface
1710  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1711  * @type: PRIMARY interface or SECONDARY interface
1712  *
1713  * Wrapper to initialize a particular EPC interface and start the workqueue
1714  * to check for commands from host. This function will write to the
1715  * EP controller HW for configuring it.
1716  */
1717 static int epf_ntb_epc_init_interface(struct epf_ntb *ntb,
1718                       enum pci_epc_interface_type type)
1719 {
1720     struct epf_ntb_epc *ntb_epc;
1721     u8 func_no, vfunc_no;
1722     struct pci_epc *epc;
1723     struct pci_epf *epf;
1724     struct device *dev;
1725     int ret;
1726
1727     ntb_epc = ntb->epc[type];
1728     epf = ntb->epf;
1729     dev = &epf->dev;
1730     epc = ntb_epc->epc;
1731     func_no = ntb_epc->func_no;
1732     vfunc_no = ntb_epc->vfunc_no;
1733
1734     ret = epf_ntb_config_sspad_bar_set(ntb->epc[type]);
1735     if (ret) {
1736         dev_err(dev, "%s intf: Config/self SPAD BAR init failed\n",
1737             pci_epc_interface_string(type));
1738         return ret;
1739     }
1740
1741     ret = epf_ntb_peer_spad_bar_set(ntb, type);
1742     if (ret) {
1743         dev_err(dev, "%s intf: Peer SPAD BAR init failed\n",
1744             pci_epc_interface_string(type));
1745         goto err_peer_spad_bar_init;
1746     }
1747
1748     ret = epf_ntb_configure_interrupt(ntb, type);
1749     if (ret) {
1750         dev_err(dev, "%s intf: Interrupt configuration failed\n",
1751             pci_epc_interface_string(type));
1752         goto err_peer_spad_bar_init;
1753     }
1754
1755     ret = epf_ntb_db_mw_bar_init(ntb, type);
1756     if (ret) {
1757         dev_err(dev, "%s intf: DB/MW BAR init failed\n",
1758             pci_epc_interface_string(type));
1759         goto err_db_mw_bar_init;
1760     }
1761
1762     if (vfunc_no <= 1) {
1763         ret = pci_epc_write_header(epc, func_no, vfunc_no, epf->header);
1764         if (ret) {
1765             dev_err(dev, "%s intf: Configuration header write failed\n",
1766                 pci_epc_interface_string(type));
1767             goto err_write_header;
1768         }
1769     }
1770
1771     INIT_DELAYED_WORK(&ntb->epc[type]->cmd_handler, epf_ntb_cmd_handler);
1772     queue_work(kpcintb_workqueue, &ntb->epc[type]->cmd_handler.work);
1773
1774     return 0;
1775
1776 err_write_header:
1777     epf_ntb_db_mw_bar_cleanup(ntb, type);
1778
1779 err_db_mw_bar_init:
1780     epf_ntb_peer_spad_bar_clear(ntb->epc[type]);
1781
1782 err_peer_spad_bar_init:
1783     epf_ntb_config_sspad_bar_clear(ntb->epc[type]);
1784
1785     return ret;
1786 }
1787
1788 /**
1789  * epf_ntb_epc_cleanup_interface() - Cleanup NTB interface
1790  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1791  * @type: PRIMARY interface or SECONDARY interface
1792  *
1793  * Wrapper to cleanup a particular NTB interface.
1794  */
1795 static void epf_ntb_epc_cleanup_interface(struct epf_ntb *ntb,
1796                       enum pci_epc_interface_type type)
1797 {
1798     struct epf_ntb_epc *ntb_epc;
1799
1800     if (type < 0)
1801         return;
1802
1803     ntb_epc = ntb->epc[type];
1804     cancel_delayed_work(&ntb_epc->cmd_handler);
1805     epf_ntb_db_mw_bar_cleanup(ntb, type);
1806     epf_ntb_peer_spad_bar_clear(ntb_epc);
1807     epf_ntb_config_sspad_bar_clear(ntb_epc);
1808 }
1809
1810 /**
1811  * epf_ntb_epc_cleanup() - Cleanup all NTB interfaces
1812  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1813  *
1814  * Wrapper to cleanup all NTB interfaces.
1815  */
1816 static void epf_ntb_epc_cleanup(struct epf_ntb *ntb)
1817 {
1818     enum pci_epc_interface_type type;
1819
1820     for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++)
1821         epf_ntb_epc_cleanup_interface(ntb, type);
1822 }
1823
1824 /**
1825  * epf_ntb_epc_init() - Initialize all NTB interfaces
1826  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1827  *
1828  * Wrapper to initialize all NTB interface and start the workqueue
1829  * to check for commands from host.
1830  */
1831 static int epf_ntb_epc_init(struct epf_ntb *ntb)
1832 {
1833     enum pci_epc_interface_type type;
1834     struct device *dev;
1835     int ret;
1836
1837     dev = &ntb->epf->dev;
1838
1839     for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {
1840         ret = epf_ntb_epc_init_interface(ntb, type);
1841         if (ret) {
1842             dev_err(dev, "%s intf: Failed to initialize\n",
1843                 pci_epc_interface_string(type));
1844             goto err_init_type;
1845         }
1846     }
1847
1848     return 0;
1849
1850 err_init_type:
1851     epf_ntb_epc_cleanup_interface(ntb, type - 1);
1852
1853     return ret;
1854 }
1855
1856 /**
1857  * epf_ntb_bind() - Initialize endpoint controller to provide NTB functionality
1858  * @epf: NTB endpoint function device
1859  *
1860  * Initialize both the endpoint controllers associated with NTB function device.
1861  * Invoked when a primary interface or secondary interface is bound to EPC
1862  * device. This function will succeed only when EPC is bound to both the
1863  * interfaces.
1864  */
1865 static int epf_ntb_bind(struct pci_epf *epf)
1866 {
1867     struct epf_ntb *ntb = epf_get_drvdata(epf);
1868     struct device *dev = &epf->dev;
1869     int ret;
1870
1871     if (!epf->epc) {
1872         dev_dbg(dev, "PRIMARY EPC interface not yet bound\n");
1873         return 0;
1874     }
1875
1876     if (!epf->sec_epc) {
1877         dev_dbg(dev, "SECONDARY EPC interface not yet bound\n");
1878         return 0;
1879     }
1880
1881     ret = epf_ntb_epc_create(ntb);
1882     if (ret) {
1883         dev_err(dev, "Failed to create NTB EPC\n");
1884         return ret;
1885     }
1886
1887     ret = epf_ntb_init_epc_bar(ntb);
1888     if (ret) {
1889         dev_err(dev, "Failed to create NTB EPC\n");
1890         goto err_bar_init;
1891     }
1892
1893     ret = epf_ntb_config_spad_bar_alloc_interface(ntb);
1894     if (ret) {
1895         dev_err(dev, "Failed to allocate BAR memory\n");
1896         goto err_bar_alloc;
1897     }
1898
1899     ret = epf_ntb_epc_init(ntb);
1900     if (ret) {
1901         dev_err(dev, "Failed to initialize EPC\n");
1902         goto err_bar_alloc;
1903     }
1904
1905     epf_set_drvdata(epf, ntb);
1906
1907     return 0;
1908
1909 err_bar_alloc:
1910     epf_ntb_config_spad_bar_free(ntb);
1911
1912 err_bar_init:
1913     epf_ntb_epc_destroy(ntb);
1914
1915     return ret;
1916 }
1917
1918 /**
1919  * epf_ntb_unbind() - Cleanup the initialization from epf_ntb_bind()
1920  * @epf: NTB endpoint function device
1921  *
1922  * Cleanup the initialization from epf_ntb_bind()
1923  */
1924 static void epf_ntb_unbind(struct pci_epf *epf)
1925 {
1926     struct epf_ntb *ntb = epf_get_drvdata(epf);
1927
1928     epf_ntb_epc_cleanup(ntb);
1929     epf_ntb_config_spad_bar_free(ntb);
1930     epf_ntb_epc_destroy(ntb);
1931 }
1932
1933 #define EPF_NTB_R(_name)                        \
1934 static ssize_t epf_ntb_##_name##_show(struct config_item *item,     \
1935                       char *page)           \
1936 {                                   \
1937     struct config_group *group = to_config_group(item);     \
1938     struct epf_ntb *ntb = to_epf_ntb(group);            \
1939                                     \
1940     return sysfs_emit(page, "%d\n", ntb->_name);            \
1941 }
1942
1943 #define EPF_NTB_W(_name)                        \
1944 static ssize_t epf_ntb_##_name##_store(struct config_item *item,    \
1945                        const char *page, size_t len)    \
1946 {                                   \
1947     struct config_group *group = to_config_group(item);     \
1948     struct epf_ntb *ntb = to_epf_ntb(group);            \
1949     u32 val;                            \
1950                                     \
1951     if (kstrtou32(page, 0, &val) < 0)               \
1952         return -EINVAL;                     \
1953                                     \
1954     ntb->_name = val;                       \
1955                                     \
1956     return len;                         \
1957 }
1958
1959 #define EPF_NTB_MW_R(_name)                     \
1960 static ssize_t epf_ntb_##_name##_show(struct config_item *item,     \
1961                       char *page)           \
1962 {                                   \
1963     struct config_group *group = to_config_group(item);     \
1964     struct epf_ntb *ntb = to_epf_ntb(group);            \
1965     int win_no;                         \
1966                                     \
1967     sscanf(#_name, "mw%d", &win_no);                \
1968                                     \
1969     return sysfs_emit(page, "%lld\n", ntb->mws_size[win_no - 1]);   \
1970 }
1971
1972 #define EPF_NTB_MW_W(_name)                     \
1973 static ssize_t epf_ntb_##_name##_store(struct config_item *item,    \
1974                        const char *page, size_t len)    \
1975 {                                   \
1976     struct config_group *group = to_config_group(item);     \
1977     struct epf_ntb *ntb = to_epf_ntb(group);            \
1978     struct device *dev = &ntb->epf->dev;                \
1979     int win_no;                         \
1980     u64 val;                            \
1981                                     \
1982     if (kstrtou64(page, 0, &val) < 0)               \
1983         return -EINVAL;                     \
1984                                     \
1985     if (sscanf(#_name, "mw%d", &win_no) != 1)           \
1986         return -EINVAL;                     \
1987                                     \
1988     if (ntb->num_mws < win_no) {                    \
1989         dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \
1990         return -EINVAL;                     \
1991     }                               \
1992                                     \
1993     ntb->mws_size[win_no - 1] = val;                \
1994                                     \
1995     return len;                         \
1996 }
1997
1998 static ssize_t epf_ntb_num_mws_store(struct config_item *item,
1999                      const char *page, size_t len)
2000 {
2001     struct config_group *group = to_config_group(item);
2002     struct epf_ntb *ntb = to_epf_ntb(group);
2003     u32 val;
2004
2005     if (kstrtou32(page, 0, &val) < 0)
2006         return -EINVAL;
2007
2008     if (val > MAX_MW)
2009         return -EINVAL;
2010
2011     ntb->num_mws = val;
2012
2013     return len;
2014 }
2015
2016 EPF_NTB_R(spad_count)
2017 EPF_NTB_W(spad_count)
2018 EPF_NTB_R(db_count)
2019 EPF_NTB_W(db_count)
2020 EPF_NTB_R(num_mws)
2021 EPF_NTB_MW_R(mw1)
2022 EPF_NTB_MW_W(mw1)
2023 EPF_NTB_MW_R(mw2)
2024 EPF_NTB_MW_W(mw2)
2025 EPF_NTB_MW_R(mw3)
2026 EPF_NTB_MW_W(mw3)
2027 EPF_NTB_MW_R(mw4)
2028 EPF_NTB_MW_W(mw4)
2029
2030 CONFIGFS_ATTR(epf_ntb_, spad_count);
2031 CONFIGFS_ATTR(epf_ntb_, db_count);
2032 CONFIGFS_ATTR(epf_ntb_, num_mws);
2033 CONFIGFS_ATTR(epf_ntb_, mw1);
2034 CONFIGFS_ATTR(epf_ntb_, mw2);
2035 CONFIGFS_ATTR(epf_ntb_, mw3);
2036 CONFIGFS_ATTR(epf_ntb_, mw4);
2037
2038 static struct configfs_attribute *epf_ntb_attrs[] = {
2039     &epf_ntb_attr_spad_count,
2040     &epf_ntb_attr_db_count,
2041     &epf_ntb_attr_num_mws,
2042     &epf_ntb_attr_mw1,
2043     &epf_ntb_attr_mw2,
2044     &epf_ntb_attr_mw3,
2045     &epf_ntb_attr_mw4,
2046     NULL,
2047 };
2048
2049 static const struct config_item_type ntb_group_type = {
2050     .ct_attrs   = epf_ntb_attrs,
2051     .ct_owner   = THIS_MODULE,
2052 };
2053
2054 /**
2055  * epf_ntb_add_cfs() - Add configfs directory specific to NTB
2056  * @epf: NTB endpoint function device
2057  * @group: A pointer to the config_group structure referencing a group of
2058  *     config_items of a specific type that belong to a specific sub-system.
2059  *
2060  * Add configfs directory specific to NTB. This directory will hold
2061  * NTB specific properties like db_count, spad_count, num_mws etc.,
2062  */
2063 static struct config_group *epf_ntb_add_cfs(struct pci_epf *epf,
2064                         struct config_group *group)
2065 {
2066     struct epf_ntb *ntb = epf_get_drvdata(epf);
2067     struct config_group *ntb_group = &ntb->group;
2068     struct device *dev = &epf->dev;
2069
2070     config_group_init_type_name(ntb_group, dev_name(dev), &ntb_group_type);
2071
2072     return ntb_group;
2073 }
2074
2075 /**
2076  * epf_ntb_probe() - Probe NTB function driver
2077  * @epf: NTB endpoint function device
2078  *
2079  * Probe NTB function driver when endpoint function bus detects a NTB
2080  * endpoint function.
2081  */
2082 static int epf_ntb_probe(struct pci_epf *epf)
2083 {
2084     struct epf_ntb *ntb;
2085     struct device *dev;
2086
2087     dev = &epf->dev;
2088
2089     ntb = devm_kzalloc(dev, sizeof(*ntb), GFP_KERNEL);
2090     if (!ntb)
2091         return -ENOMEM;
2092
2093     epf->header = &epf_ntb_header;
2094     ntb->epf = epf;
2095     epf_set_drvdata(epf, ntb);
2096
2097     return 0;
2098 }
2099
2100 static struct pci_epf_ops epf_ntb_ops = {
2101     .bind   = epf_ntb_bind,
2102     .unbind = epf_ntb_unbind,
2103     .add_cfs = epf_ntb_add_cfs,
2104 };
2105
2106 static const struct pci_epf_device_id epf_ntb_ids[] = {
2107     {
2108         .name = "pci_epf_ntb",
2109     },
2110     {},
2111 };
2112
2113 static struct pci_epf_driver epf_ntb_driver = {
2114     .driver.name    = "pci_epf_ntb",
2115     .probe      = epf_ntb_probe,
2116     .id_table   = epf_ntb_ids,
2117     .ops        = &epf_ntb_ops,
2118     .owner      = THIS_MODULE,
2119 };
2120
2121 static int __init epf_ntb_init(void)
2122 {
2123     int ret;
2124
2125     kpcintb_workqueue = alloc_workqueue("kpcintb", WQ_MEM_RECLAIM |
2126                         WQ_HIGHPRI, 0);
2127     ret = pci_epf_register_driver(&epf_ntb_driver);
2128     if (ret) {
2129         destroy_workqueue(kpcintb_workqueue);
2130         pr_err("Failed to register pci epf ntb driver --> %d\n", ret);
2131         return ret;
2132     }
2133
2134     return 0;
2135 }
2136 module_init(epf_ntb_init);
2137
2138 static void __exit epf_ntb_exit(void)
2139 {
2140     pci_epf_unregister_driver(&epf_ntb_driver);
2141     destroy_workqueue(kpcintb_workqueue);
2142 }
2143 module_exit(epf_ntb_exit);
2144
2145 MODULE_DESCRIPTION("PCI EPF NTB DRIVER");
2146 MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>");
2147 MODULE_LICENSE("GPL v2");