Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Microsemi Switchtec(tm) PCIe Management Driver
0004  * Copyright (c) 2017, Microsemi Corporation
0005  */
0006 
0007 #include <linux/interrupt.h>
0008 #include <linux/io-64-nonatomic-lo-hi.h>
0009 #include <linux/delay.h>
0010 #include <linux/kthread.h>
0011 #include <linux/module.h>
0012 #include <linux/ntb.h>
0013 #include <linux/pci.h>
0014 #include <linux/switchtec.h>
0015 
0016 MODULE_DESCRIPTION("Microsemi Switchtec(tm) NTB Driver");
0017 MODULE_VERSION("0.1");
0018 MODULE_LICENSE("GPL");
0019 MODULE_AUTHOR("Microsemi Corporation");
0020 
0021 static ulong max_mw_size = SZ_2M;
0022 module_param(max_mw_size, ulong, 0644);
0023 MODULE_PARM_DESC(max_mw_size,
0024     "Max memory window size reported to the upper layer");
0025 
0026 static bool use_lut_mws;
0027 module_param(use_lut_mws, bool, 0644);
0028 MODULE_PARM_DESC(use_lut_mws,
0029          "Enable the use of the LUT based memory windows");
0030 
0031 #define SWITCHTEC_NTB_MAGIC 0x45CC0001
0032 #define MAX_MWS     128
0033 
0034 struct shared_mw {
0035     u32 magic;
0036     u32 link_sta;
0037     u32 partition_id;
0038     u64 mw_sizes[MAX_MWS];
0039     u32 spad[128];
0040 };
0041 
0042 #define MAX_DIRECT_MW ARRAY_SIZE(((struct ntb_ctrl_regs *)(0))->bar_entry)
0043 #define LUT_SIZE SZ_64K
0044 
0045 struct switchtec_ntb {
0046     struct ntb_dev ntb;
0047     struct switchtec_dev *stdev;
0048 
0049     int self_partition;
0050     int peer_partition;
0051 
0052     int doorbell_irq;
0053     int message_irq;
0054 
0055     struct ntb_info_regs __iomem *mmio_ntb;
0056     struct ntb_ctrl_regs __iomem *mmio_ctrl;
0057     struct ntb_dbmsg_regs __iomem *mmio_dbmsg;
0058     struct ntb_ctrl_regs __iomem *mmio_self_ctrl;
0059     struct ntb_ctrl_regs __iomem *mmio_peer_ctrl;
0060     struct ntb_dbmsg_regs __iomem *mmio_self_dbmsg;
0061     struct ntb_dbmsg_regs __iomem *mmio_peer_dbmsg;
0062 
0063     void __iomem *mmio_xlink_win;
0064 
0065     struct shared_mw *self_shared;
0066     struct shared_mw __iomem *peer_shared;
0067     dma_addr_t self_shared_dma;
0068 
0069     u64 db_mask;
0070     u64 db_valid_mask;
0071     int db_shift;
0072     int db_peer_shift;
0073 
0074     /* synchronize rmw access of db_mask and hw reg */
0075     spinlock_t db_mask_lock;
0076 
0077     int nr_direct_mw;
0078     int nr_lut_mw;
0079     int nr_rsvd_luts;
0080     int direct_mw_to_bar[MAX_DIRECT_MW];
0081 
0082     int peer_nr_direct_mw;
0083     int peer_nr_lut_mw;
0084     int peer_direct_mw_to_bar[MAX_DIRECT_MW];
0085 
0086     bool link_is_up;
0087     enum ntb_speed link_speed;
0088     enum ntb_width link_width;
0089     struct work_struct check_link_status_work;
0090     bool link_force_down;
0091 };
0092 
0093 static struct switchtec_ntb *ntb_sndev(struct ntb_dev *ntb)
0094 {
0095     return container_of(ntb, struct switchtec_ntb, ntb);
0096 }
0097 
0098 static int switchtec_ntb_part_op(struct switchtec_ntb *sndev,
0099                  struct ntb_ctrl_regs __iomem *ctl,
0100                  u32 op, int wait_status)
0101 {
0102     static const char * const op_text[] = {
0103         [NTB_CTRL_PART_OP_LOCK] = "lock",
0104         [NTB_CTRL_PART_OP_CFG] = "configure",
0105         [NTB_CTRL_PART_OP_RESET] = "reset",
0106     };
0107 
0108     int i;
0109     u32 ps;
0110     int status;
0111 
0112     switch (op) {
0113     case NTB_CTRL_PART_OP_LOCK:
0114         status = NTB_CTRL_PART_STATUS_LOCKING;
0115         break;
0116     case NTB_CTRL_PART_OP_CFG:
0117         status = NTB_CTRL_PART_STATUS_CONFIGURING;
0118         break;
0119     case NTB_CTRL_PART_OP_RESET:
0120         status = NTB_CTRL_PART_STATUS_RESETTING;
0121         break;
0122     default:
0123         return -EINVAL;
0124     }
0125 
0126     iowrite32(op, &ctl->partition_op);
0127 
0128     for (i = 0; i < 1000; i++) {
0129         if (msleep_interruptible(50) != 0) {
0130             iowrite32(NTB_CTRL_PART_OP_RESET, &ctl->partition_op);
0131             return -EINTR;
0132         }
0133 
0134         ps = ioread32(&ctl->partition_status) & 0xFFFF;
0135 
0136         if (ps != status)
0137             break;
0138     }
0139 
0140     if (ps == wait_status)
0141         return 0;
0142 
0143     if (ps == status) {
0144         dev_err(&sndev->stdev->dev,
0145             "Timed out while performing %s (%d). (%08x)\n",
0146             op_text[op], op,
0147             ioread32(&ctl->partition_status));
0148 
0149         return -ETIMEDOUT;
0150     }
0151 
0152     return -EIO;
0153 }
0154 
0155 static int switchtec_ntb_send_msg(struct switchtec_ntb *sndev, int idx,
0156                   u32 val)
0157 {
0158     if (idx < 0 || idx >= ARRAY_SIZE(sndev->mmio_peer_dbmsg->omsg))
0159         return -EINVAL;
0160 
0161     iowrite32(val, &sndev->mmio_peer_dbmsg->omsg[idx].msg);
0162 
0163     return 0;
0164 }
0165 
0166 static int switchtec_ntb_mw_count(struct ntb_dev *ntb, int pidx)
0167 {
0168     struct switchtec_ntb *sndev = ntb_sndev(ntb);
0169     int nr_direct_mw = sndev->peer_nr_direct_mw;
0170     int nr_lut_mw = sndev->peer_nr_lut_mw - sndev->nr_rsvd_luts;
0171 
0172     if (pidx != NTB_DEF_PEER_IDX)
0173         return -EINVAL;
0174 
0175     if (!use_lut_mws)
0176         nr_lut_mw = 0;
0177 
0178     return nr_direct_mw + nr_lut_mw;
0179 }
0180 
0181 static int lut_index(struct switchtec_ntb *sndev, int mw_idx)
0182 {
0183     return mw_idx - sndev->nr_direct_mw + sndev->nr_rsvd_luts;
0184 }
0185 
0186 static int peer_lut_index(struct switchtec_ntb *sndev, int mw_idx)
0187 {
0188     return mw_idx - sndev->peer_nr_direct_mw + sndev->nr_rsvd_luts;
0189 }
0190 
0191 static int switchtec_ntb_mw_get_align(struct ntb_dev *ntb, int pidx,
0192                       int widx, resource_size_t *addr_align,
0193                       resource_size_t *size_align,
0194                       resource_size_t *size_max)
0195 {
0196     struct switchtec_ntb *sndev = ntb_sndev(ntb);
0197     int lut;
0198     resource_size_t size;
0199 
0200     if (pidx != NTB_DEF_PEER_IDX)
0201         return -EINVAL;
0202 
0203     lut = widx >= sndev->peer_nr_direct_mw;
0204     size = ioread64(&sndev->peer_shared->mw_sizes[widx]);
0205 
0206     if (size == 0)
0207         return -EINVAL;
0208 
0209     if (addr_align)
0210         *addr_align = lut ? size : SZ_4K;
0211 
0212     if (size_align)
0213         *size_align = lut ? size : SZ_4K;
0214 
0215     if (size_max)
0216         *size_max = size;
0217 
0218     return 0;
0219 }
0220 
0221 static void switchtec_ntb_mw_clr_direct(struct switchtec_ntb *sndev, int idx)
0222 {
0223     struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
0224     int bar = sndev->peer_direct_mw_to_bar[idx];
0225     u32 ctl_val;
0226 
0227     ctl_val = ioread32(&ctl->bar_entry[bar].ctl);
0228     ctl_val &= ~NTB_CTRL_BAR_DIR_WIN_EN;
0229     iowrite32(ctl_val, &ctl->bar_entry[bar].ctl);
0230     iowrite32(0, &ctl->bar_entry[bar].win_size);
0231     iowrite32(0, &ctl->bar_ext_entry[bar].win_size);
0232     iowrite64(sndev->self_partition, &ctl->bar_entry[bar].xlate_addr);
0233 }
0234 
0235 static void switchtec_ntb_mw_clr_lut(struct switchtec_ntb *sndev, int idx)
0236 {
0237     struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
0238 
0239     iowrite64(0, &ctl->lut_entry[peer_lut_index(sndev, idx)]);
0240 }
0241 
0242 static void switchtec_ntb_mw_set_direct(struct switchtec_ntb *sndev, int idx,
0243                     dma_addr_t addr, resource_size_t size)
0244 {
0245     int xlate_pos = ilog2(size);
0246     int bar = sndev->peer_direct_mw_to_bar[idx];
0247     struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
0248     u32 ctl_val;
0249 
0250     ctl_val = ioread32(&ctl->bar_entry[bar].ctl);
0251     ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN;
0252 
0253     iowrite32(ctl_val, &ctl->bar_entry[bar].ctl);
0254     iowrite32(xlate_pos | (lower_32_bits(size) & 0xFFFFF000),
0255           &ctl->bar_entry[bar].win_size);
0256     iowrite32(upper_32_bits(size), &ctl->bar_ext_entry[bar].win_size);
0257     iowrite64(sndev->self_partition | addr,
0258           &ctl->bar_entry[bar].xlate_addr);
0259 }
0260 
0261 static void switchtec_ntb_mw_set_lut(struct switchtec_ntb *sndev, int idx,
0262                      dma_addr_t addr, resource_size_t size)
0263 {
0264     struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
0265 
0266     iowrite64((NTB_CTRL_LUT_EN | (sndev->self_partition << 1) | addr),
0267           &ctl->lut_entry[peer_lut_index(sndev, idx)]);
0268 }
0269 
0270 static int switchtec_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx,
0271                       dma_addr_t addr, resource_size_t size)
0272 {
0273     struct switchtec_ntb *sndev = ntb_sndev(ntb);
0274     struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
0275     int xlate_pos = ilog2(size);
0276     int nr_direct_mw = sndev->peer_nr_direct_mw;
0277     int rc;
0278 
0279     if (pidx != NTB_DEF_PEER_IDX)
0280         return -EINVAL;
0281 
0282     dev_dbg(&sndev->stdev->dev, "MW %d: part %d addr %pad size %pap\n",
0283         widx, pidx, &addr, &size);
0284 
0285     if (widx >= switchtec_ntb_mw_count(ntb, pidx))
0286         return -EINVAL;
0287 
0288     if (size != 0 && xlate_pos < 12)
0289         return -EINVAL;
0290 
0291     if (!IS_ALIGNED(addr, BIT_ULL(xlate_pos))) {
0292         /*
0293          * In certain circumstances we can get a buffer that is
0294          * not aligned to its size. (Most of the time
0295          * dma_alloc_coherent ensures this). This can happen when
0296          * using large buffers allocated by the CMA
0297          * (see CMA_CONFIG_ALIGNMENT)
0298          */
0299         dev_err(&sndev->stdev->dev,
0300             "ERROR: Memory window address is not aligned to its size!\n");
0301         return -EINVAL;
0302     }
0303 
0304     rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK,
0305                    NTB_CTRL_PART_STATUS_LOCKED);
0306     if (rc)
0307         return rc;
0308 
0309     if (size == 0) {
0310         if (widx < nr_direct_mw)
0311             switchtec_ntb_mw_clr_direct(sndev, widx);
0312         else
0313             switchtec_ntb_mw_clr_lut(sndev, widx);
0314     } else {
0315         if (widx < nr_direct_mw)
0316             switchtec_ntb_mw_set_direct(sndev, widx, addr, size);
0317         else
0318             switchtec_ntb_mw_set_lut(sndev, widx, addr, size);
0319     }
0320 
0321     rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
0322                    NTB_CTRL_PART_STATUS_NORMAL);
0323 
0324     if (rc == -EIO) {
0325         dev_err(&sndev->stdev->dev,
0326             "Hardware reported an error configuring mw %d: %08x\n",
0327             widx, ioread32(&ctl->bar_error));
0328 
0329         if (widx < nr_direct_mw)
0330             switchtec_ntb_mw_clr_direct(sndev, widx);
0331         else
0332             switchtec_ntb_mw_clr_lut(sndev, widx);
0333 
0334         switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
0335                       NTB_CTRL_PART_STATUS_NORMAL);
0336     }
0337 
0338     return rc;
0339 }
0340 
0341 static int switchtec_ntb_peer_mw_count(struct ntb_dev *ntb)
0342 {
0343     struct switchtec_ntb *sndev = ntb_sndev(ntb);
0344     int nr_lut_mw = sndev->nr_lut_mw - sndev->nr_rsvd_luts;
0345 
0346     return sndev->nr_direct_mw + (use_lut_mws ? nr_lut_mw : 0);
0347 }
0348 
0349 static int switchtec_ntb_direct_get_addr(struct switchtec_ntb *sndev,
0350                      int idx, phys_addr_t *base,
0351                      resource_size_t *size)
0352 {
0353     int bar = sndev->direct_mw_to_bar[idx];
0354     size_t offset = 0;
0355 
0356     if (bar < 0)
0357         return -EINVAL;
0358 
0359     if (idx == 0) {
0360         /*
0361          * This is the direct BAR shared with the LUTs
0362          * which means the actual window will be offset
0363          * by the size of all the LUT entries.
0364          */
0365 
0366         offset = LUT_SIZE * sndev->nr_lut_mw;
0367     }
0368 
0369     if (base)
0370         *base = pci_resource_start(sndev->ntb.pdev, bar) + offset;
0371 
0372     if (size) {
0373         *size = pci_resource_len(sndev->ntb.pdev, bar) - offset;
0374         if (offset && *size > offset)
0375             *size = offset;
0376 
0377         if (*size > max_mw_size)
0378             *size = max_mw_size;
0379     }
0380 
0381     return 0;
0382 }
0383 
0384 static int switchtec_ntb_lut_get_addr(struct switchtec_ntb *sndev,
0385                       int idx, phys_addr_t *base,
0386                       resource_size_t *size)
0387 {
0388     int bar = sndev->direct_mw_to_bar[0];
0389     int offset;
0390 
0391     offset = LUT_SIZE * lut_index(sndev, idx);
0392 
0393     if (base)
0394         *base = pci_resource_start(sndev->ntb.pdev, bar) + offset;
0395 
0396     if (size)
0397         *size = LUT_SIZE;
0398 
0399     return 0;
0400 }
0401 
0402 static int switchtec_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
0403                       phys_addr_t *base,
0404                       resource_size_t *size)
0405 {
0406     struct switchtec_ntb *sndev = ntb_sndev(ntb);
0407 
0408     if (idx < sndev->nr_direct_mw)
0409         return switchtec_ntb_direct_get_addr(sndev, idx, base, size);
0410     else if (idx < switchtec_ntb_peer_mw_count(ntb))
0411         return switchtec_ntb_lut_get_addr(sndev, idx, base, size);
0412     else
0413         return -EINVAL;
0414 }
0415 
0416 static void switchtec_ntb_part_link_speed(struct switchtec_ntb *sndev,
0417                       int partition,
0418                       enum ntb_speed *speed,
0419                       enum ntb_width *width)
0420 {
0421     struct switchtec_dev *stdev = sndev->stdev;
0422     struct part_cfg_regs __iomem *part_cfg =
0423         &stdev->mmio_part_cfg_all[partition];
0424 
0425     u32 pff = ioread32(&part_cfg->vep_pff_inst_id) & 0xFF;
0426     u32 linksta = ioread32(&stdev->mmio_pff_csr[pff].pci_cap_region[13]);
0427 
0428     if (speed)
0429         *speed = (linksta >> 16) & 0xF;
0430 
0431     if (width)
0432         *width = (linksta >> 20) & 0x3F;
0433 }
0434 
0435 static void switchtec_ntb_set_link_speed(struct switchtec_ntb *sndev)
0436 {
0437     enum ntb_speed self_speed, peer_speed;
0438     enum ntb_width self_width, peer_width;
0439 
0440     if (!sndev->link_is_up) {
0441         sndev->link_speed = NTB_SPEED_NONE;
0442         sndev->link_width = NTB_WIDTH_NONE;
0443         return;
0444     }
0445 
0446     switchtec_ntb_part_link_speed(sndev, sndev->self_partition,
0447                       &self_speed, &self_width);
0448     switchtec_ntb_part_link_speed(sndev, sndev->peer_partition,
0449                       &peer_speed, &peer_width);
0450 
0451     sndev->link_speed = min(self_speed, peer_speed);
0452     sndev->link_width = min(self_width, peer_width);
0453 }
0454 
0455 static int crosslink_is_enabled(struct switchtec_ntb *sndev)
0456 {
0457     struct ntb_info_regs __iomem *inf = sndev->mmio_ntb;
0458 
0459     return ioread8(&inf->ntp_info[sndev->peer_partition].xlink_enabled);
0460 }
0461 
0462 static void crosslink_init_dbmsgs(struct switchtec_ntb *sndev)
0463 {
0464     int i;
0465     u32 msg_map = 0;
0466 
0467     if (!crosslink_is_enabled(sndev))
0468         return;
0469 
0470     for (i = 0; i < ARRAY_SIZE(sndev->mmio_peer_dbmsg->imsg); i++) {
0471         int m = i | sndev->self_partition << 2;
0472 
0473         msg_map |= m << i * 8;
0474     }
0475 
0476     iowrite32(msg_map, &sndev->mmio_peer_dbmsg->msg_map);
0477     iowrite64(sndev->db_valid_mask << sndev->db_peer_shift,
0478           &sndev->mmio_peer_dbmsg->odb_mask);
0479 }
0480 
0481 enum switchtec_msg {
0482     LINK_MESSAGE = 0,
0483     MSG_LINK_UP = 1,
0484     MSG_LINK_DOWN = 2,
0485     MSG_CHECK_LINK = 3,
0486     MSG_LINK_FORCE_DOWN = 4,
0487 };
0488 
0489 static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev);
0490 
0491 static void switchtec_ntb_link_status_update(struct switchtec_ntb *sndev)
0492 {
0493     int link_sta;
0494     int old = sndev->link_is_up;
0495 
0496     link_sta = sndev->self_shared->link_sta;
0497     if (link_sta) {
0498         u64 peer = ioread64(&sndev->peer_shared->magic);
0499 
0500         if ((peer & 0xFFFFFFFF) == SWITCHTEC_NTB_MAGIC)
0501             link_sta = peer >> 32;
0502         else
0503             link_sta = 0;
0504     }
0505 
0506     sndev->link_is_up = link_sta;
0507     switchtec_ntb_set_link_speed(sndev);
0508 
0509     if (link_sta != old) {
0510         switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_CHECK_LINK);
0511         ntb_link_event(&sndev->ntb);
0512         dev_info(&sndev->stdev->dev, "ntb link %s\n",
0513              link_sta ? "up" : "down");
0514 
0515         if (link_sta)
0516             crosslink_init_dbmsgs(sndev);
0517     }
0518 }
0519 
0520 static void check_link_status_work(struct work_struct *work)
0521 {
0522     struct switchtec_ntb *sndev;
0523 
0524     sndev = container_of(work, struct switchtec_ntb,
0525                  check_link_status_work);
0526 
0527     if (sndev->link_force_down) {
0528         sndev->link_force_down = false;
0529         switchtec_ntb_reinit_peer(sndev);
0530 
0531         if (sndev->link_is_up) {
0532             sndev->link_is_up = 0;
0533             ntb_link_event(&sndev->ntb);
0534             dev_info(&sndev->stdev->dev, "ntb link forced down\n");
0535         }
0536 
0537         return;
0538     }
0539 
0540     switchtec_ntb_link_status_update(sndev);
0541 }
0542 
0543 static void switchtec_ntb_check_link(struct switchtec_ntb *sndev,
0544                       enum switchtec_msg msg)
0545 {
0546     if (msg == MSG_LINK_FORCE_DOWN)
0547         sndev->link_force_down = true;
0548 
0549     schedule_work(&sndev->check_link_status_work);
0550 }
0551 
0552 static void switchtec_ntb_link_notification(struct switchtec_dev *stdev)
0553 {
0554     struct switchtec_ntb *sndev = stdev->sndev;
0555 
0556     switchtec_ntb_check_link(sndev, MSG_CHECK_LINK);
0557 }
0558 
0559 static u64 switchtec_ntb_link_is_up(struct ntb_dev *ntb,
0560                     enum ntb_speed *speed,
0561                     enum ntb_width *width)
0562 {
0563     struct switchtec_ntb *sndev = ntb_sndev(ntb);
0564 
0565     if (speed)
0566         *speed = sndev->link_speed;
0567     if (width)
0568         *width = sndev->link_width;
0569 
0570     return sndev->link_is_up;
0571 }
0572 
0573 static int switchtec_ntb_link_enable(struct ntb_dev *ntb,
0574                      enum ntb_speed max_speed,
0575                      enum ntb_width max_width)
0576 {
0577     struct switchtec_ntb *sndev = ntb_sndev(ntb);
0578 
0579     dev_dbg(&sndev->stdev->dev, "enabling link\n");
0580 
0581     sndev->self_shared->link_sta = 1;
0582     switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_UP);
0583 
0584     switchtec_ntb_link_status_update(sndev);
0585 
0586     return 0;
0587 }
0588 
0589 static int switchtec_ntb_link_disable(struct ntb_dev *ntb)
0590 {
0591     struct switchtec_ntb *sndev = ntb_sndev(ntb);
0592 
0593     dev_dbg(&sndev->stdev->dev, "disabling link\n");
0594 
0595     sndev->self_shared->link_sta = 0;
0596     switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_DOWN);
0597 
0598     switchtec_ntb_link_status_update(sndev);
0599 
0600     return 0;
0601 }
0602 
0603 static u64 switchtec_ntb_db_valid_mask(struct ntb_dev *ntb)
0604 {
0605     struct switchtec_ntb *sndev = ntb_sndev(ntb);
0606 
0607     return sndev->db_valid_mask;
0608 }
0609 
0610 static int switchtec_ntb_db_vector_count(struct ntb_dev *ntb)
0611 {
0612     return 1;
0613 }
0614 
0615 static u64 switchtec_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
0616 {
0617     struct switchtec_ntb *sndev = ntb_sndev(ntb);
0618 
0619     if (db_vector < 0 || db_vector > 1)
0620         return 0;
0621 
0622     return sndev->db_valid_mask;
0623 }
0624 
0625 static u64 switchtec_ntb_db_read(struct ntb_dev *ntb)
0626 {
0627     u64 ret;
0628     struct switchtec_ntb *sndev = ntb_sndev(ntb);
0629 
0630     ret = ioread64(&sndev->mmio_self_dbmsg->idb) >> sndev->db_shift;
0631 
0632     return ret & sndev->db_valid_mask;
0633 }
0634 
0635 static int switchtec_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
0636 {
0637     struct switchtec_ntb *sndev = ntb_sndev(ntb);
0638 
0639     iowrite64(db_bits << sndev->db_shift, &sndev->mmio_self_dbmsg->idb);
0640 
0641     return 0;
0642 }
0643 
0644 static int switchtec_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
0645 {
0646     unsigned long irqflags;
0647     struct switchtec_ntb *sndev = ntb_sndev(ntb);
0648 
0649     if (db_bits & ~sndev->db_valid_mask)
0650         return -EINVAL;
0651 
0652     spin_lock_irqsave(&sndev->db_mask_lock, irqflags);
0653 
0654     sndev->db_mask |= db_bits << sndev->db_shift;
0655     iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask);
0656 
0657     spin_unlock_irqrestore(&sndev->db_mask_lock, irqflags);
0658 
0659     return 0;
0660 }
0661 
0662 static int switchtec_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
0663 {
0664     unsigned long irqflags;
0665     struct switchtec_ntb *sndev = ntb_sndev(ntb);
0666 
0667     if (db_bits & ~sndev->db_valid_mask)
0668         return -EINVAL;
0669 
0670     spin_lock_irqsave(&sndev->db_mask_lock, irqflags);
0671 
0672     sndev->db_mask &= ~(db_bits << sndev->db_shift);
0673     iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask);
0674 
0675     spin_unlock_irqrestore(&sndev->db_mask_lock, irqflags);
0676 
0677     return 0;
0678 }
0679 
0680 static u64 switchtec_ntb_db_read_mask(struct ntb_dev *ntb)
0681 {
0682     struct switchtec_ntb *sndev = ntb_sndev(ntb);
0683 
0684     return (sndev->db_mask >> sndev->db_shift) & sndev->db_valid_mask;
0685 }
0686 
0687 static int switchtec_ntb_peer_db_addr(struct ntb_dev *ntb,
0688                       phys_addr_t *db_addr,
0689                       resource_size_t *db_size,
0690                       u64 *db_data,
0691                       int db_bit)
0692 {
0693     struct switchtec_ntb *sndev = ntb_sndev(ntb);
0694     unsigned long offset;
0695 
0696     if (unlikely(db_bit >= BITS_PER_LONG_LONG))
0697         return -EINVAL;
0698 
0699     offset = (unsigned long)sndev->mmio_peer_dbmsg->odb -
0700         (unsigned long)sndev->stdev->mmio;
0701 
0702     offset += sndev->db_shift / 8;
0703 
0704     if (db_addr)
0705         *db_addr = pci_resource_start(ntb->pdev, 0) + offset;
0706     if (db_size)
0707         *db_size = sizeof(u32);
0708     if (db_data)
0709         *db_data = BIT_ULL(db_bit) << sndev->db_peer_shift;
0710 
0711     return 0;
0712 }
0713 
0714 static int switchtec_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
0715 {
0716     struct switchtec_ntb *sndev = ntb_sndev(ntb);
0717 
0718     iowrite64(db_bits << sndev->db_peer_shift,
0719           &sndev->mmio_peer_dbmsg->odb);
0720 
0721     return 0;
0722 }
0723 
0724 static int switchtec_ntb_spad_count(struct ntb_dev *ntb)
0725 {
0726     struct switchtec_ntb *sndev = ntb_sndev(ntb);
0727 
0728     return ARRAY_SIZE(sndev->self_shared->spad);
0729 }
0730 
0731 static u32 switchtec_ntb_spad_read(struct ntb_dev *ntb, int idx)
0732 {
0733     struct switchtec_ntb *sndev = ntb_sndev(ntb);
0734 
0735     if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad))
0736         return 0;
0737 
0738     if (!sndev->self_shared)
0739         return 0;
0740 
0741     return sndev->self_shared->spad[idx];
0742 }
0743 
0744 static int switchtec_ntb_spad_write(struct ntb_dev *ntb, int idx, u32 val)
0745 {
0746     struct switchtec_ntb *sndev = ntb_sndev(ntb);
0747 
0748     if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad))
0749         return -EINVAL;
0750 
0751     if (!sndev->self_shared)
0752         return -EIO;
0753 
0754     sndev->self_shared->spad[idx] = val;
0755 
0756     return 0;
0757 }
0758 
0759 static u32 switchtec_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx,
0760                     int sidx)
0761 {
0762     struct switchtec_ntb *sndev = ntb_sndev(ntb);
0763 
0764     if (pidx != NTB_DEF_PEER_IDX)
0765         return -EINVAL;
0766 
0767     if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad))
0768         return 0;
0769 
0770     if (!sndev->peer_shared)
0771         return 0;
0772 
0773     return ioread32(&sndev->peer_shared->spad[sidx]);
0774 }
0775 
0776 static int switchtec_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx,
0777                      int sidx, u32 val)
0778 {
0779     struct switchtec_ntb *sndev = ntb_sndev(ntb);
0780 
0781     if (pidx != NTB_DEF_PEER_IDX)
0782         return -EINVAL;
0783 
0784     if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad))
0785         return -EINVAL;
0786 
0787     if (!sndev->peer_shared)
0788         return -EIO;
0789 
0790     iowrite32(val, &sndev->peer_shared->spad[sidx]);
0791 
0792     return 0;
0793 }
0794 
0795 static int switchtec_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx,
0796                     int sidx, phys_addr_t *spad_addr)
0797 {
0798     struct switchtec_ntb *sndev = ntb_sndev(ntb);
0799     unsigned long offset;
0800 
0801     if (pidx != NTB_DEF_PEER_IDX)
0802         return -EINVAL;
0803 
0804     offset = (unsigned long)&sndev->peer_shared->spad[sidx] -
0805         (unsigned long)sndev->stdev->mmio;
0806 
0807     if (spad_addr)
0808         *spad_addr = pci_resource_start(ntb->pdev, 0) + offset;
0809 
0810     return 0;
0811 }
0812 
0813 static const struct ntb_dev_ops switchtec_ntb_ops = {
0814     .mw_count       = switchtec_ntb_mw_count,
0815     .mw_get_align       = switchtec_ntb_mw_get_align,
0816     .mw_set_trans       = switchtec_ntb_mw_set_trans,
0817     .peer_mw_count      = switchtec_ntb_peer_mw_count,
0818     .peer_mw_get_addr   = switchtec_ntb_peer_mw_get_addr,
0819     .link_is_up     = switchtec_ntb_link_is_up,
0820     .link_enable        = switchtec_ntb_link_enable,
0821     .link_disable       = switchtec_ntb_link_disable,
0822     .db_valid_mask      = switchtec_ntb_db_valid_mask,
0823     .db_vector_count    = switchtec_ntb_db_vector_count,
0824     .db_vector_mask     = switchtec_ntb_db_vector_mask,
0825     .db_read        = switchtec_ntb_db_read,
0826     .db_clear       = switchtec_ntb_db_clear,
0827     .db_set_mask        = switchtec_ntb_db_set_mask,
0828     .db_clear_mask      = switchtec_ntb_db_clear_mask,
0829     .db_read_mask       = switchtec_ntb_db_read_mask,
0830     .peer_db_addr       = switchtec_ntb_peer_db_addr,
0831     .peer_db_set        = switchtec_ntb_peer_db_set,
0832     .spad_count     = switchtec_ntb_spad_count,
0833     .spad_read      = switchtec_ntb_spad_read,
0834     .spad_write     = switchtec_ntb_spad_write,
0835     .peer_spad_read     = switchtec_ntb_peer_spad_read,
0836     .peer_spad_write    = switchtec_ntb_peer_spad_write,
0837     .peer_spad_addr     = switchtec_ntb_peer_spad_addr,
0838 };
0839 
0840 static int switchtec_ntb_init_sndev(struct switchtec_ntb *sndev)
0841 {
0842     u64 tpart_vec;
0843     int self;
0844     u64 part_map;
0845 
0846     sndev->ntb.pdev = sndev->stdev->pdev;
0847     sndev->ntb.topo = NTB_TOPO_SWITCH;
0848     sndev->ntb.ops = &switchtec_ntb_ops;
0849 
0850     INIT_WORK(&sndev->check_link_status_work, check_link_status_work);
0851     sndev->link_force_down = false;
0852 
0853     sndev->self_partition = sndev->stdev->partition;
0854 
0855     sndev->mmio_ntb = sndev->stdev->mmio_ntb;
0856 
0857     self = sndev->self_partition;
0858     tpart_vec = ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_high);
0859     tpart_vec <<= 32;
0860     tpart_vec |= ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_low);
0861 
0862     part_map = ioread64(&sndev->mmio_ntb->ep_map);
0863     tpart_vec &= part_map;
0864     part_map &= ~(1 << sndev->self_partition);
0865 
0866     if (!tpart_vec) {
0867         if (sndev->stdev->partition_count != 2) {
0868             dev_err(&sndev->stdev->dev,
0869                 "ntb target partition not defined\n");
0870             return -ENODEV;
0871         }
0872 
0873         if (!part_map) {
0874             dev_err(&sndev->stdev->dev,
0875                 "peer partition is not NT partition\n");
0876             return -ENODEV;
0877         }
0878 
0879         sndev->peer_partition = __ffs64(part_map);
0880     } else {
0881         if (__ffs64(tpart_vec) != (fls64(tpart_vec) - 1)) {
0882             dev_err(&sndev->stdev->dev,
0883                 "ntb driver only supports 1 pair of 1-1 ntb mapping\n");
0884             return -ENODEV;
0885         }
0886 
0887         sndev->peer_partition = __ffs64(tpart_vec);
0888         if (!(part_map & (1ULL << sndev->peer_partition))) {
0889             dev_err(&sndev->stdev->dev,
0890                 "ntb target partition is not NT partition\n");
0891             return -ENODEV;
0892         }
0893     }
0894 
0895     dev_dbg(&sndev->stdev->dev, "Partition ID %d of %d\n",
0896         sndev->self_partition, sndev->stdev->partition_count);
0897 
0898     sndev->mmio_ctrl = (void * __iomem)sndev->mmio_ntb +
0899         SWITCHTEC_NTB_REG_CTRL_OFFSET;
0900     sndev->mmio_dbmsg = (void * __iomem)sndev->mmio_ntb +
0901         SWITCHTEC_NTB_REG_DBMSG_OFFSET;
0902 
0903     sndev->mmio_self_ctrl = &sndev->mmio_ctrl[sndev->self_partition];
0904     sndev->mmio_peer_ctrl = &sndev->mmio_ctrl[sndev->peer_partition];
0905     sndev->mmio_self_dbmsg = &sndev->mmio_dbmsg[sndev->self_partition];
0906     sndev->mmio_peer_dbmsg = sndev->mmio_self_dbmsg;
0907 
0908     return 0;
0909 }
0910 
0911 static int config_rsvd_lut_win(struct switchtec_ntb *sndev,
0912                    struct ntb_ctrl_regs __iomem *ctl,
0913                    int lut_idx, int partition, u64 addr)
0914 {
0915     int peer_bar = sndev->peer_direct_mw_to_bar[0];
0916     u32 ctl_val;
0917     int rc;
0918 
0919     rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK,
0920                    NTB_CTRL_PART_STATUS_LOCKED);
0921     if (rc)
0922         return rc;
0923 
0924     ctl_val = ioread32(&ctl->bar_entry[peer_bar].ctl);
0925     ctl_val &= 0xFF;
0926     ctl_val |= NTB_CTRL_BAR_LUT_WIN_EN;
0927     ctl_val |= ilog2(LUT_SIZE) << 8;
0928     ctl_val |= (sndev->nr_lut_mw - 1) << 14;
0929     iowrite32(ctl_val, &ctl->bar_entry[peer_bar].ctl);
0930 
0931     iowrite64((NTB_CTRL_LUT_EN | (partition << 1) | addr),
0932           &ctl->lut_entry[lut_idx]);
0933 
0934     rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
0935                    NTB_CTRL_PART_STATUS_NORMAL);
0936     if (rc) {
0937         u32 bar_error, lut_error;
0938 
0939         bar_error = ioread32(&ctl->bar_error);
0940         lut_error = ioread32(&ctl->lut_error);
0941         dev_err(&sndev->stdev->dev,
0942             "Error setting up reserved lut window: %08x / %08x\n",
0943             bar_error, lut_error);
0944         return rc;
0945     }
0946 
0947     return 0;
0948 }
0949 
0950 static int config_req_id_table(struct switchtec_ntb *sndev,
0951                    struct ntb_ctrl_regs __iomem *mmio_ctrl,
0952                    int *req_ids, int count)
0953 {
0954     int i, rc = 0;
0955     u32 error;
0956     u32 proxy_id;
0957 
0958     if (ioread16(&mmio_ctrl->req_id_table_size) < count) {
0959         dev_err(&sndev->stdev->dev,
0960             "Not enough requester IDs available.\n");
0961         return -EFAULT;
0962     }
0963 
0964     rc = switchtec_ntb_part_op(sndev, mmio_ctrl,
0965                    NTB_CTRL_PART_OP_LOCK,
0966                    NTB_CTRL_PART_STATUS_LOCKED);
0967     if (rc)
0968         return rc;
0969 
0970     for (i = 0; i < count; i++) {
0971         iowrite32(req_ids[i] << 16 | NTB_CTRL_REQ_ID_EN,
0972               &mmio_ctrl->req_id_table[i]);
0973 
0974         proxy_id = ioread32(&mmio_ctrl->req_id_table[i]);
0975         dev_dbg(&sndev->stdev->dev,
0976             "Requester ID %02X:%02X.%X -> BB:%02X.%X\n",
0977             req_ids[i] >> 8, (req_ids[i] >> 3) & 0x1F,
0978             req_ids[i] & 0x7, (proxy_id >> 4) & 0x1F,
0979             (proxy_id >> 1) & 0x7);
0980     }
0981 
0982     rc = switchtec_ntb_part_op(sndev, mmio_ctrl,
0983                    NTB_CTRL_PART_OP_CFG,
0984                    NTB_CTRL_PART_STATUS_NORMAL);
0985 
0986     if (rc == -EIO) {
0987         error = ioread32(&mmio_ctrl->req_id_error);
0988         dev_err(&sndev->stdev->dev,
0989             "Error setting up the requester ID table: %08x\n",
0990             error);
0991     }
0992 
0993     return 0;
0994 }
0995 
0996 static int crosslink_setup_mws(struct switchtec_ntb *sndev, int ntb_lut_idx,
0997                    u64 *mw_addrs, int mw_count)
0998 {
0999     int rc, i;
1000     struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_self_ctrl;
1001     u64 addr;
1002     size_t size, offset;
1003     int bar;
1004     int xlate_pos;
1005     u32 ctl_val;
1006 
1007     rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK,
1008                    NTB_CTRL_PART_STATUS_LOCKED);
1009     if (rc)
1010         return rc;
1011 
1012     for (i = 0; i < sndev->nr_lut_mw; i++) {
1013         if (i == ntb_lut_idx)
1014             continue;
1015 
1016         addr = mw_addrs[0] + LUT_SIZE * i;
1017 
1018         iowrite64((NTB_CTRL_LUT_EN | (sndev->peer_partition << 1) |
1019                addr),
1020               &ctl->lut_entry[i]);
1021     }
1022 
1023     sndev->nr_direct_mw = min_t(int, sndev->nr_direct_mw, mw_count);
1024 
1025     for (i = 0; i < sndev->nr_direct_mw; i++) {
1026         bar = sndev->direct_mw_to_bar[i];
1027         offset = (i == 0) ? LUT_SIZE * sndev->nr_lut_mw : 0;
1028         addr = mw_addrs[i] + offset;
1029         size = pci_resource_len(sndev->ntb.pdev, bar) - offset;
1030         xlate_pos = ilog2(size);
1031 
1032         if (offset && size > offset)
1033             size = offset;
1034 
1035         ctl_val = ioread32(&ctl->bar_entry[bar].ctl);
1036         ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN;
1037 
1038         iowrite32(ctl_val, &ctl->bar_entry[bar].ctl);
1039         iowrite32(xlate_pos | (lower_32_bits(size) & 0xFFFFF000),
1040               &ctl->bar_entry[bar].win_size);
1041         iowrite32(upper_32_bits(size), &ctl->bar_ext_entry[bar].win_size);
1042         iowrite64(sndev->peer_partition | addr,
1043               &ctl->bar_entry[bar].xlate_addr);
1044     }
1045 
1046     rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
1047                    NTB_CTRL_PART_STATUS_NORMAL);
1048     if (rc) {
1049         u32 bar_error, lut_error;
1050 
1051         bar_error = ioread32(&ctl->bar_error);
1052         lut_error = ioread32(&ctl->lut_error);
1053         dev_err(&sndev->stdev->dev,
1054             "Error setting up cross link windows: %08x / %08x\n",
1055             bar_error, lut_error);
1056         return rc;
1057     }
1058 
1059     return 0;
1060 }
1061 
1062 static int crosslink_setup_req_ids(struct switchtec_ntb *sndev,
1063     struct ntb_ctrl_regs __iomem *mmio_ctrl)
1064 {
1065     int req_ids[16];
1066     int i;
1067     u32 proxy_id;
1068 
1069     for (i = 0; i < ARRAY_SIZE(req_ids); i++) {
1070         proxy_id = ioread32(&sndev->mmio_self_ctrl->req_id_table[i]);
1071 
1072         if (!(proxy_id & NTB_CTRL_REQ_ID_EN))
1073             break;
1074 
1075         req_ids[i] = ((proxy_id >> 1) & 0xFF);
1076     }
1077 
1078     return config_req_id_table(sndev, mmio_ctrl, req_ids, i);
1079 }
1080 
1081 /*
1082  * In crosslink configuration there is a virtual partition in the
1083  * middle of the two switches. The BARs in this partition have to be
1084  * enumerated and assigned addresses.
1085  */
1086 static int crosslink_enum_partition(struct switchtec_ntb *sndev,
1087                     u64 *bar_addrs)
1088 {
1089     struct part_cfg_regs __iomem *part_cfg =
1090         &sndev->stdev->mmio_part_cfg_all[sndev->peer_partition];
1091     u32 pff = ioread32(&part_cfg->vep_pff_inst_id) & 0xFF;
1092     struct pff_csr_regs __iomem *mmio_pff =
1093         &sndev->stdev->mmio_pff_csr[pff];
1094     const u64 bar_space = 0x1000000000LL;
1095     u64 bar_addr;
1096     int bar_cnt = 0;
1097     int i;
1098 
1099     iowrite16(0x6, &mmio_pff->pcicmd);
1100 
1101     for (i = 0; i < ARRAY_SIZE(mmio_pff->pci_bar64); i++) {
1102         iowrite64(bar_space * i, &mmio_pff->pci_bar64[i]);
1103         bar_addr = ioread64(&mmio_pff->pci_bar64[i]);
1104         bar_addr &= ~0xf;
1105 
1106         dev_dbg(&sndev->stdev->dev,
1107             "Crosslink BAR%d addr: %llx\n",
1108             i*2, bar_addr);
1109 
1110         if (bar_addr != bar_space * i)
1111             continue;
1112 
1113         bar_addrs[bar_cnt++] = bar_addr;
1114     }
1115 
1116     return bar_cnt;
1117 }
1118 
1119 static int switchtec_ntb_init_crosslink(struct switchtec_ntb *sndev)
1120 {
1121     int rc;
1122     int bar = sndev->direct_mw_to_bar[0];
1123     const int ntb_lut_idx = 1;
1124     u64 bar_addrs[6];
1125     u64 addr;
1126     int offset;
1127     int bar_cnt;
1128 
1129     if (!crosslink_is_enabled(sndev))
1130         return 0;
1131 
1132     dev_info(&sndev->stdev->dev, "Using crosslink configuration\n");
1133     sndev->ntb.topo = NTB_TOPO_CROSSLINK;
1134 
1135     bar_cnt = crosslink_enum_partition(sndev, bar_addrs);
1136     if (bar_cnt < sndev->nr_direct_mw + 1) {
1137         dev_err(&sndev->stdev->dev,
1138             "Error enumerating crosslink partition\n");
1139         return -EINVAL;
1140     }
1141 
1142     addr = (bar_addrs[0] + SWITCHTEC_GAS_NTB_OFFSET +
1143         SWITCHTEC_NTB_REG_DBMSG_OFFSET +
1144         sizeof(struct ntb_dbmsg_regs) * sndev->peer_partition);
1145 
1146     offset = addr & (LUT_SIZE - 1);
1147     addr -= offset;
1148 
1149     rc = config_rsvd_lut_win(sndev, sndev->mmio_self_ctrl, ntb_lut_idx,
1150                  sndev->peer_partition, addr);
1151     if (rc)
1152         return rc;
1153 
1154     rc = crosslink_setup_mws(sndev, ntb_lut_idx, &bar_addrs[1],
1155                  bar_cnt - 1);
1156     if (rc)
1157         return rc;
1158 
1159     rc = crosslink_setup_req_ids(sndev, sndev->mmio_peer_ctrl);
1160     if (rc)
1161         return rc;
1162 
1163     sndev->mmio_xlink_win = pci_iomap_range(sndev->stdev->pdev, bar,
1164                         LUT_SIZE, LUT_SIZE);
1165     if (!sndev->mmio_xlink_win) {
1166         rc = -ENOMEM;
1167         return rc;
1168     }
1169 
1170     sndev->mmio_peer_dbmsg = sndev->mmio_xlink_win + offset;
1171     sndev->nr_rsvd_luts++;
1172 
1173     crosslink_init_dbmsgs(sndev);
1174 
1175     return 0;
1176 }
1177 
1178 static void switchtec_ntb_deinit_crosslink(struct switchtec_ntb *sndev)
1179 {
1180     if (sndev->mmio_xlink_win)
1181         pci_iounmap(sndev->stdev->pdev, sndev->mmio_xlink_win);
1182 }
1183 
1184 static int map_bars(int *map, struct ntb_ctrl_regs __iomem *ctrl)
1185 {
1186     int i;
1187     int cnt = 0;
1188 
1189     for (i = 0; i < ARRAY_SIZE(ctrl->bar_entry); i++) {
1190         u32 r = ioread32(&ctrl->bar_entry[i].ctl);
1191 
1192         if (r & NTB_CTRL_BAR_VALID)
1193             map[cnt++] = i;
1194     }
1195 
1196     return cnt;
1197 }
1198 
1199 static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev)
1200 {
1201     sndev->nr_direct_mw = map_bars(sndev->direct_mw_to_bar,
1202                        sndev->mmio_self_ctrl);
1203 
1204     sndev->nr_lut_mw = ioread16(&sndev->mmio_self_ctrl->lut_table_entries);
1205     sndev->nr_lut_mw = rounddown_pow_of_two(sndev->nr_lut_mw);
1206 
1207     dev_dbg(&sndev->stdev->dev, "MWs: %d direct, %d lut\n",
1208         sndev->nr_direct_mw, sndev->nr_lut_mw);
1209 
1210     sndev->peer_nr_direct_mw = map_bars(sndev->peer_direct_mw_to_bar,
1211                         sndev->mmio_peer_ctrl);
1212 
1213     sndev->peer_nr_lut_mw =
1214         ioread16(&sndev->mmio_peer_ctrl->lut_table_entries);
1215     sndev->peer_nr_lut_mw = rounddown_pow_of_two(sndev->peer_nr_lut_mw);
1216 
1217     dev_dbg(&sndev->stdev->dev, "Peer MWs: %d direct, %d lut\n",
1218         sndev->peer_nr_direct_mw, sndev->peer_nr_lut_mw);
1219 
1220 }
1221 
1222 /*
1223  * There are 64 doorbells in the switch hardware but this is
1224  * shared among all partitions. So we must split them in half
1225  * (32 for each partition). However, the message interrupts are
1226  * also shared with the top 4 doorbells so we just limit this to
1227  * 28 doorbells per partition.
1228  *
1229  * In crosslink mode, each side has it's own dbmsg register so
1230  * they can each use all 60 of the available doorbells.
1231  */
1232 static void switchtec_ntb_init_db(struct switchtec_ntb *sndev)
1233 {
1234     sndev->db_mask = 0x0FFFFFFFFFFFFFFFULL;
1235 
1236     if (sndev->mmio_peer_dbmsg != sndev->mmio_self_dbmsg) {
1237         sndev->db_shift = 0;
1238         sndev->db_peer_shift = 0;
1239         sndev->db_valid_mask = sndev->db_mask;
1240     } else if (sndev->self_partition < sndev->peer_partition) {
1241         sndev->db_shift = 0;
1242         sndev->db_peer_shift = 32;
1243         sndev->db_valid_mask = 0x0FFFFFFF;
1244     } else {
1245         sndev->db_shift = 32;
1246         sndev->db_peer_shift = 0;
1247         sndev->db_valid_mask = 0x0FFFFFFF;
1248     }
1249 
1250     iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask);
1251     iowrite64(sndev->db_valid_mask << sndev->db_peer_shift,
1252           &sndev->mmio_peer_dbmsg->odb_mask);
1253 
1254     dev_dbg(&sndev->stdev->dev, "dbs: shift %d/%d, mask %016llx\n",
1255         sndev->db_shift, sndev->db_peer_shift, sndev->db_valid_mask);
1256 }
1257 
1258 static void switchtec_ntb_init_msgs(struct switchtec_ntb *sndev)
1259 {
1260     int i;
1261     u32 msg_map = 0;
1262 
1263     for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) {
1264         int m = i | sndev->peer_partition << 2;
1265 
1266         msg_map |= m << i * 8;
1267     }
1268 
1269     iowrite32(msg_map, &sndev->mmio_self_dbmsg->msg_map);
1270 
1271     for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++)
1272         iowrite64(NTB_DBMSG_IMSG_STATUS | NTB_DBMSG_IMSG_MASK,
1273               &sndev->mmio_self_dbmsg->imsg[i]);
1274 }
1275 
1276 static int
1277 switchtec_ntb_init_req_id_table(struct switchtec_ntb *sndev)
1278 {
1279     int req_ids[2];
1280 
1281     /*
1282      * Root Complex Requester ID (which is 0:00.0)
1283      */
1284     req_ids[0] = 0;
1285 
1286     /*
1287      * Host Bridge Requester ID (as read from the mmap address)
1288      */
1289     req_ids[1] = ioread16(&sndev->mmio_ntb->requester_id);
1290 
1291     return config_req_id_table(sndev, sndev->mmio_self_ctrl, req_ids,
1292                    ARRAY_SIZE(req_ids));
1293 }
1294 
1295 static void switchtec_ntb_init_shared(struct switchtec_ntb *sndev)
1296 {
1297     int i;
1298 
1299     memset(sndev->self_shared, 0, LUT_SIZE);
1300     sndev->self_shared->magic = SWITCHTEC_NTB_MAGIC;
1301     sndev->self_shared->partition_id = sndev->stdev->partition;
1302 
1303     for (i = 0; i < sndev->nr_direct_mw; i++) {
1304         int bar = sndev->direct_mw_to_bar[i];
1305         resource_size_t sz = pci_resource_len(sndev->stdev->pdev, bar);
1306 
1307         if (i == 0)
1308             sz = min_t(resource_size_t, sz,
1309                    LUT_SIZE * sndev->nr_lut_mw);
1310 
1311         sndev->self_shared->mw_sizes[i] = sz;
1312     }
1313 
1314     for (i = 0; i < sndev->nr_lut_mw; i++) {
1315         int idx = sndev->nr_direct_mw + i;
1316 
1317         sndev->self_shared->mw_sizes[idx] = LUT_SIZE;
1318     }
1319 }
1320 
1321 static int switchtec_ntb_init_shared_mw(struct switchtec_ntb *sndev)
1322 {
1323     int self_bar = sndev->direct_mw_to_bar[0];
1324     int rc;
1325 
1326     sndev->nr_rsvd_luts++;
1327     sndev->self_shared = dma_alloc_coherent(&sndev->stdev->pdev->dev,
1328                         LUT_SIZE,
1329                         &sndev->self_shared_dma,
1330                         GFP_KERNEL);
1331     if (!sndev->self_shared) {
1332         dev_err(&sndev->stdev->dev,
1333             "unable to allocate memory for shared mw\n");
1334         return -ENOMEM;
1335     }
1336 
1337     switchtec_ntb_init_shared(sndev);
1338 
1339     rc = config_rsvd_lut_win(sndev, sndev->mmio_peer_ctrl, 0,
1340                  sndev->self_partition,
1341                  sndev->self_shared_dma);
1342     if (rc)
1343         goto unalloc_and_exit;
1344 
1345     sndev->peer_shared = pci_iomap(sndev->stdev->pdev, self_bar, LUT_SIZE);
1346     if (!sndev->peer_shared) {
1347         rc = -ENOMEM;
1348         goto unalloc_and_exit;
1349     }
1350 
1351     dev_dbg(&sndev->stdev->dev, "Shared MW Ready\n");
1352     return 0;
1353 
1354 unalloc_and_exit:
1355     dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE,
1356               sndev->self_shared, sndev->self_shared_dma);
1357 
1358     return rc;
1359 }
1360 
1361 static void switchtec_ntb_deinit_shared_mw(struct switchtec_ntb *sndev)
1362 {
1363     if (sndev->peer_shared)
1364         pci_iounmap(sndev->stdev->pdev, sndev->peer_shared);
1365 
1366     if (sndev->self_shared)
1367         dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE,
1368                   sndev->self_shared,
1369                   sndev->self_shared_dma);
1370     sndev->nr_rsvd_luts--;
1371 }
1372 
1373 static irqreturn_t switchtec_ntb_doorbell_isr(int irq, void *dev)
1374 {
1375     struct switchtec_ntb *sndev = dev;
1376 
1377     dev_dbg(&sndev->stdev->dev, "doorbell\n");
1378 
1379     ntb_db_event(&sndev->ntb, 0);
1380 
1381     return IRQ_HANDLED;
1382 }
1383 
1384 static irqreturn_t switchtec_ntb_message_isr(int irq, void *dev)
1385 {
1386     int i;
1387     struct switchtec_ntb *sndev = dev;
1388 
1389     for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) {
1390         u64 msg = ioread64(&sndev->mmio_self_dbmsg->imsg[i]);
1391 
1392         if (msg & NTB_DBMSG_IMSG_STATUS) {
1393             dev_dbg(&sndev->stdev->dev, "message: %d %08x\n",
1394                 i, (u32)msg);
1395             iowrite8(1, &sndev->mmio_self_dbmsg->imsg[i].status);
1396 
1397             if (i == LINK_MESSAGE)
1398                 switchtec_ntb_check_link(sndev, msg);
1399         }
1400     }
1401 
1402     return IRQ_HANDLED;
1403 }
1404 
1405 static int switchtec_ntb_init_db_msg_irq(struct switchtec_ntb *sndev)
1406 {
1407     int i;
1408     int rc;
1409     int doorbell_irq = 0;
1410     int message_irq = 0;
1411     int event_irq;
1412     int idb_vecs = sizeof(sndev->mmio_self_dbmsg->idb_vec_map);
1413 
1414     event_irq = ioread32(&sndev->stdev->mmio_part_cfg->vep_vector_number);
1415 
1416     while (doorbell_irq == event_irq)
1417         doorbell_irq++;
1418     while (message_irq == doorbell_irq ||
1419            message_irq == event_irq)
1420         message_irq++;
1421 
1422     dev_dbg(&sndev->stdev->dev, "irqs - event: %d, db: %d, msgs: %d\n",
1423         event_irq, doorbell_irq, message_irq);
1424 
1425     for (i = 0; i < idb_vecs - 4; i++)
1426         iowrite8(doorbell_irq,
1427              &sndev->mmio_self_dbmsg->idb_vec_map[i]);
1428 
1429     for (; i < idb_vecs; i++)
1430         iowrite8(message_irq,
1431              &sndev->mmio_self_dbmsg->idb_vec_map[i]);
1432 
1433     sndev->doorbell_irq = pci_irq_vector(sndev->stdev->pdev, doorbell_irq);
1434     sndev->message_irq = pci_irq_vector(sndev->stdev->pdev, message_irq);
1435 
1436     rc = request_irq(sndev->doorbell_irq,
1437              switchtec_ntb_doorbell_isr, 0,
1438              "switchtec_ntb_doorbell", sndev);
1439     if (rc)
1440         return rc;
1441 
1442     rc = request_irq(sndev->message_irq,
1443              switchtec_ntb_message_isr, 0,
1444              "switchtec_ntb_message", sndev);
1445     if (rc) {
1446         free_irq(sndev->doorbell_irq, sndev);
1447         return rc;
1448     }
1449 
1450     return 0;
1451 }
1452 
1453 static void switchtec_ntb_deinit_db_msg_irq(struct switchtec_ntb *sndev)
1454 {
1455     free_irq(sndev->doorbell_irq, sndev);
1456     free_irq(sndev->message_irq, sndev);
1457 }
1458 
1459 static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev)
1460 {
1461     int rc;
1462 
1463     if (crosslink_is_enabled(sndev))
1464         return 0;
1465 
1466     dev_info(&sndev->stdev->dev, "reinitialize shared memory window\n");
1467     rc = config_rsvd_lut_win(sndev, sndev->mmio_peer_ctrl, 0,
1468                  sndev->self_partition,
1469                  sndev->self_shared_dma);
1470     return rc;
1471 }
1472 
1473 static int switchtec_ntb_add(struct device *dev,
1474                  struct class_interface *class_intf)
1475 {
1476     struct switchtec_dev *stdev = to_stdev(dev);
1477     struct switchtec_ntb *sndev;
1478     int rc;
1479 
1480     stdev->sndev = NULL;
1481 
1482     if (stdev->pdev->class != (PCI_CLASS_BRIDGE_OTHER << 8))
1483         return -ENODEV;
1484 
1485     sndev = kzalloc_node(sizeof(*sndev), GFP_KERNEL, dev_to_node(dev));
1486     if (!sndev)
1487         return -ENOMEM;
1488 
1489     sndev->stdev = stdev;
1490     rc = switchtec_ntb_init_sndev(sndev);
1491     if (rc)
1492         goto free_and_exit;
1493 
1494     switchtec_ntb_init_mw(sndev);
1495 
1496     rc = switchtec_ntb_init_req_id_table(sndev);
1497     if (rc)
1498         goto free_and_exit;
1499 
1500     rc = switchtec_ntb_init_crosslink(sndev);
1501     if (rc)
1502         goto free_and_exit;
1503 
1504     switchtec_ntb_init_db(sndev);
1505     switchtec_ntb_init_msgs(sndev);
1506 
1507     rc = switchtec_ntb_init_shared_mw(sndev);
1508     if (rc)
1509         goto deinit_crosslink;
1510 
1511     rc = switchtec_ntb_init_db_msg_irq(sndev);
1512     if (rc)
1513         goto deinit_shared_and_exit;
1514 
1515     /*
1516      * If this host crashed, the other host may think the link is
1517      * still up. Tell them to force it down (it will go back up
1518      * once we register the ntb device).
1519      */
1520     switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_FORCE_DOWN);
1521 
1522     rc = ntb_register_device(&sndev->ntb);
1523     if (rc)
1524         goto deinit_and_exit;
1525 
1526     stdev->sndev = sndev;
1527     stdev->link_notifier = switchtec_ntb_link_notification;
1528     dev_info(dev, "NTB device registered\n");
1529 
1530     return 0;
1531 
1532 deinit_and_exit:
1533     switchtec_ntb_deinit_db_msg_irq(sndev);
1534 deinit_shared_and_exit:
1535     switchtec_ntb_deinit_shared_mw(sndev);
1536 deinit_crosslink:
1537     switchtec_ntb_deinit_crosslink(sndev);
1538 free_and_exit:
1539     kfree(sndev);
1540     dev_err(dev, "failed to register ntb device: %d\n", rc);
1541     return rc;
1542 }
1543 
1544 static void switchtec_ntb_remove(struct device *dev,
1545                  struct class_interface *class_intf)
1546 {
1547     struct switchtec_dev *stdev = to_stdev(dev);
1548     struct switchtec_ntb *sndev = stdev->sndev;
1549 
1550     if (!sndev)
1551         return;
1552 
1553     stdev->link_notifier = NULL;
1554     stdev->sndev = NULL;
1555     ntb_unregister_device(&sndev->ntb);
1556     switchtec_ntb_deinit_db_msg_irq(sndev);
1557     switchtec_ntb_deinit_shared_mw(sndev);
1558     switchtec_ntb_deinit_crosslink(sndev);
1559     kfree(sndev);
1560     dev_info(dev, "ntb device unregistered\n");
1561 }
1562 
1563 static struct class_interface switchtec_interface  = {
1564     .add_dev = switchtec_ntb_add,
1565     .remove_dev = switchtec_ntb_remove,
1566 };
1567 
1568 static int __init switchtec_ntb_init(void)
1569 {
1570     switchtec_interface.class = switchtec_class;
1571     return class_interface_register(&switchtec_interface);
1572 }
1573 module_init(switchtec_ntb_init);
1574 
1575 static void __exit switchtec_ntb_exit(void)
1576 {
1577     class_interface_unregister(&switchtec_interface);
1578 }
1579 module_exit(switchtec_ntb_exit);
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!