Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
0002 /* Copyright (c) 2015-2018 Mellanox Technologies. All rights reserved */
0003 
0004 #include <linux/kernel.h>
0005 #include <linux/module.h>
0006 #include <linux/export.h>
0007 #include <linux/err.h>
0008 #include <linux/device.h>
0009 #include <linux/pci.h>
0010 #include <linux/interrupt.h>
0011 #include <linux/wait.h>
0012 #include <linux/types.h>
0013 #include <linux/skbuff.h>
0014 #include <linux/if_vlan.h>
0015 #include <linux/log2.h>
0016 #include <linux/string.h>
0017 
0018 #include "pci_hw.h"
0019 #include "pci.h"
0020 #include "core.h"
0021 #include "cmd.h"
0022 #include "port.h"
0023 #include "resources.h"
0024 
0025 #define mlxsw_pci_write32(mlxsw_pci, reg, val) \
0026     iowrite32be(val, (mlxsw_pci)->hw_addr + (MLXSW_PCI_ ## reg))
0027 #define mlxsw_pci_read32(mlxsw_pci, reg) \
0028     ioread32be((mlxsw_pci)->hw_addr + (MLXSW_PCI_ ## reg))
0029 
0030 enum mlxsw_pci_queue_type {
0031     MLXSW_PCI_QUEUE_TYPE_SDQ,
0032     MLXSW_PCI_QUEUE_TYPE_RDQ,
0033     MLXSW_PCI_QUEUE_TYPE_CQ,
0034     MLXSW_PCI_QUEUE_TYPE_EQ,
0035 };
0036 
0037 #define MLXSW_PCI_QUEUE_TYPE_COUNT  4
0038 
0039 static const u16 mlxsw_pci_doorbell_type_offset[] = {
0040     MLXSW_PCI_DOORBELL_SDQ_OFFSET,  /* for type MLXSW_PCI_QUEUE_TYPE_SDQ */
0041     MLXSW_PCI_DOORBELL_RDQ_OFFSET,  /* for type MLXSW_PCI_QUEUE_TYPE_RDQ */
0042     MLXSW_PCI_DOORBELL_CQ_OFFSET,   /* for type MLXSW_PCI_QUEUE_TYPE_CQ */
0043     MLXSW_PCI_DOORBELL_EQ_OFFSET,   /* for type MLXSW_PCI_QUEUE_TYPE_EQ */
0044 };
0045 
0046 static const u16 mlxsw_pci_doorbell_arm_type_offset[] = {
0047     0, /* unused */
0048     0, /* unused */
0049     MLXSW_PCI_DOORBELL_ARM_CQ_OFFSET, /* for type MLXSW_PCI_QUEUE_TYPE_CQ */
0050     MLXSW_PCI_DOORBELL_ARM_EQ_OFFSET, /* for type MLXSW_PCI_QUEUE_TYPE_EQ */
0051 };
0052 
0053 struct mlxsw_pci_mem_item {
0054     char *buf;
0055     dma_addr_t mapaddr;
0056     size_t size;
0057 };
0058 
0059 struct mlxsw_pci_queue_elem_info {
0060     char *elem; /* pointer to actual dma mapped element mem chunk */
0061     union {
0062         struct {
0063             struct sk_buff *skb;
0064         } sdq;
0065         struct {
0066             struct sk_buff *skb;
0067         } rdq;
0068     } u;
0069 };
0070 
0071 struct mlxsw_pci_queue {
0072     spinlock_t lock; /* for queue accesses */
0073     struct mlxsw_pci_mem_item mem_item;
0074     struct mlxsw_pci_queue_elem_info *elem_info;
0075     u16 producer_counter;
0076     u16 consumer_counter;
0077     u16 count; /* number of elements in queue */
0078     u8 num; /* queue number */
0079     u8 elem_size; /* size of one element */
0080     enum mlxsw_pci_queue_type type;
0081     struct tasklet_struct tasklet; /* queue processing tasklet */
0082     struct mlxsw_pci *pci;
0083     union {
0084         struct {
0085             u32 comp_sdq_count;
0086             u32 comp_rdq_count;
0087             enum mlxsw_pci_cqe_v v;
0088         } cq;
0089         struct {
0090             u32 ev_cmd_count;
0091             u32 ev_comp_count;
0092             u32 ev_other_count;
0093         } eq;
0094     } u;
0095 };
0096 
0097 struct mlxsw_pci_queue_type_group {
0098     struct mlxsw_pci_queue *q;
0099     u8 count; /* number of queues in group */
0100 };
0101 
0102 struct mlxsw_pci {
0103     struct pci_dev *pdev;
0104     u8 __iomem *hw_addr;
0105     u64 free_running_clock_offset;
0106     u64 utc_sec_offset;
0107     u64 utc_nsec_offset;
0108     struct mlxsw_pci_queue_type_group queues[MLXSW_PCI_QUEUE_TYPE_COUNT];
0109     u32 doorbell_offset;
0110     struct mlxsw_core *core;
0111     struct {
0112         struct mlxsw_pci_mem_item *items;
0113         unsigned int count;
0114     } fw_area;
0115     struct {
0116         struct mlxsw_pci_mem_item out_mbox;
0117         struct mlxsw_pci_mem_item in_mbox;
0118         struct mutex lock; /* Lock access to command registers */
0119         bool nopoll;
0120         wait_queue_head_t wait;
0121         bool wait_done;
0122         struct {
0123             u8 status;
0124             u64 out_param;
0125         } comp;
0126     } cmd;
0127     struct mlxsw_bus_info bus_info;
0128     const struct pci_device_id *id;
0129     enum mlxsw_pci_cqe_v max_cqe_ver; /* Maximal supported CQE version */
0130     u8 num_sdq_cqs; /* Number of CQs used for SDQs */
0131 };
0132 
0133 static void mlxsw_pci_queue_tasklet_schedule(struct mlxsw_pci_queue *q)
0134 {
0135     tasklet_schedule(&q->tasklet);
0136 }
0137 
0138 static char *__mlxsw_pci_queue_elem_get(struct mlxsw_pci_queue *q,
0139                     size_t elem_size, int elem_index)
0140 {
0141     return q->mem_item.buf + (elem_size * elem_index);
0142 }
0143 
0144 static struct mlxsw_pci_queue_elem_info *
0145 mlxsw_pci_queue_elem_info_get(struct mlxsw_pci_queue *q, int elem_index)
0146 {
0147     return &q->elem_info[elem_index];
0148 }
0149 
0150 static struct mlxsw_pci_queue_elem_info *
0151 mlxsw_pci_queue_elem_info_producer_get(struct mlxsw_pci_queue *q)
0152 {
0153     int index = q->producer_counter & (q->count - 1);
0154 
0155     if ((u16) (q->producer_counter - q->consumer_counter) == q->count)
0156         return NULL;
0157     return mlxsw_pci_queue_elem_info_get(q, index);
0158 }
0159 
0160 static struct mlxsw_pci_queue_elem_info *
0161 mlxsw_pci_queue_elem_info_consumer_get(struct mlxsw_pci_queue *q)
0162 {
0163     int index = q->consumer_counter & (q->count - 1);
0164 
0165     return mlxsw_pci_queue_elem_info_get(q, index);
0166 }
0167 
0168 static char *mlxsw_pci_queue_elem_get(struct mlxsw_pci_queue *q, int elem_index)
0169 {
0170     return mlxsw_pci_queue_elem_info_get(q, elem_index)->elem;
0171 }
0172 
0173 static bool mlxsw_pci_elem_hw_owned(struct mlxsw_pci_queue *q, bool owner_bit)
0174 {
0175     return owner_bit != !!(q->consumer_counter & q->count);
0176 }
0177 
0178 static struct mlxsw_pci_queue_type_group *
0179 mlxsw_pci_queue_type_group_get(struct mlxsw_pci *mlxsw_pci,
0180                    enum mlxsw_pci_queue_type q_type)
0181 {
0182     return &mlxsw_pci->queues[q_type];
0183 }
0184 
0185 static u8 __mlxsw_pci_queue_count(struct mlxsw_pci *mlxsw_pci,
0186                   enum mlxsw_pci_queue_type q_type)
0187 {
0188     struct mlxsw_pci_queue_type_group *queue_group;
0189 
0190     queue_group = mlxsw_pci_queue_type_group_get(mlxsw_pci, q_type);
0191     return queue_group->count;
0192 }
0193 
0194 static u8 mlxsw_pci_sdq_count(struct mlxsw_pci *mlxsw_pci)
0195 {
0196     return __mlxsw_pci_queue_count(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_SDQ);
0197 }
0198 
0199 static u8 mlxsw_pci_cq_count(struct mlxsw_pci *mlxsw_pci)
0200 {
0201     return __mlxsw_pci_queue_count(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_CQ);
0202 }
0203 
0204 static struct mlxsw_pci_queue *
0205 __mlxsw_pci_queue_get(struct mlxsw_pci *mlxsw_pci,
0206               enum mlxsw_pci_queue_type q_type, u8 q_num)
0207 {
0208     return &mlxsw_pci->queues[q_type].q[q_num];
0209 }
0210 
0211 static struct mlxsw_pci_queue *mlxsw_pci_sdq_get(struct mlxsw_pci *mlxsw_pci,
0212                          u8 q_num)
0213 {
0214     return __mlxsw_pci_queue_get(mlxsw_pci,
0215                      MLXSW_PCI_QUEUE_TYPE_SDQ, q_num);
0216 }
0217 
0218 static struct mlxsw_pci_queue *mlxsw_pci_rdq_get(struct mlxsw_pci *mlxsw_pci,
0219                          u8 q_num)
0220 {
0221     return __mlxsw_pci_queue_get(mlxsw_pci,
0222                      MLXSW_PCI_QUEUE_TYPE_RDQ, q_num);
0223 }
0224 
0225 static struct mlxsw_pci_queue *mlxsw_pci_cq_get(struct mlxsw_pci *mlxsw_pci,
0226                         u8 q_num)
0227 {
0228     return __mlxsw_pci_queue_get(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_CQ, q_num);
0229 }
0230 
0231 static struct mlxsw_pci_queue *mlxsw_pci_eq_get(struct mlxsw_pci *mlxsw_pci,
0232                         u8 q_num)
0233 {
0234     return __mlxsw_pci_queue_get(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_EQ, q_num);
0235 }
0236 
0237 static void __mlxsw_pci_queue_doorbell_set(struct mlxsw_pci *mlxsw_pci,
0238                        struct mlxsw_pci_queue *q,
0239                        u16 val)
0240 {
0241     mlxsw_pci_write32(mlxsw_pci,
0242               DOORBELL(mlxsw_pci->doorbell_offset,
0243                    mlxsw_pci_doorbell_type_offset[q->type],
0244                    q->num), val);
0245 }
0246 
0247 static void __mlxsw_pci_queue_doorbell_arm_set(struct mlxsw_pci *mlxsw_pci,
0248                            struct mlxsw_pci_queue *q,
0249                            u16 val)
0250 {
0251     mlxsw_pci_write32(mlxsw_pci,
0252               DOORBELL(mlxsw_pci->doorbell_offset,
0253                    mlxsw_pci_doorbell_arm_type_offset[q->type],
0254                    q->num), val);
0255 }
0256 
0257 static void mlxsw_pci_queue_doorbell_producer_ring(struct mlxsw_pci *mlxsw_pci,
0258                            struct mlxsw_pci_queue *q)
0259 {
0260     wmb(); /* ensure all writes are done before we ring a bell */
0261     __mlxsw_pci_queue_doorbell_set(mlxsw_pci, q, q->producer_counter);
0262 }
0263 
0264 static void mlxsw_pci_queue_doorbell_consumer_ring(struct mlxsw_pci *mlxsw_pci,
0265                            struct mlxsw_pci_queue *q)
0266 {
0267     wmb(); /* ensure all writes are done before we ring a bell */
0268     __mlxsw_pci_queue_doorbell_set(mlxsw_pci, q,
0269                        q->consumer_counter + q->count);
0270 }
0271 
0272 static void
0273 mlxsw_pci_queue_doorbell_arm_consumer_ring(struct mlxsw_pci *mlxsw_pci,
0274                        struct mlxsw_pci_queue *q)
0275 {
0276     wmb(); /* ensure all writes are done before we ring a bell */
0277     __mlxsw_pci_queue_doorbell_arm_set(mlxsw_pci, q, q->consumer_counter);
0278 }
0279 
0280 static dma_addr_t __mlxsw_pci_queue_page_get(struct mlxsw_pci_queue *q,
0281                          int page_index)
0282 {
0283     return q->mem_item.mapaddr + MLXSW_PCI_PAGE_SIZE * page_index;
0284 }
0285 
0286 static int mlxsw_pci_sdq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
0287                   struct mlxsw_pci_queue *q)
0288 {
0289     int tclass;
0290     int lp;
0291     int i;
0292     int err;
0293 
0294     q->producer_counter = 0;
0295     q->consumer_counter = 0;
0296     tclass = q->num == MLXSW_PCI_SDQ_EMAD_INDEX ? MLXSW_PCI_SDQ_EMAD_TC :
0297                               MLXSW_PCI_SDQ_CTL_TC;
0298     lp = q->num == MLXSW_PCI_SDQ_EMAD_INDEX ? MLXSW_CMD_MBOX_SW2HW_DQ_SDQ_LP_IGNORE_WQE :
0299                           MLXSW_CMD_MBOX_SW2HW_DQ_SDQ_LP_WQE;
0300 
0301     /* Set CQ of same number of this SDQ. */
0302     mlxsw_cmd_mbox_sw2hw_dq_cq_set(mbox, q->num);
0303     mlxsw_cmd_mbox_sw2hw_dq_sdq_lp_set(mbox, lp);
0304     mlxsw_cmd_mbox_sw2hw_dq_sdq_tclass_set(mbox, tclass);
0305     mlxsw_cmd_mbox_sw2hw_dq_log2_dq_sz_set(mbox, 3); /* 8 pages */
0306     for (i = 0; i < MLXSW_PCI_AQ_PAGES; i++) {
0307         dma_addr_t mapaddr = __mlxsw_pci_queue_page_get(q, i);
0308 
0309         mlxsw_cmd_mbox_sw2hw_dq_pa_set(mbox, i, mapaddr);
0310     }
0311 
0312     err = mlxsw_cmd_sw2hw_sdq(mlxsw_pci->core, mbox, q->num);
0313     if (err)
0314         return err;
0315     mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q);
0316     return 0;
0317 }
0318 
0319 static void mlxsw_pci_sdq_fini(struct mlxsw_pci *mlxsw_pci,
0320                    struct mlxsw_pci_queue *q)
0321 {
0322     mlxsw_cmd_hw2sw_sdq(mlxsw_pci->core, q->num);
0323 }
0324 
0325 static int mlxsw_pci_wqe_frag_map(struct mlxsw_pci *mlxsw_pci, char *wqe,
0326                   int index, char *frag_data, size_t frag_len,
0327                   int direction)
0328 {
0329     struct pci_dev *pdev = mlxsw_pci->pdev;
0330     dma_addr_t mapaddr;
0331 
0332     mapaddr = dma_map_single(&pdev->dev, frag_data, frag_len, direction);
0333     if (unlikely(dma_mapping_error(&pdev->dev, mapaddr))) {
0334         dev_err_ratelimited(&pdev->dev, "failed to dma map tx frag\n");
0335         return -EIO;
0336     }
0337     mlxsw_pci_wqe_address_set(wqe, index, mapaddr);
0338     mlxsw_pci_wqe_byte_count_set(wqe, index, frag_len);
0339     return 0;
0340 }
0341 
0342 static void mlxsw_pci_wqe_frag_unmap(struct mlxsw_pci *mlxsw_pci, char *wqe,
0343                      int index, int direction)
0344 {
0345     struct pci_dev *pdev = mlxsw_pci->pdev;
0346     size_t frag_len = mlxsw_pci_wqe_byte_count_get(wqe, index);
0347     dma_addr_t mapaddr = mlxsw_pci_wqe_address_get(wqe, index);
0348 
0349     if (!frag_len)
0350         return;
0351     dma_unmap_single(&pdev->dev, mapaddr, frag_len, direction);
0352 }
0353 
0354 static int mlxsw_pci_rdq_skb_alloc(struct mlxsw_pci *mlxsw_pci,
0355                    struct mlxsw_pci_queue_elem_info *elem_info)
0356 {
0357     size_t buf_len = MLXSW_PORT_MAX_MTU;
0358     char *wqe = elem_info->elem;
0359     struct sk_buff *skb;
0360     int err;
0361 
0362     skb = netdev_alloc_skb_ip_align(NULL, buf_len);
0363     if (!skb)
0364         return -ENOMEM;
0365 
0366     err = mlxsw_pci_wqe_frag_map(mlxsw_pci, wqe, 0, skb->data,
0367                      buf_len, DMA_FROM_DEVICE);
0368     if (err)
0369         goto err_frag_map;
0370 
0371     elem_info->u.rdq.skb = skb;
0372     return 0;
0373 
0374 err_frag_map:
0375     dev_kfree_skb_any(skb);
0376     return err;
0377 }
0378 
0379 static void mlxsw_pci_rdq_skb_free(struct mlxsw_pci *mlxsw_pci,
0380                    struct mlxsw_pci_queue_elem_info *elem_info)
0381 {
0382     struct sk_buff *skb;
0383     char *wqe;
0384 
0385     skb = elem_info->u.rdq.skb;
0386     wqe = elem_info->elem;
0387 
0388     mlxsw_pci_wqe_frag_unmap(mlxsw_pci, wqe, 0, DMA_FROM_DEVICE);
0389     dev_kfree_skb_any(skb);
0390 }
0391 
0392 static int mlxsw_pci_rdq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
0393                   struct mlxsw_pci_queue *q)
0394 {
0395     struct mlxsw_pci_queue_elem_info *elem_info;
0396     u8 sdq_count = mlxsw_pci_sdq_count(mlxsw_pci);
0397     int i;
0398     int err;
0399 
0400     q->producer_counter = 0;
0401     q->consumer_counter = 0;
0402 
0403     /* Set CQ of same number of this RDQ with base
0404      * above SDQ count as the lower ones are assigned to SDQs.
0405      */
0406     mlxsw_cmd_mbox_sw2hw_dq_cq_set(mbox, sdq_count + q->num);
0407     mlxsw_cmd_mbox_sw2hw_dq_log2_dq_sz_set(mbox, 3); /* 8 pages */
0408     for (i = 0; i < MLXSW_PCI_AQ_PAGES; i++) {
0409         dma_addr_t mapaddr = __mlxsw_pci_queue_page_get(q, i);
0410 
0411         mlxsw_cmd_mbox_sw2hw_dq_pa_set(mbox, i, mapaddr);
0412     }
0413 
0414     err = mlxsw_cmd_sw2hw_rdq(mlxsw_pci->core, mbox, q->num);
0415     if (err)
0416         return err;
0417 
0418     mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q);
0419 
0420     for (i = 0; i < q->count; i++) {
0421         elem_info = mlxsw_pci_queue_elem_info_producer_get(q);
0422         BUG_ON(!elem_info);
0423         err = mlxsw_pci_rdq_skb_alloc(mlxsw_pci, elem_info);
0424         if (err)
0425             goto rollback;
0426         /* Everything is set up, ring doorbell to pass elem to HW */
0427         q->producer_counter++;
0428         mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q);
0429     }
0430 
0431     return 0;
0432 
0433 rollback:
0434     for (i--; i >= 0; i--) {
0435         elem_info = mlxsw_pci_queue_elem_info_get(q, i);
0436         mlxsw_pci_rdq_skb_free(mlxsw_pci, elem_info);
0437     }
0438     mlxsw_cmd_hw2sw_rdq(mlxsw_pci->core, q->num);
0439 
0440     return err;
0441 }
0442 
0443 static void mlxsw_pci_rdq_fini(struct mlxsw_pci *mlxsw_pci,
0444                    struct mlxsw_pci_queue *q)
0445 {
0446     struct mlxsw_pci_queue_elem_info *elem_info;
0447     int i;
0448 
0449     mlxsw_cmd_hw2sw_rdq(mlxsw_pci->core, q->num);
0450     for (i = 0; i < q->count; i++) {
0451         elem_info = mlxsw_pci_queue_elem_info_get(q, i);
0452         mlxsw_pci_rdq_skb_free(mlxsw_pci, elem_info);
0453     }
0454 }
0455 
0456 static void mlxsw_pci_cq_pre_init(struct mlxsw_pci *mlxsw_pci,
0457                   struct mlxsw_pci_queue *q)
0458 {
0459     q->u.cq.v = mlxsw_pci->max_cqe_ver;
0460 
0461     if (q->u.cq.v == MLXSW_PCI_CQE_V2 &&
0462         q->num < mlxsw_pci->num_sdq_cqs &&
0463         !mlxsw_core_sdq_supports_cqe_v2(mlxsw_pci->core))
0464         q->u.cq.v = MLXSW_PCI_CQE_V1;
0465 }
0466 
0467 static int mlxsw_pci_cq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
0468                  struct mlxsw_pci_queue *q)
0469 {
0470     int i;
0471     int err;
0472 
0473     q->consumer_counter = 0;
0474 
0475     for (i = 0; i < q->count; i++) {
0476         char *elem = mlxsw_pci_queue_elem_get(q, i);
0477 
0478         mlxsw_pci_cqe_owner_set(q->u.cq.v, elem, 1);
0479     }
0480 
0481     if (q->u.cq.v == MLXSW_PCI_CQE_V1)
0482         mlxsw_cmd_mbox_sw2hw_cq_cqe_ver_set(mbox,
0483                 MLXSW_CMD_MBOX_SW2HW_CQ_CQE_VER_1);
0484     else if (q->u.cq.v == MLXSW_PCI_CQE_V2)
0485         mlxsw_cmd_mbox_sw2hw_cq_cqe_ver_set(mbox,
0486                 MLXSW_CMD_MBOX_SW2HW_CQ_CQE_VER_2);
0487 
0488     mlxsw_cmd_mbox_sw2hw_cq_c_eqn_set(mbox, MLXSW_PCI_EQ_COMP_NUM);
0489     mlxsw_cmd_mbox_sw2hw_cq_st_set(mbox, 0);
0490     mlxsw_cmd_mbox_sw2hw_cq_log_cq_size_set(mbox, ilog2(q->count));
0491     for (i = 0; i < MLXSW_PCI_AQ_PAGES; i++) {
0492         dma_addr_t mapaddr = __mlxsw_pci_queue_page_get(q, i);
0493 
0494         mlxsw_cmd_mbox_sw2hw_cq_pa_set(mbox, i, mapaddr);
0495     }
0496     err = mlxsw_cmd_sw2hw_cq(mlxsw_pci->core, mbox, q->num);
0497     if (err)
0498         return err;
0499     mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q);
0500     mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q);
0501     return 0;
0502 }
0503 
0504 static void mlxsw_pci_cq_fini(struct mlxsw_pci *mlxsw_pci,
0505                   struct mlxsw_pci_queue *q)
0506 {
0507     mlxsw_cmd_hw2sw_cq(mlxsw_pci->core, q->num);
0508 }
0509 
0510 static unsigned int mlxsw_pci_read32_off(struct mlxsw_pci *mlxsw_pci,
0511                      ptrdiff_t off)
0512 {
0513     return ioread32be(mlxsw_pci->hw_addr + off);
0514 }
0515 
0516 static void mlxsw_pci_skb_cb_ts_set(struct mlxsw_pci *mlxsw_pci,
0517                     struct sk_buff *skb,
0518                     enum mlxsw_pci_cqe_v cqe_v, char *cqe)
0519 {
0520     if (cqe_v != MLXSW_PCI_CQE_V2)
0521         return;
0522 
0523     if (mlxsw_pci_cqe2_time_stamp_type_get(cqe) !=
0524         MLXSW_PCI_CQE_TIME_STAMP_TYPE_UTC)
0525         return;
0526 
0527     mlxsw_skb_cb(skb)->cqe_ts.sec = mlxsw_pci_cqe2_time_stamp_sec_get(cqe);
0528     mlxsw_skb_cb(skb)->cqe_ts.nsec =
0529         mlxsw_pci_cqe2_time_stamp_nsec_get(cqe);
0530 }
0531 
0532 static void mlxsw_pci_cqe_sdq_handle(struct mlxsw_pci *mlxsw_pci,
0533                      struct mlxsw_pci_queue *q,
0534                      u16 consumer_counter_limit,
0535                      enum mlxsw_pci_cqe_v cqe_v,
0536                      char *cqe)
0537 {
0538     struct pci_dev *pdev = mlxsw_pci->pdev;
0539     struct mlxsw_pci_queue_elem_info *elem_info;
0540     struct mlxsw_tx_info tx_info;
0541     char *wqe;
0542     struct sk_buff *skb;
0543     int i;
0544 
0545     spin_lock(&q->lock);
0546     elem_info = mlxsw_pci_queue_elem_info_consumer_get(q);
0547     tx_info = mlxsw_skb_cb(elem_info->u.sdq.skb)->tx_info;
0548     skb = elem_info->u.sdq.skb;
0549     wqe = elem_info->elem;
0550     for (i = 0; i < MLXSW_PCI_WQE_SG_ENTRIES; i++)
0551         mlxsw_pci_wqe_frag_unmap(mlxsw_pci, wqe, i, DMA_TO_DEVICE);
0552 
0553     if (unlikely(!tx_info.is_emad &&
0554              skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
0555         mlxsw_pci_skb_cb_ts_set(mlxsw_pci, skb, cqe_v, cqe);
0556         mlxsw_core_ptp_transmitted(mlxsw_pci->core, skb,
0557                        tx_info.local_port);
0558         skb = NULL;
0559     }
0560 
0561     if (skb)
0562         dev_kfree_skb_any(skb);
0563     elem_info->u.sdq.skb = NULL;
0564 
0565     if (q->consumer_counter++ != consumer_counter_limit)
0566         dev_dbg_ratelimited(&pdev->dev, "Consumer counter does not match limit in SDQ\n");
0567     spin_unlock(&q->lock);
0568 }
0569 
0570 static void mlxsw_pci_cqe_rdq_md_tx_port_init(struct sk_buff *skb,
0571                           const char *cqe)
0572 {
0573     struct mlxsw_skb_cb *cb = mlxsw_skb_cb(skb);
0574 
0575     if (mlxsw_pci_cqe2_tx_lag_get(cqe)) {
0576         cb->rx_md_info.tx_port_is_lag = true;
0577         cb->rx_md_info.tx_lag_id = mlxsw_pci_cqe2_tx_lag_id_get(cqe);
0578         cb->rx_md_info.tx_lag_port_index =
0579             mlxsw_pci_cqe2_tx_lag_subport_get(cqe);
0580     } else {
0581         cb->rx_md_info.tx_port_is_lag = false;
0582         cb->rx_md_info.tx_sys_port =
0583             mlxsw_pci_cqe2_tx_system_port_get(cqe);
0584     }
0585 
0586     if (cb->rx_md_info.tx_sys_port != MLXSW_PCI_CQE2_TX_PORT_MULTI_PORT &&
0587         cb->rx_md_info.tx_sys_port != MLXSW_PCI_CQE2_TX_PORT_INVALID)
0588         cb->rx_md_info.tx_port_valid = 1;
0589     else
0590         cb->rx_md_info.tx_port_valid = 0;
0591 }
0592 
0593 static void mlxsw_pci_cqe_rdq_md_init(struct sk_buff *skb, const char *cqe)
0594 {
0595     struct mlxsw_skb_cb *cb = mlxsw_skb_cb(skb);
0596 
0597     cb->rx_md_info.tx_congestion = mlxsw_pci_cqe2_mirror_cong_get(cqe);
0598     if (cb->rx_md_info.tx_congestion != MLXSW_PCI_CQE2_MIRROR_CONG_INVALID)
0599         cb->rx_md_info.tx_congestion_valid = 1;
0600     else
0601         cb->rx_md_info.tx_congestion_valid = 0;
0602     cb->rx_md_info.tx_congestion <<= MLXSW_PCI_CQE2_MIRROR_CONG_SHIFT;
0603 
0604     cb->rx_md_info.latency = mlxsw_pci_cqe2_mirror_latency_get(cqe);
0605     if (cb->rx_md_info.latency != MLXSW_PCI_CQE2_MIRROR_LATENCY_INVALID)
0606         cb->rx_md_info.latency_valid = 1;
0607     else
0608         cb->rx_md_info.latency_valid = 0;
0609 
0610     cb->rx_md_info.tx_tc = mlxsw_pci_cqe2_mirror_tclass_get(cqe);
0611     if (cb->rx_md_info.tx_tc != MLXSW_PCI_CQE2_MIRROR_TCLASS_INVALID)
0612         cb->rx_md_info.tx_tc_valid = 1;
0613     else
0614         cb->rx_md_info.tx_tc_valid = 0;
0615 
0616     mlxsw_pci_cqe_rdq_md_tx_port_init(skb, cqe);
0617 }
0618 
0619 static void mlxsw_pci_cqe_rdq_handle(struct mlxsw_pci *mlxsw_pci,
0620                      struct mlxsw_pci_queue *q,
0621                      u16 consumer_counter_limit,
0622                      enum mlxsw_pci_cqe_v cqe_v, char *cqe)
0623 {
0624     struct pci_dev *pdev = mlxsw_pci->pdev;
0625     struct mlxsw_pci_queue_elem_info *elem_info;
0626     struct mlxsw_rx_info rx_info = {};
0627     char wqe[MLXSW_PCI_WQE_SIZE];
0628     struct sk_buff *skb;
0629     u16 byte_count;
0630     int err;
0631 
0632     elem_info = mlxsw_pci_queue_elem_info_consumer_get(q);
0633     skb = elem_info->u.rdq.skb;
0634     memcpy(wqe, elem_info->elem, MLXSW_PCI_WQE_SIZE);
0635 
0636     if (q->consumer_counter++ != consumer_counter_limit)
0637         dev_dbg_ratelimited(&pdev->dev, "Consumer counter does not match limit in RDQ\n");
0638 
0639     err = mlxsw_pci_rdq_skb_alloc(mlxsw_pci, elem_info);
0640     if (err) {
0641         dev_err_ratelimited(&pdev->dev, "Failed to alloc skb for RDQ\n");
0642         goto out;
0643     }
0644 
0645     mlxsw_pci_wqe_frag_unmap(mlxsw_pci, wqe, 0, DMA_FROM_DEVICE);
0646 
0647     if (mlxsw_pci_cqe_lag_get(cqe_v, cqe)) {
0648         rx_info.is_lag = true;
0649         rx_info.u.lag_id = mlxsw_pci_cqe_lag_id_get(cqe_v, cqe);
0650         rx_info.lag_port_index =
0651             mlxsw_pci_cqe_lag_subport_get(cqe_v, cqe);
0652     } else {
0653         rx_info.is_lag = false;
0654         rx_info.u.sys_port = mlxsw_pci_cqe_system_port_get(cqe);
0655     }
0656 
0657     rx_info.trap_id = mlxsw_pci_cqe_trap_id_get(cqe);
0658 
0659     if (rx_info.trap_id == MLXSW_TRAP_ID_DISCARD_INGRESS_ACL ||
0660         rx_info.trap_id == MLXSW_TRAP_ID_DISCARD_EGRESS_ACL) {
0661         u32 cookie_index = 0;
0662 
0663         if (mlxsw_pci->max_cqe_ver >= MLXSW_PCI_CQE_V2)
0664             cookie_index = mlxsw_pci_cqe2_user_def_val_orig_pkt_len_get(cqe);
0665         mlxsw_skb_cb(skb)->rx_md_info.cookie_index = cookie_index;
0666     } else if (rx_info.trap_id >= MLXSW_TRAP_ID_MIRROR_SESSION0 &&
0667            rx_info.trap_id <= MLXSW_TRAP_ID_MIRROR_SESSION7 &&
0668            mlxsw_pci->max_cqe_ver >= MLXSW_PCI_CQE_V2) {
0669         rx_info.mirror_reason = mlxsw_pci_cqe2_mirror_reason_get(cqe);
0670         mlxsw_pci_cqe_rdq_md_init(skb, cqe);
0671     } else if (rx_info.trap_id == MLXSW_TRAP_ID_PKT_SAMPLE &&
0672            mlxsw_pci->max_cqe_ver >= MLXSW_PCI_CQE_V2) {
0673         mlxsw_pci_cqe_rdq_md_tx_port_init(skb, cqe);
0674     }
0675 
0676     mlxsw_pci_skb_cb_ts_set(mlxsw_pci, skb, cqe_v, cqe);
0677 
0678     byte_count = mlxsw_pci_cqe_byte_count_get(cqe);
0679     if (mlxsw_pci_cqe_crc_get(cqe_v, cqe))
0680         byte_count -= ETH_FCS_LEN;
0681     skb_put(skb, byte_count);
0682     mlxsw_core_skb_receive(mlxsw_pci->core, skb, &rx_info);
0683 
0684 out:
0685     /* Everything is set up, ring doorbell to pass elem to HW */
0686     q->producer_counter++;
0687     mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q);
0688     return;
0689 }
0690 
0691 static char *mlxsw_pci_cq_sw_cqe_get(struct mlxsw_pci_queue *q)
0692 {
0693     struct mlxsw_pci_queue_elem_info *elem_info;
0694     char *elem;
0695     bool owner_bit;
0696 
0697     elem_info = mlxsw_pci_queue_elem_info_consumer_get(q);
0698     elem = elem_info->elem;
0699     owner_bit = mlxsw_pci_cqe_owner_get(q->u.cq.v, elem);
0700     if (mlxsw_pci_elem_hw_owned(q, owner_bit))
0701         return NULL;
0702     q->consumer_counter++;
0703     rmb(); /* make sure we read owned bit before the rest of elem */
0704     return elem;
0705 }
0706 
0707 static void mlxsw_pci_cq_tasklet(struct tasklet_struct *t)
0708 {
0709     struct mlxsw_pci_queue *q = from_tasklet(q, t, tasklet);
0710     struct mlxsw_pci *mlxsw_pci = q->pci;
0711     char *cqe;
0712     int items = 0;
0713     int credits = q->count >> 1;
0714 
0715     while ((cqe = mlxsw_pci_cq_sw_cqe_get(q))) {
0716         u16 wqe_counter = mlxsw_pci_cqe_wqe_counter_get(cqe);
0717         u8 sendq = mlxsw_pci_cqe_sr_get(q->u.cq.v, cqe);
0718         u8 dqn = mlxsw_pci_cqe_dqn_get(q->u.cq.v, cqe);
0719         char ncqe[MLXSW_PCI_CQE_SIZE_MAX];
0720 
0721         memcpy(ncqe, cqe, q->elem_size);
0722         mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q);
0723 
0724         if (sendq) {
0725             struct mlxsw_pci_queue *sdq;
0726 
0727             sdq = mlxsw_pci_sdq_get(mlxsw_pci, dqn);
0728             mlxsw_pci_cqe_sdq_handle(mlxsw_pci, sdq,
0729                          wqe_counter, q->u.cq.v, ncqe);
0730             q->u.cq.comp_sdq_count++;
0731         } else {
0732             struct mlxsw_pci_queue *rdq;
0733 
0734             rdq = mlxsw_pci_rdq_get(mlxsw_pci, dqn);
0735             mlxsw_pci_cqe_rdq_handle(mlxsw_pci, rdq,
0736                          wqe_counter, q->u.cq.v, ncqe);
0737             q->u.cq.comp_rdq_count++;
0738         }
0739         if (++items == credits)
0740             break;
0741     }
0742     if (items)
0743         mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q);
0744 }
0745 
0746 static u16 mlxsw_pci_cq_elem_count(const struct mlxsw_pci_queue *q)
0747 {
0748     return q->u.cq.v == MLXSW_PCI_CQE_V2 ? MLXSW_PCI_CQE2_COUNT :
0749                            MLXSW_PCI_CQE01_COUNT;
0750 }
0751 
0752 static u8 mlxsw_pci_cq_elem_size(const struct mlxsw_pci_queue *q)
0753 {
0754     return q->u.cq.v == MLXSW_PCI_CQE_V2 ? MLXSW_PCI_CQE2_SIZE :
0755                            MLXSW_PCI_CQE01_SIZE;
0756 }
0757 
0758 static int mlxsw_pci_eq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
0759                  struct mlxsw_pci_queue *q)
0760 {
0761     int i;
0762     int err;
0763 
0764     q->consumer_counter = 0;
0765 
0766     for (i = 0; i < q->count; i++) {
0767         char *elem = mlxsw_pci_queue_elem_get(q, i);
0768 
0769         mlxsw_pci_eqe_owner_set(elem, 1);
0770     }
0771 
0772     mlxsw_cmd_mbox_sw2hw_eq_int_msix_set(mbox, 1); /* MSI-X used */
0773     mlxsw_cmd_mbox_sw2hw_eq_st_set(mbox, 1); /* armed */
0774     mlxsw_cmd_mbox_sw2hw_eq_log_eq_size_set(mbox, ilog2(q->count));
0775     for (i = 0; i < MLXSW_PCI_AQ_PAGES; i++) {
0776         dma_addr_t mapaddr = __mlxsw_pci_queue_page_get(q, i);
0777 
0778         mlxsw_cmd_mbox_sw2hw_eq_pa_set(mbox, i, mapaddr);
0779     }
0780     err = mlxsw_cmd_sw2hw_eq(mlxsw_pci->core, mbox, q->num);
0781     if (err)
0782         return err;
0783     mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q);
0784     mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q);
0785     return 0;
0786 }
0787 
0788 static void mlxsw_pci_eq_fini(struct mlxsw_pci *mlxsw_pci,
0789                   struct mlxsw_pci_queue *q)
0790 {
0791     mlxsw_cmd_hw2sw_eq(mlxsw_pci->core, q->num);
0792 }
0793 
0794 static void mlxsw_pci_eq_cmd_event(struct mlxsw_pci *mlxsw_pci, char *eqe)
0795 {
0796     mlxsw_pci->cmd.comp.status = mlxsw_pci_eqe_cmd_status_get(eqe);
0797     mlxsw_pci->cmd.comp.out_param =
0798         ((u64) mlxsw_pci_eqe_cmd_out_param_h_get(eqe)) << 32 |
0799         mlxsw_pci_eqe_cmd_out_param_l_get(eqe);
0800     mlxsw_pci->cmd.wait_done = true;
0801     wake_up(&mlxsw_pci->cmd.wait);
0802 }
0803 
0804 static char *mlxsw_pci_eq_sw_eqe_get(struct mlxsw_pci_queue *q)
0805 {
0806     struct mlxsw_pci_queue_elem_info *elem_info;
0807     char *elem;
0808     bool owner_bit;
0809 
0810     elem_info = mlxsw_pci_queue_elem_info_consumer_get(q);
0811     elem = elem_info->elem;
0812     owner_bit = mlxsw_pci_eqe_owner_get(elem);
0813     if (mlxsw_pci_elem_hw_owned(q, owner_bit))
0814         return NULL;
0815     q->consumer_counter++;
0816     rmb(); /* make sure we read owned bit before the rest of elem */
0817     return elem;
0818 }
0819 
0820 static void mlxsw_pci_eq_tasklet(struct tasklet_struct *t)
0821 {
0822     struct mlxsw_pci_queue *q = from_tasklet(q, t, tasklet);
0823     struct mlxsw_pci *mlxsw_pci = q->pci;
0824     u8 cq_count = mlxsw_pci_cq_count(mlxsw_pci);
0825     unsigned long active_cqns[BITS_TO_LONGS(MLXSW_PCI_CQS_MAX)];
0826     char *eqe;
0827     u8 cqn;
0828     bool cq_handle = false;
0829     int items = 0;
0830     int credits = q->count >> 1;
0831 
0832     memset(&active_cqns, 0, sizeof(active_cqns));
0833 
0834     while ((eqe = mlxsw_pci_eq_sw_eqe_get(q))) {
0835 
0836         /* Command interface completion events are always received on
0837          * queue MLXSW_PCI_EQ_ASYNC_NUM (EQ0) and completion events
0838          * are mapped to queue MLXSW_PCI_EQ_COMP_NUM (EQ1).
0839          */
0840         switch (q->num) {
0841         case MLXSW_PCI_EQ_ASYNC_NUM:
0842             mlxsw_pci_eq_cmd_event(mlxsw_pci, eqe);
0843             q->u.eq.ev_cmd_count++;
0844             break;
0845         case MLXSW_PCI_EQ_COMP_NUM:
0846             cqn = mlxsw_pci_eqe_cqn_get(eqe);
0847             set_bit(cqn, active_cqns);
0848             cq_handle = true;
0849             q->u.eq.ev_comp_count++;
0850             break;
0851         default:
0852             q->u.eq.ev_other_count++;
0853         }
0854         if (++items == credits)
0855             break;
0856     }
0857     if (items) {
0858         mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q);
0859         mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q);
0860     }
0861 
0862     if (!cq_handle)
0863         return;
0864     for_each_set_bit(cqn, active_cqns, cq_count) {
0865         q = mlxsw_pci_cq_get(mlxsw_pci, cqn);
0866         mlxsw_pci_queue_tasklet_schedule(q);
0867     }
0868 }
0869 
0870 struct mlxsw_pci_queue_ops {
0871     const char *name;
0872     enum mlxsw_pci_queue_type type;
0873     void (*pre_init)(struct mlxsw_pci *mlxsw_pci,
0874              struct mlxsw_pci_queue *q);
0875     int (*init)(struct mlxsw_pci *mlxsw_pci, char *mbox,
0876             struct mlxsw_pci_queue *q);
0877     void (*fini)(struct mlxsw_pci *mlxsw_pci,
0878              struct mlxsw_pci_queue *q);
0879     void (*tasklet)(struct tasklet_struct *t);
0880     u16 (*elem_count_f)(const struct mlxsw_pci_queue *q);
0881     u8 (*elem_size_f)(const struct mlxsw_pci_queue *q);
0882     u16 elem_count;
0883     u8 elem_size;
0884 };
0885 
0886 static const struct mlxsw_pci_queue_ops mlxsw_pci_sdq_ops = {
0887     .type       = MLXSW_PCI_QUEUE_TYPE_SDQ,
0888     .init       = mlxsw_pci_sdq_init,
0889     .fini       = mlxsw_pci_sdq_fini,
0890     .elem_count = MLXSW_PCI_WQE_COUNT,
0891     .elem_size  = MLXSW_PCI_WQE_SIZE,
0892 };
0893 
0894 static const struct mlxsw_pci_queue_ops mlxsw_pci_rdq_ops = {
0895     .type       = MLXSW_PCI_QUEUE_TYPE_RDQ,
0896     .init       = mlxsw_pci_rdq_init,
0897     .fini       = mlxsw_pci_rdq_fini,
0898     .elem_count = MLXSW_PCI_WQE_COUNT,
0899     .elem_size  = MLXSW_PCI_WQE_SIZE
0900 };
0901 
0902 static const struct mlxsw_pci_queue_ops mlxsw_pci_cq_ops = {
0903     .type       = MLXSW_PCI_QUEUE_TYPE_CQ,
0904     .pre_init   = mlxsw_pci_cq_pre_init,
0905     .init       = mlxsw_pci_cq_init,
0906     .fini       = mlxsw_pci_cq_fini,
0907     .tasklet    = mlxsw_pci_cq_tasklet,
0908     .elem_count_f   = mlxsw_pci_cq_elem_count,
0909     .elem_size_f    = mlxsw_pci_cq_elem_size
0910 };
0911 
0912 static const struct mlxsw_pci_queue_ops mlxsw_pci_eq_ops = {
0913     .type       = MLXSW_PCI_QUEUE_TYPE_EQ,
0914     .init       = mlxsw_pci_eq_init,
0915     .fini       = mlxsw_pci_eq_fini,
0916     .tasklet    = mlxsw_pci_eq_tasklet,
0917     .elem_count = MLXSW_PCI_EQE_COUNT,
0918     .elem_size  = MLXSW_PCI_EQE_SIZE
0919 };
0920 
0921 static int mlxsw_pci_queue_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
0922                 const struct mlxsw_pci_queue_ops *q_ops,
0923                 struct mlxsw_pci_queue *q, u8 q_num)
0924 {
0925     struct mlxsw_pci_mem_item *mem_item = &q->mem_item;
0926     int i;
0927     int err;
0928 
0929     q->num = q_num;
0930     if (q_ops->pre_init)
0931         q_ops->pre_init(mlxsw_pci, q);
0932 
0933     spin_lock_init(&q->lock);
0934     q->count = q_ops->elem_count_f ? q_ops->elem_count_f(q) :
0935                      q_ops->elem_count;
0936     q->elem_size = q_ops->elem_size_f ? q_ops->elem_size_f(q) :
0937                         q_ops->elem_size;
0938     q->type = q_ops->type;
0939     q->pci = mlxsw_pci;
0940 
0941     if (q_ops->tasklet)
0942         tasklet_setup(&q->tasklet, q_ops->tasklet);
0943 
0944     mem_item->size = MLXSW_PCI_AQ_SIZE;
0945     mem_item->buf = dma_alloc_coherent(&mlxsw_pci->pdev->dev,
0946                        mem_item->size, &mem_item->mapaddr,
0947                        GFP_KERNEL);
0948     if (!mem_item->buf)
0949         return -ENOMEM;
0950 
0951     q->elem_info = kcalloc(q->count, sizeof(*q->elem_info), GFP_KERNEL);
0952     if (!q->elem_info) {
0953         err = -ENOMEM;
0954         goto err_elem_info_alloc;
0955     }
0956 
0957     /* Initialize dma mapped elements info elem_info for
0958      * future easy access.
0959      */
0960     for (i = 0; i < q->count; i++) {
0961         struct mlxsw_pci_queue_elem_info *elem_info;
0962 
0963         elem_info = mlxsw_pci_queue_elem_info_get(q, i);
0964         elem_info->elem =
0965             __mlxsw_pci_queue_elem_get(q, q->elem_size, i);
0966     }
0967 
0968     mlxsw_cmd_mbox_zero(mbox);
0969     err = q_ops->init(mlxsw_pci, mbox, q);
0970     if (err)
0971         goto err_q_ops_init;
0972     return 0;
0973 
0974 err_q_ops_init:
0975     kfree(q->elem_info);
0976 err_elem_info_alloc:
0977     dma_free_coherent(&mlxsw_pci->pdev->dev, mem_item->size,
0978               mem_item->buf, mem_item->mapaddr);
0979     return err;
0980 }
0981 
0982 static void mlxsw_pci_queue_fini(struct mlxsw_pci *mlxsw_pci,
0983                  const struct mlxsw_pci_queue_ops *q_ops,
0984                  struct mlxsw_pci_queue *q)
0985 {
0986     struct mlxsw_pci_mem_item *mem_item = &q->mem_item;
0987 
0988     q_ops->fini(mlxsw_pci, q);
0989     kfree(q->elem_info);
0990     dma_free_coherent(&mlxsw_pci->pdev->dev, mem_item->size,
0991               mem_item->buf, mem_item->mapaddr);
0992 }
0993 
0994 static int mlxsw_pci_queue_group_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
0995                       const struct mlxsw_pci_queue_ops *q_ops,
0996                       u8 num_qs)
0997 {
0998     struct mlxsw_pci_queue_type_group *queue_group;
0999     int i;
1000     int err;
1001 
1002     queue_group = mlxsw_pci_queue_type_group_get(mlxsw_pci, q_ops->type);
1003     queue_group->q = kcalloc(num_qs, sizeof(*queue_group->q), GFP_KERNEL);
1004     if (!queue_group->q)
1005         return -ENOMEM;
1006 
1007     for (i = 0; i < num_qs; i++) {
1008         err = mlxsw_pci_queue_init(mlxsw_pci, mbox, q_ops,
1009                        &queue_group->q[i], i);
1010         if (err)
1011             goto err_queue_init;
1012     }
1013     queue_group->count = num_qs;
1014 
1015     return 0;
1016 
1017 err_queue_init:
1018     for (i--; i >= 0; i--)
1019         mlxsw_pci_queue_fini(mlxsw_pci, q_ops, &queue_group->q[i]);
1020     kfree(queue_group->q);
1021     return err;
1022 }
1023 
1024 static void mlxsw_pci_queue_group_fini(struct mlxsw_pci *mlxsw_pci,
1025                        const struct mlxsw_pci_queue_ops *q_ops)
1026 {
1027     struct mlxsw_pci_queue_type_group *queue_group;
1028     int i;
1029 
1030     queue_group = mlxsw_pci_queue_type_group_get(mlxsw_pci, q_ops->type);
1031     for (i = 0; i < queue_group->count; i++)
1032         mlxsw_pci_queue_fini(mlxsw_pci, q_ops, &queue_group->q[i]);
1033     kfree(queue_group->q);
1034 }
1035 
1036 static int mlxsw_pci_aqs_init(struct mlxsw_pci *mlxsw_pci, char *mbox)
1037 {
1038     struct pci_dev *pdev = mlxsw_pci->pdev;
1039     u8 num_sdqs;
1040     u8 sdq_log2sz;
1041     u8 num_rdqs;
1042     u8 rdq_log2sz;
1043     u8 num_cqs;
1044     u8 cq_log2sz;
1045     u8 cqv2_log2sz;
1046     u8 num_eqs;
1047     u8 eq_log2sz;
1048     int err;
1049 
1050     mlxsw_cmd_mbox_zero(mbox);
1051     err = mlxsw_cmd_query_aq_cap(mlxsw_pci->core, mbox);
1052     if (err)
1053         return err;
1054 
1055     num_sdqs = mlxsw_cmd_mbox_query_aq_cap_max_num_sdqs_get(mbox);
1056     sdq_log2sz = mlxsw_cmd_mbox_query_aq_cap_log_max_sdq_sz_get(mbox);
1057     num_rdqs = mlxsw_cmd_mbox_query_aq_cap_max_num_rdqs_get(mbox);
1058     rdq_log2sz = mlxsw_cmd_mbox_query_aq_cap_log_max_rdq_sz_get(mbox);
1059     num_cqs = mlxsw_cmd_mbox_query_aq_cap_max_num_cqs_get(mbox);
1060     cq_log2sz = mlxsw_cmd_mbox_query_aq_cap_log_max_cq_sz_get(mbox);
1061     cqv2_log2sz = mlxsw_cmd_mbox_query_aq_cap_log_max_cqv2_sz_get(mbox);
1062     num_eqs = mlxsw_cmd_mbox_query_aq_cap_max_num_eqs_get(mbox);
1063     eq_log2sz = mlxsw_cmd_mbox_query_aq_cap_log_max_eq_sz_get(mbox);
1064 
1065     if (num_sdqs + num_rdqs > num_cqs ||
1066         num_sdqs < MLXSW_PCI_SDQS_MIN ||
1067         num_cqs > MLXSW_PCI_CQS_MAX || num_eqs != MLXSW_PCI_EQS_COUNT) {
1068         dev_err(&pdev->dev, "Unsupported number of queues\n");
1069         return -EINVAL;
1070     }
1071 
1072     if ((1 << sdq_log2sz != MLXSW_PCI_WQE_COUNT) ||
1073         (1 << rdq_log2sz != MLXSW_PCI_WQE_COUNT) ||
1074         (1 << cq_log2sz != MLXSW_PCI_CQE01_COUNT) ||
1075         (mlxsw_pci->max_cqe_ver == MLXSW_PCI_CQE_V2 &&
1076          (1 << cqv2_log2sz != MLXSW_PCI_CQE2_COUNT)) ||
1077         (1 << eq_log2sz != MLXSW_PCI_EQE_COUNT)) {
1078         dev_err(&pdev->dev, "Unsupported number of async queue descriptors\n");
1079         return -EINVAL;
1080     }
1081 
1082     mlxsw_pci->num_sdq_cqs = num_sdqs;
1083 
1084     err = mlxsw_pci_queue_group_init(mlxsw_pci, mbox, &mlxsw_pci_eq_ops,
1085                      num_eqs);
1086     if (err) {
1087         dev_err(&pdev->dev, "Failed to initialize event queues\n");
1088         return err;
1089     }
1090 
1091     err = mlxsw_pci_queue_group_init(mlxsw_pci, mbox, &mlxsw_pci_cq_ops,
1092                      num_cqs);
1093     if (err) {
1094         dev_err(&pdev->dev, "Failed to initialize completion queues\n");
1095         goto err_cqs_init;
1096     }
1097 
1098     err = mlxsw_pci_queue_group_init(mlxsw_pci, mbox, &mlxsw_pci_sdq_ops,
1099                      num_sdqs);
1100     if (err) {
1101         dev_err(&pdev->dev, "Failed to initialize send descriptor queues\n");
1102         goto err_sdqs_init;
1103     }
1104 
1105     err = mlxsw_pci_queue_group_init(mlxsw_pci, mbox, &mlxsw_pci_rdq_ops,
1106                      num_rdqs);
1107     if (err) {
1108         dev_err(&pdev->dev, "Failed to initialize receive descriptor queues\n");
1109         goto err_rdqs_init;
1110     }
1111 
1112     /* We have to poll in command interface until queues are initialized */
1113     mlxsw_pci->cmd.nopoll = true;
1114     return 0;
1115 
1116 err_rdqs_init:
1117     mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_sdq_ops);
1118 err_sdqs_init:
1119     mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_cq_ops);
1120 err_cqs_init:
1121     mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_eq_ops);
1122     return err;
1123 }
1124 
1125 static void mlxsw_pci_aqs_fini(struct mlxsw_pci *mlxsw_pci)
1126 {
1127     mlxsw_pci->cmd.nopoll = false;
1128     mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_rdq_ops);
1129     mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_sdq_ops);
1130     mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_cq_ops);
1131     mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_eq_ops);
1132 }
1133 
1134 static void
1135 mlxsw_pci_config_profile_swid_config(struct mlxsw_pci *mlxsw_pci,
1136                      char *mbox, int index,
1137                      const struct mlxsw_swid_config *swid)
1138 {
1139     u8 mask = 0;
1140 
1141     if (swid->used_type) {
1142         mlxsw_cmd_mbox_config_profile_swid_config_type_set(
1143             mbox, index, swid->type);
1144         mask |= 1;
1145     }
1146     if (swid->used_properties) {
1147         mlxsw_cmd_mbox_config_profile_swid_config_properties_set(
1148             mbox, index, swid->properties);
1149         mask |= 2;
1150     }
1151     mlxsw_cmd_mbox_config_profile_swid_config_mask_set(mbox, index, mask);
1152 }
1153 
1154 static int
1155 mlxsw_pci_profile_get_kvd_sizes(const struct mlxsw_pci *mlxsw_pci,
1156                 const struct mlxsw_config_profile *profile,
1157                 struct mlxsw_res *res)
1158 {
1159     u64 single_size, double_size, linear_size;
1160     int err;
1161 
1162     err = mlxsw_core_kvd_sizes_get(mlxsw_pci->core, profile,
1163                        &single_size, &double_size,
1164                        &linear_size);
1165     if (err)
1166         return err;
1167 
1168     MLXSW_RES_SET(res, KVD_SINGLE_SIZE, single_size);
1169     MLXSW_RES_SET(res, KVD_DOUBLE_SIZE, double_size);
1170     MLXSW_RES_SET(res, KVD_LINEAR_SIZE, linear_size);
1171 
1172     return 0;
1173 }
1174 
1175 static int mlxsw_pci_config_profile(struct mlxsw_pci *mlxsw_pci, char *mbox,
1176                     const struct mlxsw_config_profile *profile,
1177                     struct mlxsw_res *res)
1178 {
1179     int i;
1180     int err;
1181 
1182     mlxsw_cmd_mbox_zero(mbox);
1183 
1184     if (profile->used_max_vepa_channels) {
1185         mlxsw_cmd_mbox_config_profile_set_max_vepa_channels_set(
1186             mbox, 1);
1187         mlxsw_cmd_mbox_config_profile_max_vepa_channels_set(
1188             mbox, profile->max_vepa_channels);
1189     }
1190     if (profile->used_max_mid) {
1191         mlxsw_cmd_mbox_config_profile_set_max_mid_set(
1192             mbox, 1);
1193         mlxsw_cmd_mbox_config_profile_max_mid_set(
1194             mbox, profile->max_mid);
1195     }
1196     if (profile->used_max_pgt) {
1197         mlxsw_cmd_mbox_config_profile_set_max_pgt_set(
1198             mbox, 1);
1199         mlxsw_cmd_mbox_config_profile_max_pgt_set(
1200             mbox, profile->max_pgt);
1201     }
1202     if (profile->used_max_system_port) {
1203         mlxsw_cmd_mbox_config_profile_set_max_system_port_set(
1204             mbox, 1);
1205         mlxsw_cmd_mbox_config_profile_max_system_port_set(
1206             mbox, profile->max_system_port);
1207     }
1208     if (profile->used_max_vlan_groups) {
1209         mlxsw_cmd_mbox_config_profile_set_max_vlan_groups_set(
1210             mbox, 1);
1211         mlxsw_cmd_mbox_config_profile_max_vlan_groups_set(
1212             mbox, profile->max_vlan_groups);
1213     }
1214     if (profile->used_max_regions) {
1215         mlxsw_cmd_mbox_config_profile_set_max_regions_set(
1216             mbox, 1);
1217         mlxsw_cmd_mbox_config_profile_max_regions_set(
1218             mbox, profile->max_regions);
1219     }
1220     if (profile->used_flood_tables) {
1221         mlxsw_cmd_mbox_config_profile_set_flood_tables_set(
1222             mbox, 1);
1223         mlxsw_cmd_mbox_config_profile_max_flood_tables_set(
1224             mbox, profile->max_flood_tables);
1225         mlxsw_cmd_mbox_config_profile_max_vid_flood_tables_set(
1226             mbox, profile->max_vid_flood_tables);
1227         mlxsw_cmd_mbox_config_profile_max_fid_offset_flood_tables_set(
1228             mbox, profile->max_fid_offset_flood_tables);
1229         mlxsw_cmd_mbox_config_profile_fid_offset_flood_table_size_set(
1230             mbox, profile->fid_offset_flood_table_size);
1231         mlxsw_cmd_mbox_config_profile_max_fid_flood_tables_set(
1232             mbox, profile->max_fid_flood_tables);
1233         mlxsw_cmd_mbox_config_profile_fid_flood_table_size_set(
1234             mbox, profile->fid_flood_table_size);
1235     }
1236     if (profile->used_flood_mode) {
1237         mlxsw_cmd_mbox_config_profile_set_flood_mode_set(
1238             mbox, 1);
1239         mlxsw_cmd_mbox_config_profile_flood_mode_set(
1240             mbox, profile->flood_mode);
1241     }
1242     if (profile->used_max_ib_mc) {
1243         mlxsw_cmd_mbox_config_profile_set_max_ib_mc_set(
1244             mbox, 1);
1245         mlxsw_cmd_mbox_config_profile_max_ib_mc_set(
1246             mbox, profile->max_ib_mc);
1247     }
1248     if (profile->used_max_pkey) {
1249         mlxsw_cmd_mbox_config_profile_set_max_pkey_set(
1250             mbox, 1);
1251         mlxsw_cmd_mbox_config_profile_max_pkey_set(
1252             mbox, profile->max_pkey);
1253     }
1254     if (profile->used_ar_sec) {
1255         mlxsw_cmd_mbox_config_profile_set_ar_sec_set(
1256             mbox, 1);
1257         mlxsw_cmd_mbox_config_profile_ar_sec_set(
1258             mbox, profile->ar_sec);
1259     }
1260     if (profile->used_adaptive_routing_group_cap) {
1261         mlxsw_cmd_mbox_config_profile_set_adaptive_routing_group_cap_set(
1262             mbox, 1);
1263         mlxsw_cmd_mbox_config_profile_adaptive_routing_group_cap_set(
1264             mbox, profile->adaptive_routing_group_cap);
1265     }
1266     if (profile->used_ubridge) {
1267         mlxsw_cmd_mbox_config_profile_set_ubridge_set(mbox, 1);
1268         mlxsw_cmd_mbox_config_profile_ubridge_set(mbox,
1269                               profile->ubridge);
1270     }
1271     if (profile->used_kvd_sizes && MLXSW_RES_VALID(res, KVD_SIZE)) {
1272         err = mlxsw_pci_profile_get_kvd_sizes(mlxsw_pci, profile, res);
1273         if (err)
1274             return err;
1275 
1276         mlxsw_cmd_mbox_config_profile_set_kvd_linear_size_set(mbox, 1);
1277         mlxsw_cmd_mbox_config_profile_kvd_linear_size_set(mbox,
1278                     MLXSW_RES_GET(res, KVD_LINEAR_SIZE));
1279         mlxsw_cmd_mbox_config_profile_set_kvd_hash_single_size_set(mbox,
1280                                        1);
1281         mlxsw_cmd_mbox_config_profile_kvd_hash_single_size_set(mbox,
1282                     MLXSW_RES_GET(res, KVD_SINGLE_SIZE));
1283         mlxsw_cmd_mbox_config_profile_set_kvd_hash_double_size_set(
1284                                 mbox, 1);
1285         mlxsw_cmd_mbox_config_profile_kvd_hash_double_size_set(mbox,
1286                     MLXSW_RES_GET(res, KVD_DOUBLE_SIZE));
1287     }
1288 
1289     for (i = 0; i < MLXSW_CONFIG_PROFILE_SWID_COUNT; i++)
1290         mlxsw_pci_config_profile_swid_config(mlxsw_pci, mbox, i,
1291                              &profile->swid_config[i]);
1292 
1293     if (mlxsw_pci->max_cqe_ver > MLXSW_PCI_CQE_V0) {
1294         mlxsw_cmd_mbox_config_profile_set_cqe_version_set(mbox, 1);
1295         mlxsw_cmd_mbox_config_profile_cqe_version_set(mbox, 1);
1296     }
1297 
1298     if (profile->used_cqe_time_stamp_type) {
1299         mlxsw_cmd_mbox_config_profile_set_cqe_time_stamp_type_set(mbox,
1300                                       1);
1301         mlxsw_cmd_mbox_config_profile_cqe_time_stamp_type_set(mbox,
1302                     profile->cqe_time_stamp_type);
1303     }
1304 
1305     return mlxsw_cmd_config_profile_set(mlxsw_pci->core, mbox);
1306 }
1307 
1308 static int mlxsw_pci_boardinfo(struct mlxsw_pci *mlxsw_pci, char *mbox)
1309 {
1310     struct mlxsw_bus_info *bus_info = &mlxsw_pci->bus_info;
1311     int err;
1312 
1313     mlxsw_cmd_mbox_zero(mbox);
1314     err = mlxsw_cmd_boardinfo(mlxsw_pci->core, mbox);
1315     if (err)
1316         return err;
1317     mlxsw_cmd_mbox_boardinfo_vsd_memcpy_from(mbox, bus_info->vsd);
1318     mlxsw_cmd_mbox_boardinfo_psid_memcpy_from(mbox, bus_info->psid);
1319     return 0;
1320 }
1321 
1322 static int mlxsw_pci_fw_area_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
1323                   u16 num_pages)
1324 {
1325     struct mlxsw_pci_mem_item *mem_item;
1326     int nent = 0;
1327     int i;
1328     int err;
1329 
1330     mlxsw_pci->fw_area.items = kcalloc(num_pages, sizeof(*mem_item),
1331                        GFP_KERNEL);
1332     if (!mlxsw_pci->fw_area.items)
1333         return -ENOMEM;
1334     mlxsw_pci->fw_area.count = num_pages;
1335 
1336     mlxsw_cmd_mbox_zero(mbox);
1337     for (i = 0; i < num_pages; i++) {
1338         mem_item = &mlxsw_pci->fw_area.items[i];
1339 
1340         mem_item->size = MLXSW_PCI_PAGE_SIZE;
1341         mem_item->buf = dma_alloc_coherent(&mlxsw_pci->pdev->dev,
1342                            mem_item->size,
1343                            &mem_item->mapaddr, GFP_KERNEL);
1344         if (!mem_item->buf) {
1345             err = -ENOMEM;
1346             goto err_alloc;
1347         }
1348         mlxsw_cmd_mbox_map_fa_pa_set(mbox, nent, mem_item->mapaddr);
1349         mlxsw_cmd_mbox_map_fa_log2size_set(mbox, nent, 0); /* 1 page */
1350         if (++nent == MLXSW_CMD_MAP_FA_VPM_ENTRIES_MAX) {
1351             err = mlxsw_cmd_map_fa(mlxsw_pci->core, mbox, nent);
1352             if (err)
1353                 goto err_cmd_map_fa;
1354             nent = 0;
1355             mlxsw_cmd_mbox_zero(mbox);
1356         }
1357     }
1358 
1359     if (nent) {
1360         err = mlxsw_cmd_map_fa(mlxsw_pci->core, mbox, nent);
1361         if (err)
1362             goto err_cmd_map_fa;
1363     }
1364 
1365     return 0;
1366 
1367 err_cmd_map_fa:
1368 err_alloc:
1369     for (i--; i >= 0; i--) {
1370         mem_item = &mlxsw_pci->fw_area.items[i];
1371 
1372         dma_free_coherent(&mlxsw_pci->pdev->dev, mem_item->size,
1373                   mem_item->buf, mem_item->mapaddr);
1374     }
1375     kfree(mlxsw_pci->fw_area.items);
1376     return err;
1377 }
1378 
1379 static void mlxsw_pci_fw_area_fini(struct mlxsw_pci *mlxsw_pci)
1380 {
1381     struct mlxsw_pci_mem_item *mem_item;
1382     int i;
1383 
1384     mlxsw_cmd_unmap_fa(mlxsw_pci->core);
1385 
1386     for (i = 0; i < mlxsw_pci->fw_area.count; i++) {
1387         mem_item = &mlxsw_pci->fw_area.items[i];
1388 
1389         dma_free_coherent(&mlxsw_pci->pdev->dev, mem_item->size,
1390                   mem_item->buf, mem_item->mapaddr);
1391     }
1392     kfree(mlxsw_pci->fw_area.items);
1393 }
1394 
1395 static irqreturn_t mlxsw_pci_eq_irq_handler(int irq, void *dev_id)
1396 {
1397     struct mlxsw_pci *mlxsw_pci = dev_id;
1398     struct mlxsw_pci_queue *q;
1399     int i;
1400 
1401     for (i = 0; i < MLXSW_PCI_EQS_COUNT; i++) {
1402         q = mlxsw_pci_eq_get(mlxsw_pci, i);
1403         mlxsw_pci_queue_tasklet_schedule(q);
1404     }
1405     return IRQ_HANDLED;
1406 }
1407 
1408 static int mlxsw_pci_mbox_alloc(struct mlxsw_pci *mlxsw_pci,
1409                 struct mlxsw_pci_mem_item *mbox)
1410 {
1411     struct pci_dev *pdev = mlxsw_pci->pdev;
1412     int err = 0;
1413 
1414     mbox->size = MLXSW_CMD_MBOX_SIZE;
1415     mbox->buf = dma_alloc_coherent(&pdev->dev, MLXSW_CMD_MBOX_SIZE,
1416                        &mbox->mapaddr, GFP_KERNEL);
1417     if (!mbox->buf) {
1418         dev_err(&pdev->dev, "Failed allocating memory for mailbox\n");
1419         err = -ENOMEM;
1420     }
1421 
1422     return err;
1423 }
1424 
1425 static void mlxsw_pci_mbox_free(struct mlxsw_pci *mlxsw_pci,
1426                 struct mlxsw_pci_mem_item *mbox)
1427 {
1428     struct pci_dev *pdev = mlxsw_pci->pdev;
1429 
1430     dma_free_coherent(&pdev->dev, MLXSW_CMD_MBOX_SIZE, mbox->buf,
1431               mbox->mapaddr);
1432 }
1433 
1434 static int mlxsw_pci_sys_ready_wait(struct mlxsw_pci *mlxsw_pci,
1435                     const struct pci_device_id *id,
1436                     u32 *p_sys_status)
1437 {
1438     unsigned long end;
1439     u32 val;
1440 
1441     /* We must wait for the HW to become responsive. */
1442     msleep(MLXSW_PCI_SW_RESET_WAIT_MSECS);
1443 
1444     end = jiffies + msecs_to_jiffies(MLXSW_PCI_SW_RESET_TIMEOUT_MSECS);
1445     do {
1446         val = mlxsw_pci_read32(mlxsw_pci, FW_READY);
1447         if ((val & MLXSW_PCI_FW_READY_MASK) == MLXSW_PCI_FW_READY_MAGIC)
1448             return 0;
1449         cond_resched();
1450     } while (time_before(jiffies, end));
1451 
1452     *p_sys_status = val & MLXSW_PCI_FW_READY_MASK;
1453 
1454     return -EBUSY;
1455 }
1456 
1457 static int mlxsw_pci_sw_reset(struct mlxsw_pci *mlxsw_pci,
1458                   const struct pci_device_id *id)
1459 {
1460     struct pci_dev *pdev = mlxsw_pci->pdev;
1461     char mrsr_pl[MLXSW_REG_MRSR_LEN];
1462     u32 sys_status;
1463     int err;
1464 
1465     err = mlxsw_pci_sys_ready_wait(mlxsw_pci, id, &sys_status);
1466     if (err) {
1467         dev_err(&pdev->dev, "Failed to reach system ready status before reset. Status is 0x%x\n",
1468             sys_status);
1469         return err;
1470     }
1471 
1472     mlxsw_reg_mrsr_pack(mrsr_pl);
1473     err = mlxsw_reg_write(mlxsw_pci->core, MLXSW_REG(mrsr), mrsr_pl);
1474     if (err)
1475         return err;
1476 
1477     err = mlxsw_pci_sys_ready_wait(mlxsw_pci, id, &sys_status);
1478     if (err) {
1479         dev_err(&pdev->dev, "Failed to reach system ready status after reset. Status is 0x%x\n",
1480             sys_status);
1481         return err;
1482     }
1483 
1484     return 0;
1485 }
1486 
1487 static int mlxsw_pci_alloc_irq_vectors(struct mlxsw_pci *mlxsw_pci)
1488 {
1489     int err;
1490 
1491     err = pci_alloc_irq_vectors(mlxsw_pci->pdev, 1, 1, PCI_IRQ_MSIX);
1492     if (err < 0)
1493         dev_err(&mlxsw_pci->pdev->dev, "MSI-X init failed\n");
1494     return err;
1495 }
1496 
1497 static void mlxsw_pci_free_irq_vectors(struct mlxsw_pci *mlxsw_pci)
1498 {
1499     pci_free_irq_vectors(mlxsw_pci->pdev);
1500 }
1501 
1502 static int mlxsw_pci_init(void *bus_priv, struct mlxsw_core *mlxsw_core,
1503               const struct mlxsw_config_profile *profile,
1504               struct mlxsw_res *res)
1505 {
1506     struct mlxsw_pci *mlxsw_pci = bus_priv;
1507     struct pci_dev *pdev = mlxsw_pci->pdev;
1508     char *mbox;
1509     u16 num_pages;
1510     int err;
1511 
1512     mlxsw_pci->core = mlxsw_core;
1513 
1514     mbox = mlxsw_cmd_mbox_alloc();
1515     if (!mbox)
1516         return -ENOMEM;
1517 
1518     err = mlxsw_pci_sw_reset(mlxsw_pci, mlxsw_pci->id);
1519     if (err)
1520         goto err_sw_reset;
1521 
1522     err = mlxsw_pci_alloc_irq_vectors(mlxsw_pci);
1523     if (err < 0) {
1524         dev_err(&pdev->dev, "MSI-X init failed\n");
1525         goto err_alloc_irq;
1526     }
1527 
1528     err = mlxsw_cmd_query_fw(mlxsw_core, mbox);
1529     if (err)
1530         goto err_query_fw;
1531 
1532     mlxsw_pci->bus_info.fw_rev.major =
1533         mlxsw_cmd_mbox_query_fw_fw_rev_major_get(mbox);
1534     mlxsw_pci->bus_info.fw_rev.minor =
1535         mlxsw_cmd_mbox_query_fw_fw_rev_minor_get(mbox);
1536     mlxsw_pci->bus_info.fw_rev.subminor =
1537         mlxsw_cmd_mbox_query_fw_fw_rev_subminor_get(mbox);
1538 
1539     if (mlxsw_cmd_mbox_query_fw_cmd_interface_rev_get(mbox) != 1) {
1540         dev_err(&pdev->dev, "Unsupported cmd interface revision ID queried from hw\n");
1541         err = -EINVAL;
1542         goto err_iface_rev;
1543     }
1544     if (mlxsw_cmd_mbox_query_fw_doorbell_page_bar_get(mbox) != 0) {
1545         dev_err(&pdev->dev, "Unsupported doorbell page bar queried from hw\n");
1546         err = -EINVAL;
1547         goto err_doorbell_page_bar;
1548     }
1549 
1550     mlxsw_pci->doorbell_offset =
1551         mlxsw_cmd_mbox_query_fw_doorbell_page_offset_get(mbox);
1552 
1553     if (mlxsw_cmd_mbox_query_fw_fr_rn_clk_bar_get(mbox) != 0) {
1554         dev_err(&pdev->dev, "Unsupported free running clock BAR queried from hw\n");
1555         err = -EINVAL;
1556         goto err_fr_rn_clk_bar;
1557     }
1558 
1559     mlxsw_pci->free_running_clock_offset =
1560         mlxsw_cmd_mbox_query_fw_free_running_clock_offset_get(mbox);
1561 
1562     if (mlxsw_cmd_mbox_query_fw_utc_sec_bar_get(mbox) != 0) {
1563         dev_err(&pdev->dev, "Unsupported UTC sec BAR queried from hw\n");
1564         err = -EINVAL;
1565         goto err_utc_sec_bar;
1566     }
1567 
1568     mlxsw_pci->utc_sec_offset =
1569         mlxsw_cmd_mbox_query_fw_utc_sec_offset_get(mbox);
1570 
1571     if (mlxsw_cmd_mbox_query_fw_utc_nsec_bar_get(mbox) != 0) {
1572         dev_err(&pdev->dev, "Unsupported UTC nsec BAR queried from hw\n");
1573         err = -EINVAL;
1574         goto err_utc_nsec_bar;
1575     }
1576 
1577     mlxsw_pci->utc_nsec_offset =
1578         mlxsw_cmd_mbox_query_fw_utc_nsec_offset_get(mbox);
1579 
1580     num_pages = mlxsw_cmd_mbox_query_fw_fw_pages_get(mbox);
1581     err = mlxsw_pci_fw_area_init(mlxsw_pci, mbox, num_pages);
1582     if (err)
1583         goto err_fw_area_init;
1584 
1585     err = mlxsw_pci_boardinfo(mlxsw_pci, mbox);
1586     if (err)
1587         goto err_boardinfo;
1588 
1589     err = mlxsw_core_resources_query(mlxsw_core, mbox, res);
1590     if (err)
1591         goto err_query_resources;
1592 
1593     if (MLXSW_CORE_RES_VALID(mlxsw_core, CQE_V2) &&
1594         MLXSW_CORE_RES_GET(mlxsw_core, CQE_V2))
1595         mlxsw_pci->max_cqe_ver = MLXSW_PCI_CQE_V2;
1596     else if (MLXSW_CORE_RES_VALID(mlxsw_core, CQE_V1) &&
1597          MLXSW_CORE_RES_GET(mlxsw_core, CQE_V1))
1598         mlxsw_pci->max_cqe_ver = MLXSW_PCI_CQE_V1;
1599     else if ((MLXSW_CORE_RES_VALID(mlxsw_core, CQE_V0) &&
1600           MLXSW_CORE_RES_GET(mlxsw_core, CQE_V0)) ||
1601          !MLXSW_CORE_RES_VALID(mlxsw_core, CQE_V0)) {
1602         mlxsw_pci->max_cqe_ver = MLXSW_PCI_CQE_V0;
1603     } else {
1604         dev_err(&pdev->dev, "Invalid supported CQE version combination reported\n");
1605         goto err_cqe_v_check;
1606     }
1607 
1608     err = mlxsw_pci_config_profile(mlxsw_pci, mbox, profile, res);
1609     if (err)
1610         goto err_config_profile;
1611 
1612     /* Some resources depend on unified bridge model, which is configured
1613      * as part of config_profile. Query the resources again to get correct
1614      * values.
1615      */
1616     err = mlxsw_core_resources_query(mlxsw_core, mbox, res);
1617     if (err)
1618         goto err_requery_resources;
1619 
1620     err = mlxsw_pci_aqs_init(mlxsw_pci, mbox);
1621     if (err)
1622         goto err_aqs_init;
1623 
1624     err = request_irq(pci_irq_vector(pdev, 0),
1625               mlxsw_pci_eq_irq_handler, 0,
1626               mlxsw_pci->bus_info.device_kind, mlxsw_pci);
1627     if (err) {
1628         dev_err(&pdev->dev, "IRQ request failed\n");
1629         goto err_request_eq_irq;
1630     }
1631 
1632     goto mbox_put;
1633 
1634 err_request_eq_irq:
1635     mlxsw_pci_aqs_fini(mlxsw_pci);
1636 err_aqs_init:
1637 err_requery_resources:
1638 err_config_profile:
1639 err_cqe_v_check:
1640 err_query_resources:
1641 err_boardinfo:
1642     mlxsw_pci_fw_area_fini(mlxsw_pci);
1643 err_fw_area_init:
1644 err_utc_nsec_bar:
1645 err_utc_sec_bar:
1646 err_fr_rn_clk_bar:
1647 err_doorbell_page_bar:
1648 err_iface_rev:
1649 err_query_fw:
1650     mlxsw_pci_free_irq_vectors(mlxsw_pci);
1651 err_alloc_irq:
1652 err_sw_reset:
1653 mbox_put:
1654     mlxsw_cmd_mbox_free(mbox);
1655     return err;
1656 }
1657 
1658 static void mlxsw_pci_fini(void *bus_priv)
1659 {
1660     struct mlxsw_pci *mlxsw_pci = bus_priv;
1661 
1662     free_irq(pci_irq_vector(mlxsw_pci->pdev, 0), mlxsw_pci);
1663     mlxsw_pci_aqs_fini(mlxsw_pci);
1664     mlxsw_pci_fw_area_fini(mlxsw_pci);
1665     mlxsw_pci_free_irq_vectors(mlxsw_pci);
1666 }
1667 
1668 static struct mlxsw_pci_queue *
1669 mlxsw_pci_sdq_pick(struct mlxsw_pci *mlxsw_pci,
1670            const struct mlxsw_tx_info *tx_info)
1671 {
1672     u8 ctl_sdq_count = mlxsw_pci_sdq_count(mlxsw_pci) - 1;
1673     u8 sdqn;
1674 
1675     if (tx_info->is_emad) {
1676         sdqn = MLXSW_PCI_SDQ_EMAD_INDEX;
1677     } else {
1678         BUILD_BUG_ON(MLXSW_PCI_SDQ_EMAD_INDEX != 0);
1679         sdqn = 1 + (tx_info->local_port % ctl_sdq_count);
1680     }
1681 
1682     return mlxsw_pci_sdq_get(mlxsw_pci, sdqn);
1683 }
1684 
1685 static bool mlxsw_pci_skb_transmit_busy(void *bus_priv,
1686                     const struct mlxsw_tx_info *tx_info)
1687 {
1688     struct mlxsw_pci *mlxsw_pci = bus_priv;
1689     struct mlxsw_pci_queue *q = mlxsw_pci_sdq_pick(mlxsw_pci, tx_info);
1690 
1691     return !mlxsw_pci_queue_elem_info_producer_get(q);
1692 }
1693 
1694 static int mlxsw_pci_skb_transmit(void *bus_priv, struct sk_buff *skb,
1695                   const struct mlxsw_tx_info *tx_info)
1696 {
1697     struct mlxsw_pci *mlxsw_pci = bus_priv;
1698     struct mlxsw_pci_queue *q;
1699     struct mlxsw_pci_queue_elem_info *elem_info;
1700     char *wqe;
1701     int i;
1702     int err;
1703 
1704     if (skb_shinfo(skb)->nr_frags > MLXSW_PCI_WQE_SG_ENTRIES - 1) {
1705         err = skb_linearize(skb);
1706         if (err)
1707             return err;
1708     }
1709 
1710     q = mlxsw_pci_sdq_pick(mlxsw_pci, tx_info);
1711     spin_lock_bh(&q->lock);
1712     elem_info = mlxsw_pci_queue_elem_info_producer_get(q);
1713     if (!elem_info) {
1714         /* queue is full */
1715         err = -EAGAIN;
1716         goto unlock;
1717     }
1718     mlxsw_skb_cb(skb)->tx_info = *tx_info;
1719     elem_info->u.sdq.skb = skb;
1720 
1721     wqe = elem_info->elem;
1722     mlxsw_pci_wqe_c_set(wqe, 1); /* always report completion */
1723     mlxsw_pci_wqe_lp_set(wqe, 0);
1724     mlxsw_pci_wqe_type_set(wqe, MLXSW_PCI_WQE_TYPE_ETHERNET);
1725 
1726     err = mlxsw_pci_wqe_frag_map(mlxsw_pci, wqe, 0, skb->data,
1727                      skb_headlen(skb), DMA_TO_DEVICE);
1728     if (err)
1729         goto unlock;
1730 
1731     for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1732         const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1733 
1734         err = mlxsw_pci_wqe_frag_map(mlxsw_pci, wqe, i + 1,
1735                          skb_frag_address(frag),
1736                          skb_frag_size(frag),
1737                          DMA_TO_DEVICE);
1738         if (err)
1739             goto unmap_frags;
1740     }
1741 
1742     if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))
1743         skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1744 
1745     /* Set unused sq entries byte count to zero. */
1746     for (i++; i < MLXSW_PCI_WQE_SG_ENTRIES; i++)
1747         mlxsw_pci_wqe_byte_count_set(wqe, i, 0);
1748 
1749     /* Everything is set up, ring producer doorbell to get HW going */
1750     q->producer_counter++;
1751     mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q);
1752 
1753     goto unlock;
1754 
1755 unmap_frags:
1756     for (; i >= 0; i--)
1757         mlxsw_pci_wqe_frag_unmap(mlxsw_pci, wqe, i, DMA_TO_DEVICE);
1758 unlock:
1759     spin_unlock_bh(&q->lock);
1760     return err;
1761 }
1762 
1763 static int mlxsw_pci_cmd_exec(void *bus_priv, u16 opcode, u8 opcode_mod,
1764                   u32 in_mod, bool out_mbox_direct,
1765                   char *in_mbox, size_t in_mbox_size,
1766                   char *out_mbox, size_t out_mbox_size,
1767                   u8 *p_status)
1768 {
1769     struct mlxsw_pci *mlxsw_pci = bus_priv;
1770     dma_addr_t in_mapaddr = 0, out_mapaddr = 0;
1771     bool evreq = mlxsw_pci->cmd.nopoll;
1772     unsigned long timeout = msecs_to_jiffies(MLXSW_PCI_CIR_TIMEOUT_MSECS);
1773     bool *p_wait_done = &mlxsw_pci->cmd.wait_done;
1774     int err;
1775 
1776     *p_status = MLXSW_CMD_STATUS_OK;
1777 
1778     err = mutex_lock_interruptible(&mlxsw_pci->cmd.lock);
1779     if (err)
1780         return err;
1781 
1782     if (in_mbox) {
1783         memcpy(mlxsw_pci->cmd.in_mbox.buf, in_mbox, in_mbox_size);
1784         in_mapaddr = mlxsw_pci->cmd.in_mbox.mapaddr;
1785     }
1786     mlxsw_pci_write32(mlxsw_pci, CIR_IN_PARAM_HI, upper_32_bits(in_mapaddr));
1787     mlxsw_pci_write32(mlxsw_pci, CIR_IN_PARAM_LO, lower_32_bits(in_mapaddr));
1788 
1789     if (out_mbox)
1790         out_mapaddr = mlxsw_pci->cmd.out_mbox.mapaddr;
1791     mlxsw_pci_write32(mlxsw_pci, CIR_OUT_PARAM_HI, upper_32_bits(out_mapaddr));
1792     mlxsw_pci_write32(mlxsw_pci, CIR_OUT_PARAM_LO, lower_32_bits(out_mapaddr));
1793 
1794     mlxsw_pci_write32(mlxsw_pci, CIR_IN_MODIFIER, in_mod);
1795     mlxsw_pci_write32(mlxsw_pci, CIR_TOKEN, 0);
1796 
1797     *p_wait_done = false;
1798 
1799     wmb(); /* all needs to be written before we write control register */
1800     mlxsw_pci_write32(mlxsw_pci, CIR_CTRL,
1801               MLXSW_PCI_CIR_CTRL_GO_BIT |
1802               (evreq ? MLXSW_PCI_CIR_CTRL_EVREQ_BIT : 0) |
1803               (opcode_mod << MLXSW_PCI_CIR_CTRL_OPCODE_MOD_SHIFT) |
1804               opcode);
1805 
1806     if (!evreq) {
1807         unsigned long end;
1808 
1809         end = jiffies + timeout;
1810         do {
1811             u32 ctrl = mlxsw_pci_read32(mlxsw_pci, CIR_CTRL);
1812 
1813             if (!(ctrl & MLXSW_PCI_CIR_CTRL_GO_BIT)) {
1814                 *p_wait_done = true;
1815                 *p_status = ctrl >> MLXSW_PCI_CIR_CTRL_STATUS_SHIFT;
1816                 break;
1817             }
1818             cond_resched();
1819         } while (time_before(jiffies, end));
1820     } else {
1821         wait_event_timeout(mlxsw_pci->cmd.wait, *p_wait_done, timeout);
1822         *p_status = mlxsw_pci->cmd.comp.status;
1823     }
1824 
1825     err = 0;
1826     if (*p_wait_done) {
1827         if (*p_status)
1828             err = -EIO;
1829     } else {
1830         err = -ETIMEDOUT;
1831     }
1832 
1833     if (!err && out_mbox && out_mbox_direct) {
1834         /* Some commands don't use output param as address to mailbox
1835          * but they store output directly into registers. In that case,
1836          * copy registers into mbox buffer.
1837          */
1838         __be32 tmp;
1839 
1840         if (!evreq) {
1841             tmp = cpu_to_be32(mlxsw_pci_read32(mlxsw_pci,
1842                                CIR_OUT_PARAM_HI));
1843             memcpy(out_mbox, &tmp, sizeof(tmp));
1844             tmp = cpu_to_be32(mlxsw_pci_read32(mlxsw_pci,
1845                                CIR_OUT_PARAM_LO));
1846             memcpy(out_mbox + sizeof(tmp), &tmp, sizeof(tmp));
1847         }
1848     } else if (!err && out_mbox) {
1849         memcpy(out_mbox, mlxsw_pci->cmd.out_mbox.buf, out_mbox_size);
1850     }
1851 
1852     mutex_unlock(&mlxsw_pci->cmd.lock);
1853 
1854     return err;
1855 }
1856 
1857 static u32 mlxsw_pci_read_frc_h(void *bus_priv)
1858 {
1859     struct mlxsw_pci *mlxsw_pci = bus_priv;
1860     u64 frc_offset_h;
1861 
1862     frc_offset_h = mlxsw_pci->free_running_clock_offset;
1863     return mlxsw_pci_read32_off(mlxsw_pci, frc_offset_h);
1864 }
1865 
1866 static u32 mlxsw_pci_read_frc_l(void *bus_priv)
1867 {
1868     struct mlxsw_pci *mlxsw_pci = bus_priv;
1869     u64 frc_offset_l;
1870 
1871     frc_offset_l = mlxsw_pci->free_running_clock_offset + 4;
1872     return mlxsw_pci_read32_off(mlxsw_pci, frc_offset_l);
1873 }
1874 
1875 static u32 mlxsw_pci_read_utc_sec(void *bus_priv)
1876 {
1877     struct mlxsw_pci *mlxsw_pci = bus_priv;
1878 
1879     return mlxsw_pci_read32_off(mlxsw_pci, mlxsw_pci->utc_sec_offset);
1880 }
1881 
1882 static u32 mlxsw_pci_read_utc_nsec(void *bus_priv)
1883 {
1884     struct mlxsw_pci *mlxsw_pci = bus_priv;
1885 
1886     return mlxsw_pci_read32_off(mlxsw_pci, mlxsw_pci->utc_nsec_offset);
1887 }
1888 
1889 static const struct mlxsw_bus mlxsw_pci_bus = {
1890     .kind           = "pci",
1891     .init           = mlxsw_pci_init,
1892     .fini           = mlxsw_pci_fini,
1893     .skb_transmit_busy  = mlxsw_pci_skb_transmit_busy,
1894     .skb_transmit       = mlxsw_pci_skb_transmit,
1895     .cmd_exec       = mlxsw_pci_cmd_exec,
1896     .read_frc_h     = mlxsw_pci_read_frc_h,
1897     .read_frc_l     = mlxsw_pci_read_frc_l,
1898     .read_utc_sec       = mlxsw_pci_read_utc_sec,
1899     .read_utc_nsec      = mlxsw_pci_read_utc_nsec,
1900     .features       = MLXSW_BUS_F_TXRX | MLXSW_BUS_F_RESET,
1901 };
1902 
1903 static int mlxsw_pci_cmd_init(struct mlxsw_pci *mlxsw_pci)
1904 {
1905     int err;
1906 
1907     mutex_init(&mlxsw_pci->cmd.lock);
1908     init_waitqueue_head(&mlxsw_pci->cmd.wait);
1909 
1910     err = mlxsw_pci_mbox_alloc(mlxsw_pci, &mlxsw_pci->cmd.in_mbox);
1911     if (err)
1912         goto err_in_mbox_alloc;
1913 
1914     err = mlxsw_pci_mbox_alloc(mlxsw_pci, &mlxsw_pci->cmd.out_mbox);
1915     if (err)
1916         goto err_out_mbox_alloc;
1917 
1918     return 0;
1919 
1920 err_out_mbox_alloc:
1921     mlxsw_pci_mbox_free(mlxsw_pci, &mlxsw_pci->cmd.in_mbox);
1922 err_in_mbox_alloc:
1923     mutex_destroy(&mlxsw_pci->cmd.lock);
1924     return err;
1925 }
1926 
1927 static void mlxsw_pci_cmd_fini(struct mlxsw_pci *mlxsw_pci)
1928 {
1929     mlxsw_pci_mbox_free(mlxsw_pci, &mlxsw_pci->cmd.out_mbox);
1930     mlxsw_pci_mbox_free(mlxsw_pci, &mlxsw_pci->cmd.in_mbox);
1931     mutex_destroy(&mlxsw_pci->cmd.lock);
1932 }
1933 
1934 static int mlxsw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1935 {
1936     const char *driver_name = dev_driver_string(&pdev->dev);
1937     struct mlxsw_pci *mlxsw_pci;
1938     int err;
1939 
1940     mlxsw_pci = kzalloc(sizeof(*mlxsw_pci), GFP_KERNEL);
1941     if (!mlxsw_pci)
1942         return -ENOMEM;
1943 
1944     err = pci_enable_device(pdev);
1945     if (err) {
1946         dev_err(&pdev->dev, "pci_enable_device failed\n");
1947         goto err_pci_enable_device;
1948     }
1949 
1950     err = pci_request_regions(pdev, driver_name);
1951     if (err) {
1952         dev_err(&pdev->dev, "pci_request_regions failed\n");
1953         goto err_pci_request_regions;
1954     }
1955 
1956     err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1957     if (err) {
1958         err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
1959         if (err) {
1960             dev_err(&pdev->dev, "dma_set_mask failed\n");
1961             goto err_pci_set_dma_mask;
1962         }
1963     }
1964 
1965     if (pci_resource_len(pdev, 0) < MLXSW_PCI_BAR0_SIZE) {
1966         dev_err(&pdev->dev, "invalid PCI region size\n");
1967         err = -EINVAL;
1968         goto err_pci_resource_len_check;
1969     }
1970 
1971     mlxsw_pci->hw_addr = ioremap(pci_resource_start(pdev, 0),
1972                      pci_resource_len(pdev, 0));
1973     if (!mlxsw_pci->hw_addr) {
1974         dev_err(&pdev->dev, "ioremap failed\n");
1975         err = -EIO;
1976         goto err_ioremap;
1977     }
1978     pci_set_master(pdev);
1979 
1980     mlxsw_pci->pdev = pdev;
1981     pci_set_drvdata(pdev, mlxsw_pci);
1982 
1983     err = mlxsw_pci_cmd_init(mlxsw_pci);
1984     if (err)
1985         goto err_pci_cmd_init;
1986 
1987     mlxsw_pci->bus_info.device_kind = driver_name;
1988     mlxsw_pci->bus_info.device_name = pci_name(mlxsw_pci->pdev);
1989     mlxsw_pci->bus_info.dev = &pdev->dev;
1990     mlxsw_pci->bus_info.read_clock_capable = true;
1991     mlxsw_pci->id = id;
1992 
1993     err = mlxsw_core_bus_device_register(&mlxsw_pci->bus_info,
1994                          &mlxsw_pci_bus, mlxsw_pci, false,
1995                          NULL, NULL);
1996     if (err) {
1997         dev_err(&pdev->dev, "cannot register bus device\n");
1998         goto err_bus_device_register;
1999     }
2000 
2001     return 0;
2002 
2003 err_bus_device_register:
2004     mlxsw_pci_cmd_fini(mlxsw_pci);
2005 err_pci_cmd_init:
2006     iounmap(mlxsw_pci->hw_addr);
2007 err_ioremap:
2008 err_pci_resource_len_check:
2009 err_pci_set_dma_mask:
2010     pci_release_regions(pdev);
2011 err_pci_request_regions:
2012     pci_disable_device(pdev);
2013 err_pci_enable_device:
2014     kfree(mlxsw_pci);
2015     return err;
2016 }
2017 
2018 static void mlxsw_pci_remove(struct pci_dev *pdev)
2019 {
2020     struct mlxsw_pci *mlxsw_pci = pci_get_drvdata(pdev);
2021 
2022     mlxsw_core_bus_device_unregister(mlxsw_pci->core, false);
2023     mlxsw_pci_cmd_fini(mlxsw_pci);
2024     iounmap(mlxsw_pci->hw_addr);
2025     pci_release_regions(mlxsw_pci->pdev);
2026     pci_disable_device(mlxsw_pci->pdev);
2027     kfree(mlxsw_pci);
2028 }
2029 
2030 int mlxsw_pci_driver_register(struct pci_driver *pci_driver)
2031 {
2032     pci_driver->probe = mlxsw_pci_probe;
2033     pci_driver->remove = mlxsw_pci_remove;
2034     pci_driver->shutdown = mlxsw_pci_remove;
2035     return pci_register_driver(pci_driver);
2036 }
2037 EXPORT_SYMBOL(mlxsw_pci_driver_register);
2038 
2039 void mlxsw_pci_driver_unregister(struct pci_driver *pci_driver)
2040 {
2041     pci_unregister_driver(pci_driver);
2042 }
2043 EXPORT_SYMBOL(mlxsw_pci_driver_unregister);
2044 
2045 static int __init mlxsw_pci_module_init(void)
2046 {
2047     return 0;
2048 }
2049 
2050 static void __exit mlxsw_pci_module_exit(void)
2051 {
2052 }
2053 
2054 module_init(mlxsw_pci_module_init);
2055 module_exit(mlxsw_pci_module_exit);
2056 
2057 MODULE_LICENSE("Dual BSD/GPL");
2058 MODULE_AUTHOR("Jiri Pirko <jiri@mellanox.com>");
2059 MODULE_DESCRIPTION("Mellanox switch PCI interface driver");
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!