Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0+
0002 /*
0003  * Mellanox BlueField SoC TmFifo driver
0004  *
0005  * Copyright (C) 2019 Mellanox Technologies
0006  */
0007 
0008 #include <linux/acpi.h>
0009 #include <linux/bitfield.h>
0010 #include <linux/circ_buf.h>
0011 #include <linux/efi.h>
0012 #include <linux/irq.h>
0013 #include <linux/module.h>
0014 #include <linux/mutex.h>
0015 #include <linux/platform_device.h>
0016 #include <linux/types.h>
0017 
0018 #include <linux/virtio_config.h>
0019 #include <linux/virtio_console.h>
0020 #include <linux/virtio_ids.h>
0021 #include <linux/virtio_net.h>
0022 #include <linux/virtio_ring.h>
0023 
0024 #include "mlxbf-tmfifo-regs.h"
0025 
0026 /* Vring size. */
0027 #define MLXBF_TMFIFO_VRING_SIZE         SZ_1K
0028 
0029 /* Console Tx buffer size. */
0030 #define MLXBF_TMFIFO_CON_TX_BUF_SIZE        SZ_32K
0031 
0032 /* Console Tx buffer reserved space. */
0033 #define MLXBF_TMFIFO_CON_TX_BUF_RSV_SIZE    8
0034 
0035 /* House-keeping timer interval. */
0036 #define MLXBF_TMFIFO_TIMER_INTERVAL     (HZ / 10)
0037 
0038 /* Virtual devices sharing the TM FIFO. */
0039 #define MLXBF_TMFIFO_VDEV_MAX       (VIRTIO_ID_CONSOLE + 1)
0040 
0041 /*
0042  * Reserve 1/16 of TmFifo space, so console messages are not starved by
0043  * the networking traffic.
0044  */
0045 #define MLXBF_TMFIFO_RESERVE_RATIO      16
0046 
0047 /* Message with data needs at least two words (for header & data). */
0048 #define MLXBF_TMFIFO_DATA_MIN_WORDS     2
0049 
0050 struct mlxbf_tmfifo;
0051 
0052 /**
0053  * mlxbf_tmfifo_vring - Structure of the TmFifo virtual ring
0054  * @va: virtual address of the ring
0055  * @dma: dma address of the ring
0056  * @vq: pointer to the virtio virtqueue
0057  * @desc: current descriptor of the pending packet
0058  * @desc_head: head descriptor of the pending packet
0059  * @cur_len: processed length of the current descriptor
0060  * @rem_len: remaining length of the pending packet
0061  * @pkt_len: total length of the pending packet
0062  * @next_avail: next avail descriptor id
0063  * @num: vring size (number of descriptors)
0064  * @align: vring alignment size
0065  * @index: vring index
0066  * @vdev_id: vring virtio id (VIRTIO_ID_xxx)
0067  * @fifo: pointer to the tmfifo structure
0068  */
0069 struct mlxbf_tmfifo_vring {
0070     void *va;
0071     dma_addr_t dma;
0072     struct virtqueue *vq;
0073     struct vring_desc *desc;
0074     struct vring_desc *desc_head;
0075     int cur_len;
0076     int rem_len;
0077     u32 pkt_len;
0078     u16 next_avail;
0079     int num;
0080     int align;
0081     int index;
0082     int vdev_id;
0083     struct mlxbf_tmfifo *fifo;
0084 };
0085 
0086 /* Interrupt types. */
0087 enum {
0088     MLXBF_TM_RX_LWM_IRQ,
0089     MLXBF_TM_RX_HWM_IRQ,
0090     MLXBF_TM_TX_LWM_IRQ,
0091     MLXBF_TM_TX_HWM_IRQ,
0092     MLXBF_TM_MAX_IRQ
0093 };
0094 
0095 /* Ring types (Rx & Tx). */
0096 enum {
0097     MLXBF_TMFIFO_VRING_RX,
0098     MLXBF_TMFIFO_VRING_TX,
0099     MLXBF_TMFIFO_VRING_MAX
0100 };
0101 
0102 /**
0103  * mlxbf_tmfifo_vdev - Structure of the TmFifo virtual device
0104  * @vdev: virtio device, in which the vdev.id.device field has the
0105  *        VIRTIO_ID_xxx id to distinguish the virtual device.
0106  * @status: status of the device
0107  * @features: supported features of the device
0108  * @vrings: array of tmfifo vrings of this device
0109  * @config.cons: virtual console config -
0110  *               select if vdev.id.device is VIRTIO_ID_CONSOLE
0111  * @config.net: virtual network config -
0112  *              select if vdev.id.device is VIRTIO_ID_NET
0113  * @tx_buf: tx buffer used to buffer data before writing into the FIFO
0114  */
0115 struct mlxbf_tmfifo_vdev {
0116     struct virtio_device vdev;
0117     u8 status;
0118     u64 features;
0119     struct mlxbf_tmfifo_vring vrings[MLXBF_TMFIFO_VRING_MAX];
0120     union {
0121         struct virtio_console_config cons;
0122         struct virtio_net_config net;
0123     } config;
0124     struct circ_buf tx_buf;
0125 };
0126 
0127 /**
0128  * mlxbf_tmfifo_irq_info - Structure of the interrupt information
0129  * @fifo: pointer to the tmfifo structure
0130  * @irq: interrupt number
0131  * @index: index into the interrupt array
0132  */
0133 struct mlxbf_tmfifo_irq_info {
0134     struct mlxbf_tmfifo *fifo;
0135     int irq;
0136     int index;
0137 };
0138 
0139 /**
0140  * mlxbf_tmfifo - Structure of the TmFifo
0141  * @vdev: array of the virtual devices running over the TmFifo
0142  * @lock: lock to protect the TmFifo access
0143  * @rx_base: mapped register base address for the Rx FIFO
0144  * @tx_base: mapped register base address for the Tx FIFO
0145  * @rx_fifo_size: number of entries of the Rx FIFO
0146  * @tx_fifo_size: number of entries of the Tx FIFO
0147  * @pend_events: pending bits for deferred events
0148  * @irq_info: interrupt information
0149  * @work: work struct for deferred process
0150  * @timer: background timer
0151  * @vring: Tx/Rx ring
0152  * @spin_lock: Tx/Rx spin lock
0153  * @is_ready: ready flag
0154  */
0155 struct mlxbf_tmfifo {
0156     struct mlxbf_tmfifo_vdev *vdev[MLXBF_TMFIFO_VDEV_MAX];
0157     struct mutex lock;      /* TmFifo lock */
0158     void __iomem *rx_base;
0159     void __iomem *tx_base;
0160     int rx_fifo_size;
0161     int tx_fifo_size;
0162     unsigned long pend_events;
0163     struct mlxbf_tmfifo_irq_info irq_info[MLXBF_TM_MAX_IRQ];
0164     struct work_struct work;
0165     struct timer_list timer;
0166     struct mlxbf_tmfifo_vring *vring[2];
0167     spinlock_t spin_lock[2];    /* spin lock */
0168     bool is_ready;
0169 };
0170 
0171 /**
0172  * mlxbf_tmfifo_msg_hdr - Structure of the TmFifo message header
0173  * @type: message type
0174  * @len: payload length in network byte order. Messages sent into the FIFO
0175  *       will be read by the other side as data stream in the same byte order.
0176  *       The length needs to be encoded into network order so both sides
0177  *       could understand it.
0178  */
0179 struct mlxbf_tmfifo_msg_hdr {
0180     u8 type;
0181     __be16 len;
0182     u8 unused[5];
0183 } __packed __aligned(sizeof(u64));
0184 
0185 /*
0186  * Default MAC.
0187  * This MAC address will be read from EFI persistent variable if configured.
0188  * It can also be reconfigured with standard Linux tools.
0189  */
0190 static u8 mlxbf_tmfifo_net_default_mac[ETH_ALEN] = {
0191     0x00, 0x1A, 0xCA, 0xFF, 0xFF, 0x01
0192 };
0193 
0194 /* EFI variable name of the MAC address. */
0195 static efi_char16_t mlxbf_tmfifo_efi_name[] = L"RshimMacAddr";
0196 
0197 /* Maximum L2 header length. */
0198 #define MLXBF_TMFIFO_NET_L2_OVERHEAD    36
0199 
0200 /* Supported virtio-net features. */
0201 #define MLXBF_TMFIFO_NET_FEATURES \
0202     (BIT_ULL(VIRTIO_NET_F_MTU) | BIT_ULL(VIRTIO_NET_F_STATUS) | \
0203      BIT_ULL(VIRTIO_NET_F_MAC))
0204 
0205 #define mlxbf_vdev_to_tmfifo(d) container_of(d, struct mlxbf_tmfifo_vdev, vdev)
0206 
0207 /* Free vrings of the FIFO device. */
0208 static void mlxbf_tmfifo_free_vrings(struct mlxbf_tmfifo *fifo,
0209                      struct mlxbf_tmfifo_vdev *tm_vdev)
0210 {
0211     struct mlxbf_tmfifo_vring *vring;
0212     int i, size;
0213 
0214     for (i = 0; i < ARRAY_SIZE(tm_vdev->vrings); i++) {
0215         vring = &tm_vdev->vrings[i];
0216         if (vring->va) {
0217             size = vring_size(vring->num, vring->align);
0218             dma_free_coherent(tm_vdev->vdev.dev.parent, size,
0219                       vring->va, vring->dma);
0220             vring->va = NULL;
0221             if (vring->vq) {
0222                 vring_del_virtqueue(vring->vq);
0223                 vring->vq = NULL;
0224             }
0225         }
0226     }
0227 }
0228 
0229 /* Allocate vrings for the FIFO. */
0230 static int mlxbf_tmfifo_alloc_vrings(struct mlxbf_tmfifo *fifo,
0231                      struct mlxbf_tmfifo_vdev *tm_vdev)
0232 {
0233     struct mlxbf_tmfifo_vring *vring;
0234     struct device *dev;
0235     dma_addr_t dma;
0236     int i, size;
0237     void *va;
0238 
0239     for (i = 0; i < ARRAY_SIZE(tm_vdev->vrings); i++) {
0240         vring = &tm_vdev->vrings[i];
0241         vring->fifo = fifo;
0242         vring->num = MLXBF_TMFIFO_VRING_SIZE;
0243         vring->align = SMP_CACHE_BYTES;
0244         vring->index = i;
0245         vring->vdev_id = tm_vdev->vdev.id.device;
0246         dev = &tm_vdev->vdev.dev;
0247 
0248         size = vring_size(vring->num, vring->align);
0249         va = dma_alloc_coherent(dev->parent, size, &dma, GFP_KERNEL);
0250         if (!va) {
0251             mlxbf_tmfifo_free_vrings(fifo, tm_vdev);
0252             dev_err(dev->parent, "dma_alloc_coherent failed\n");
0253             return -ENOMEM;
0254         }
0255 
0256         vring->va = va;
0257         vring->dma = dma;
0258     }
0259 
0260     return 0;
0261 }
0262 
0263 /* Disable interrupts of the FIFO device. */
0264 static void mlxbf_tmfifo_disable_irqs(struct mlxbf_tmfifo *fifo)
0265 {
0266     int i, irq;
0267 
0268     for (i = 0; i < MLXBF_TM_MAX_IRQ; i++) {
0269         irq = fifo->irq_info[i].irq;
0270         fifo->irq_info[i].irq = 0;
0271         disable_irq(irq);
0272     }
0273 }
0274 
0275 /* Interrupt handler. */
0276 static irqreturn_t mlxbf_tmfifo_irq_handler(int irq, void *arg)
0277 {
0278     struct mlxbf_tmfifo_irq_info *irq_info = arg;
0279 
0280     if (!test_and_set_bit(irq_info->index, &irq_info->fifo->pend_events))
0281         schedule_work(&irq_info->fifo->work);
0282 
0283     return IRQ_HANDLED;
0284 }
0285 
0286 /* Get the next packet descriptor from the vring. */
0287 static struct vring_desc *
0288 mlxbf_tmfifo_get_next_desc(struct mlxbf_tmfifo_vring *vring)
0289 {
0290     const struct vring *vr = virtqueue_get_vring(vring->vq);
0291     struct virtio_device *vdev = vring->vq->vdev;
0292     unsigned int idx, head;
0293 
0294     if (vring->next_avail == virtio16_to_cpu(vdev, vr->avail->idx))
0295         return NULL;
0296 
0297     /* Make sure 'avail->idx' is visible already. */
0298     virtio_rmb(false);
0299 
0300     idx = vring->next_avail % vr->num;
0301     head = virtio16_to_cpu(vdev, vr->avail->ring[idx]);
0302     if (WARN_ON(head >= vr->num))
0303         return NULL;
0304 
0305     vring->next_avail++;
0306 
0307     return &vr->desc[head];
0308 }
0309 
0310 /* Release virtio descriptor. */
0311 static void mlxbf_tmfifo_release_desc(struct mlxbf_tmfifo_vring *vring,
0312                       struct vring_desc *desc, u32 len)
0313 {
0314     const struct vring *vr = virtqueue_get_vring(vring->vq);
0315     struct virtio_device *vdev = vring->vq->vdev;
0316     u16 idx, vr_idx;
0317 
0318     vr_idx = virtio16_to_cpu(vdev, vr->used->idx);
0319     idx = vr_idx % vr->num;
0320     vr->used->ring[idx].id = cpu_to_virtio32(vdev, desc - vr->desc);
0321     vr->used->ring[idx].len = cpu_to_virtio32(vdev, len);
0322 
0323     /*
0324      * Virtio could poll and check the 'idx' to decide whether the desc is
0325      * done or not. Add a memory barrier here to make sure the update above
0326      * completes before updating the idx.
0327      */
0328     virtio_mb(false);
0329     vr->used->idx = cpu_to_virtio16(vdev, vr_idx + 1);
0330 }
0331 
0332 /* Get the total length of the descriptor chain. */
0333 static u32 mlxbf_tmfifo_get_pkt_len(struct mlxbf_tmfifo_vring *vring,
0334                     struct vring_desc *desc)
0335 {
0336     const struct vring *vr = virtqueue_get_vring(vring->vq);
0337     struct virtio_device *vdev = vring->vq->vdev;
0338     u32 len = 0, idx;
0339 
0340     while (desc) {
0341         len += virtio32_to_cpu(vdev, desc->len);
0342         if (!(virtio16_to_cpu(vdev, desc->flags) & VRING_DESC_F_NEXT))
0343             break;
0344         idx = virtio16_to_cpu(vdev, desc->next);
0345         desc = &vr->desc[idx];
0346     }
0347 
0348     return len;
0349 }
0350 
0351 static void mlxbf_tmfifo_release_pending_pkt(struct mlxbf_tmfifo_vring *vring)
0352 {
0353     struct vring_desc *desc_head;
0354     u32 len = 0;
0355 
0356     if (vring->desc_head) {
0357         desc_head = vring->desc_head;
0358         len = vring->pkt_len;
0359     } else {
0360         desc_head = mlxbf_tmfifo_get_next_desc(vring);
0361         len = mlxbf_tmfifo_get_pkt_len(vring, desc_head);
0362     }
0363 
0364     if (desc_head)
0365         mlxbf_tmfifo_release_desc(vring, desc_head, len);
0366 
0367     vring->pkt_len = 0;
0368     vring->desc = NULL;
0369     vring->desc_head = NULL;
0370 }
0371 
0372 static void mlxbf_tmfifo_init_net_desc(struct mlxbf_tmfifo_vring *vring,
0373                        struct vring_desc *desc, bool is_rx)
0374 {
0375     struct virtio_device *vdev = vring->vq->vdev;
0376     struct virtio_net_hdr *net_hdr;
0377 
0378     net_hdr = phys_to_virt(virtio64_to_cpu(vdev, desc->addr));
0379     memset(net_hdr, 0, sizeof(*net_hdr));
0380 }
0381 
0382 /* Get and initialize the next packet. */
0383 static struct vring_desc *
0384 mlxbf_tmfifo_get_next_pkt(struct mlxbf_tmfifo_vring *vring, bool is_rx)
0385 {
0386     struct vring_desc *desc;
0387 
0388     desc = mlxbf_tmfifo_get_next_desc(vring);
0389     if (desc && is_rx && vring->vdev_id == VIRTIO_ID_NET)
0390         mlxbf_tmfifo_init_net_desc(vring, desc, is_rx);
0391 
0392     vring->desc_head = desc;
0393     vring->desc = desc;
0394 
0395     return desc;
0396 }
0397 
0398 /* House-keeping timer. */
0399 static void mlxbf_tmfifo_timer(struct timer_list *t)
0400 {
0401     struct mlxbf_tmfifo *fifo = container_of(t, struct mlxbf_tmfifo, timer);
0402     int rx, tx;
0403 
0404     rx = !test_and_set_bit(MLXBF_TM_RX_HWM_IRQ, &fifo->pend_events);
0405     tx = !test_and_set_bit(MLXBF_TM_TX_LWM_IRQ, &fifo->pend_events);
0406 
0407     if (rx || tx)
0408         schedule_work(&fifo->work);
0409 
0410     mod_timer(&fifo->timer, jiffies + MLXBF_TMFIFO_TIMER_INTERVAL);
0411 }
0412 
0413 /* Copy one console packet into the output buffer. */
0414 static void mlxbf_tmfifo_console_output_one(struct mlxbf_tmfifo_vdev *cons,
0415                         struct mlxbf_tmfifo_vring *vring,
0416                         struct vring_desc *desc)
0417 {
0418     const struct vring *vr = virtqueue_get_vring(vring->vq);
0419     struct virtio_device *vdev = &cons->vdev;
0420     u32 len, idx, seg;
0421     void *addr;
0422 
0423     while (desc) {
0424         addr = phys_to_virt(virtio64_to_cpu(vdev, desc->addr));
0425         len = virtio32_to_cpu(vdev, desc->len);
0426 
0427         seg = CIRC_SPACE_TO_END(cons->tx_buf.head, cons->tx_buf.tail,
0428                     MLXBF_TMFIFO_CON_TX_BUF_SIZE);
0429         if (len <= seg) {
0430             memcpy(cons->tx_buf.buf + cons->tx_buf.head, addr, len);
0431         } else {
0432             memcpy(cons->tx_buf.buf + cons->tx_buf.head, addr, seg);
0433             addr += seg;
0434             memcpy(cons->tx_buf.buf, addr, len - seg);
0435         }
0436         cons->tx_buf.head = (cons->tx_buf.head + len) %
0437             MLXBF_TMFIFO_CON_TX_BUF_SIZE;
0438 
0439         if (!(virtio16_to_cpu(vdev, desc->flags) & VRING_DESC_F_NEXT))
0440             break;
0441         idx = virtio16_to_cpu(vdev, desc->next);
0442         desc = &vr->desc[idx];
0443     }
0444 }
0445 
0446 /* Copy console data into the output buffer. */
0447 static void mlxbf_tmfifo_console_output(struct mlxbf_tmfifo_vdev *cons,
0448                     struct mlxbf_tmfifo_vring *vring)
0449 {
0450     struct vring_desc *desc;
0451     u32 len, avail;
0452 
0453     desc = mlxbf_tmfifo_get_next_desc(vring);
0454     while (desc) {
0455         /* Release the packet if not enough space. */
0456         len = mlxbf_tmfifo_get_pkt_len(vring, desc);
0457         avail = CIRC_SPACE(cons->tx_buf.head, cons->tx_buf.tail,
0458                    MLXBF_TMFIFO_CON_TX_BUF_SIZE);
0459         if (len + MLXBF_TMFIFO_CON_TX_BUF_RSV_SIZE > avail) {
0460             mlxbf_tmfifo_release_desc(vring, desc, len);
0461             break;
0462         }
0463 
0464         mlxbf_tmfifo_console_output_one(cons, vring, desc);
0465         mlxbf_tmfifo_release_desc(vring, desc, len);
0466         desc = mlxbf_tmfifo_get_next_desc(vring);
0467     }
0468 }
0469 
0470 /* Get the number of available words in Rx FIFO for receiving. */
0471 static int mlxbf_tmfifo_get_rx_avail(struct mlxbf_tmfifo *fifo)
0472 {
0473     u64 sts;
0474 
0475     sts = readq(fifo->rx_base + MLXBF_TMFIFO_RX_STS);
0476     return FIELD_GET(MLXBF_TMFIFO_RX_STS__COUNT_MASK, sts);
0477 }
0478 
0479 /* Get the number of available words in the TmFifo for sending. */
0480 static int mlxbf_tmfifo_get_tx_avail(struct mlxbf_tmfifo *fifo, int vdev_id)
0481 {
0482     int tx_reserve;
0483     u32 count;
0484     u64 sts;
0485 
0486     /* Reserve some room in FIFO for console messages. */
0487     if (vdev_id == VIRTIO_ID_NET)
0488         tx_reserve = fifo->tx_fifo_size / MLXBF_TMFIFO_RESERVE_RATIO;
0489     else
0490         tx_reserve = 1;
0491 
0492     sts = readq(fifo->tx_base + MLXBF_TMFIFO_TX_STS);
0493     count = FIELD_GET(MLXBF_TMFIFO_TX_STS__COUNT_MASK, sts);
0494     return fifo->tx_fifo_size - tx_reserve - count;
0495 }
0496 
0497 /* Console Tx (move data from the output buffer into the TmFifo). */
0498 static void mlxbf_tmfifo_console_tx(struct mlxbf_tmfifo *fifo, int avail)
0499 {
0500     struct mlxbf_tmfifo_msg_hdr hdr;
0501     struct mlxbf_tmfifo_vdev *cons;
0502     unsigned long flags;
0503     int size, seg;
0504     void *addr;
0505     u64 data;
0506 
0507     /* Return if not enough space available. */
0508     if (avail < MLXBF_TMFIFO_DATA_MIN_WORDS)
0509         return;
0510 
0511     cons = fifo->vdev[VIRTIO_ID_CONSOLE];
0512     if (!cons || !cons->tx_buf.buf)
0513         return;
0514 
0515     /* Return if no data to send. */
0516     size = CIRC_CNT(cons->tx_buf.head, cons->tx_buf.tail,
0517             MLXBF_TMFIFO_CON_TX_BUF_SIZE);
0518     if (size == 0)
0519         return;
0520 
0521     /* Adjust the size to available space. */
0522     if (size + sizeof(hdr) > avail * sizeof(u64))
0523         size = avail * sizeof(u64) - sizeof(hdr);
0524 
0525     /* Write header. */
0526     hdr.type = VIRTIO_ID_CONSOLE;
0527     hdr.len = htons(size);
0528     writeq(*(u64 *)&hdr, fifo->tx_base + MLXBF_TMFIFO_TX_DATA);
0529 
0530     /* Use spin-lock to protect the 'cons->tx_buf'. */
0531     spin_lock_irqsave(&fifo->spin_lock[0], flags);
0532 
0533     while (size > 0) {
0534         addr = cons->tx_buf.buf + cons->tx_buf.tail;
0535 
0536         seg = CIRC_CNT_TO_END(cons->tx_buf.head, cons->tx_buf.tail,
0537                       MLXBF_TMFIFO_CON_TX_BUF_SIZE);
0538         if (seg >= sizeof(u64)) {
0539             memcpy(&data, addr, sizeof(u64));
0540         } else {
0541             memcpy(&data, addr, seg);
0542             memcpy((u8 *)&data + seg, cons->tx_buf.buf,
0543                    sizeof(u64) - seg);
0544         }
0545         writeq(data, fifo->tx_base + MLXBF_TMFIFO_TX_DATA);
0546 
0547         if (size >= sizeof(u64)) {
0548             cons->tx_buf.tail = (cons->tx_buf.tail + sizeof(u64)) %
0549                 MLXBF_TMFIFO_CON_TX_BUF_SIZE;
0550             size -= sizeof(u64);
0551         } else {
0552             cons->tx_buf.tail = (cons->tx_buf.tail + size) %
0553                 MLXBF_TMFIFO_CON_TX_BUF_SIZE;
0554             size = 0;
0555         }
0556     }
0557 
0558     spin_unlock_irqrestore(&fifo->spin_lock[0], flags);
0559 }
0560 
0561 /* Rx/Tx one word in the descriptor buffer. */
0562 static void mlxbf_tmfifo_rxtx_word(struct mlxbf_tmfifo_vring *vring,
0563                    struct vring_desc *desc,
0564                    bool is_rx, int len)
0565 {
0566     struct virtio_device *vdev = vring->vq->vdev;
0567     struct mlxbf_tmfifo *fifo = vring->fifo;
0568     void *addr;
0569     u64 data;
0570 
0571     /* Get the buffer address of this desc. */
0572     addr = phys_to_virt(virtio64_to_cpu(vdev, desc->addr));
0573 
0574     /* Read a word from FIFO for Rx. */
0575     if (is_rx)
0576         data = readq(fifo->rx_base + MLXBF_TMFIFO_RX_DATA);
0577 
0578     if (vring->cur_len + sizeof(u64) <= len) {
0579         /* The whole word. */
0580         if (is_rx)
0581             memcpy(addr + vring->cur_len, &data, sizeof(u64));
0582         else
0583             memcpy(&data, addr + vring->cur_len, sizeof(u64));
0584         vring->cur_len += sizeof(u64);
0585     } else {
0586         /* Leftover bytes. */
0587         if (is_rx)
0588             memcpy(addr + vring->cur_len, &data,
0589                    len - vring->cur_len);
0590         else
0591             memcpy(&data, addr + vring->cur_len,
0592                    len - vring->cur_len);
0593         vring->cur_len = len;
0594     }
0595 
0596     /* Write the word into FIFO for Tx. */
0597     if (!is_rx)
0598         writeq(data, fifo->tx_base + MLXBF_TMFIFO_TX_DATA);
0599 }
0600 
0601 /*
0602  * Rx/Tx packet header.
0603  *
0604  * In Rx case, the packet might be found to belong to a different vring since
0605  * the TmFifo is shared by different services. In such case, the 'vring_change'
0606  * flag is set.
0607  */
0608 static void mlxbf_tmfifo_rxtx_header(struct mlxbf_tmfifo_vring *vring,
0609                      struct vring_desc *desc,
0610                      bool is_rx, bool *vring_change)
0611 {
0612     struct mlxbf_tmfifo *fifo = vring->fifo;
0613     struct virtio_net_config *config;
0614     struct mlxbf_tmfifo_msg_hdr hdr;
0615     int vdev_id, hdr_len;
0616 
0617     /* Read/Write packet header. */
0618     if (is_rx) {
0619         /* Drain one word from the FIFO. */
0620         *(u64 *)&hdr = readq(fifo->rx_base + MLXBF_TMFIFO_RX_DATA);
0621 
0622         /* Skip the length 0 packets (keepalive). */
0623         if (hdr.len == 0)
0624             return;
0625 
0626         /* Check packet type. */
0627         if (hdr.type == VIRTIO_ID_NET) {
0628             vdev_id = VIRTIO_ID_NET;
0629             hdr_len = sizeof(struct virtio_net_hdr);
0630             config = &fifo->vdev[vdev_id]->config.net;
0631             /* A legacy-only interface for now. */
0632             if (ntohs(hdr.len) >
0633                 __virtio16_to_cpu(virtio_legacy_is_little_endian(),
0634                           config->mtu) +
0635                 MLXBF_TMFIFO_NET_L2_OVERHEAD)
0636                 return;
0637         } else {
0638             vdev_id = VIRTIO_ID_CONSOLE;
0639             hdr_len = 0;
0640         }
0641 
0642         /*
0643          * Check whether the new packet still belongs to this vring.
0644          * If not, update the pkt_len of the new vring.
0645          */
0646         if (vdev_id != vring->vdev_id) {
0647             struct mlxbf_tmfifo_vdev *tm_dev2 = fifo->vdev[vdev_id];
0648 
0649             if (!tm_dev2)
0650                 return;
0651             vring->desc = desc;
0652             vring = &tm_dev2->vrings[MLXBF_TMFIFO_VRING_RX];
0653             *vring_change = true;
0654         }
0655         vring->pkt_len = ntohs(hdr.len) + hdr_len;
0656     } else {
0657         /* Network virtio has an extra header. */
0658         hdr_len = (vring->vdev_id == VIRTIO_ID_NET) ?
0659                sizeof(struct virtio_net_hdr) : 0;
0660         vring->pkt_len = mlxbf_tmfifo_get_pkt_len(vring, desc);
0661         hdr.type = (vring->vdev_id == VIRTIO_ID_NET) ?
0662                 VIRTIO_ID_NET : VIRTIO_ID_CONSOLE;
0663         hdr.len = htons(vring->pkt_len - hdr_len);
0664         writeq(*(u64 *)&hdr, fifo->tx_base + MLXBF_TMFIFO_TX_DATA);
0665     }
0666 
0667     vring->cur_len = hdr_len;
0668     vring->rem_len = vring->pkt_len;
0669     fifo->vring[is_rx] = vring;
0670 }
0671 
0672 /*
0673  * Rx/Tx one descriptor.
0674  *
0675  * Return true to indicate more data available.
0676  */
0677 static bool mlxbf_tmfifo_rxtx_one_desc(struct mlxbf_tmfifo_vring *vring,
0678                        bool is_rx, int *avail)
0679 {
0680     const struct vring *vr = virtqueue_get_vring(vring->vq);
0681     struct mlxbf_tmfifo *fifo = vring->fifo;
0682     struct virtio_device *vdev;
0683     bool vring_change = false;
0684     struct vring_desc *desc;
0685     unsigned long flags;
0686     u32 len, idx;
0687 
0688     vdev = &fifo->vdev[vring->vdev_id]->vdev;
0689 
0690     /* Get the descriptor of the next packet. */
0691     if (!vring->desc) {
0692         desc = mlxbf_tmfifo_get_next_pkt(vring, is_rx);
0693         if (!desc)
0694             return false;
0695     } else {
0696         desc = vring->desc;
0697     }
0698 
0699     /* Beginning of a packet. Start to Rx/Tx packet header. */
0700     if (vring->pkt_len == 0) {
0701         mlxbf_tmfifo_rxtx_header(vring, desc, is_rx, &vring_change);
0702         (*avail)--;
0703 
0704         /* Return if new packet is for another ring. */
0705         if (vring_change)
0706             return false;
0707         goto mlxbf_tmfifo_desc_done;
0708     }
0709 
0710     /* Get the length of this desc. */
0711     len = virtio32_to_cpu(vdev, desc->len);
0712     if (len > vring->rem_len)
0713         len = vring->rem_len;
0714 
0715     /* Rx/Tx one word (8 bytes) if not done. */
0716     if (vring->cur_len < len) {
0717         mlxbf_tmfifo_rxtx_word(vring, desc, is_rx, len);
0718         (*avail)--;
0719     }
0720 
0721     /* Check again whether it's done. */
0722     if (vring->cur_len == len) {
0723         vring->cur_len = 0;
0724         vring->rem_len -= len;
0725 
0726         /* Get the next desc on the chain. */
0727         if (vring->rem_len > 0 &&
0728             (virtio16_to_cpu(vdev, desc->flags) & VRING_DESC_F_NEXT)) {
0729             idx = virtio16_to_cpu(vdev, desc->next);
0730             desc = &vr->desc[idx];
0731             goto mlxbf_tmfifo_desc_done;
0732         }
0733 
0734         /* Done and release the pending packet. */
0735         mlxbf_tmfifo_release_pending_pkt(vring);
0736         desc = NULL;
0737         fifo->vring[is_rx] = NULL;
0738 
0739         /*
0740          * Make sure the load/store are in order before
0741          * returning back to virtio.
0742          */
0743         virtio_mb(false);
0744 
0745         /* Notify upper layer that packet is done. */
0746         spin_lock_irqsave(&fifo->spin_lock[is_rx], flags);
0747         vring_interrupt(0, vring->vq);
0748         spin_unlock_irqrestore(&fifo->spin_lock[is_rx], flags);
0749     }
0750 
0751 mlxbf_tmfifo_desc_done:
0752     /* Save the current desc. */
0753     vring->desc = desc;
0754 
0755     return true;
0756 }
0757 
0758 /* Rx & Tx processing of a queue. */
0759 static void mlxbf_tmfifo_rxtx(struct mlxbf_tmfifo_vring *vring, bool is_rx)
0760 {
0761     int avail = 0, devid = vring->vdev_id;
0762     struct mlxbf_tmfifo *fifo;
0763     bool more;
0764 
0765     fifo = vring->fifo;
0766 
0767     /* Return if vdev is not ready. */
0768     if (!fifo->vdev[devid])
0769         return;
0770 
0771     /* Return if another vring is running. */
0772     if (fifo->vring[is_rx] && fifo->vring[is_rx] != vring)
0773         return;
0774 
0775     /* Only handle console and network for now. */
0776     if (WARN_ON(devid != VIRTIO_ID_NET && devid != VIRTIO_ID_CONSOLE))
0777         return;
0778 
0779     do {
0780         /* Get available FIFO space. */
0781         if (avail == 0) {
0782             if (is_rx)
0783                 avail = mlxbf_tmfifo_get_rx_avail(fifo);
0784             else
0785                 avail = mlxbf_tmfifo_get_tx_avail(fifo, devid);
0786             if (avail <= 0)
0787                 break;
0788         }
0789 
0790         /* Console output always comes from the Tx buffer. */
0791         if (!is_rx && devid == VIRTIO_ID_CONSOLE) {
0792             mlxbf_tmfifo_console_tx(fifo, avail);
0793             break;
0794         }
0795 
0796         /* Handle one descriptor. */
0797         more = mlxbf_tmfifo_rxtx_one_desc(vring, is_rx, &avail);
0798     } while (more);
0799 }
0800 
0801 /* Handle Rx or Tx queues. */
0802 static void mlxbf_tmfifo_work_rxtx(struct mlxbf_tmfifo *fifo, int queue_id,
0803                    int irq_id, bool is_rx)
0804 {
0805     struct mlxbf_tmfifo_vdev *tm_vdev;
0806     struct mlxbf_tmfifo_vring *vring;
0807     int i;
0808 
0809     if (!test_and_clear_bit(irq_id, &fifo->pend_events) ||
0810         !fifo->irq_info[irq_id].irq)
0811         return;
0812 
0813     for (i = 0; i < MLXBF_TMFIFO_VDEV_MAX; i++) {
0814         tm_vdev = fifo->vdev[i];
0815         if (tm_vdev) {
0816             vring = &tm_vdev->vrings[queue_id];
0817             if (vring->vq)
0818                 mlxbf_tmfifo_rxtx(vring, is_rx);
0819         }
0820     }
0821 }
0822 
0823 /* Work handler for Rx and Tx case. */
0824 static void mlxbf_tmfifo_work_handler(struct work_struct *work)
0825 {
0826     struct mlxbf_tmfifo *fifo;
0827 
0828     fifo = container_of(work, struct mlxbf_tmfifo, work);
0829     if (!fifo->is_ready)
0830         return;
0831 
0832     mutex_lock(&fifo->lock);
0833 
0834     /* Tx (Send data to the TmFifo). */
0835     mlxbf_tmfifo_work_rxtx(fifo, MLXBF_TMFIFO_VRING_TX,
0836                    MLXBF_TM_TX_LWM_IRQ, false);
0837 
0838     /* Rx (Receive data from the TmFifo). */
0839     mlxbf_tmfifo_work_rxtx(fifo, MLXBF_TMFIFO_VRING_RX,
0840                    MLXBF_TM_RX_HWM_IRQ, true);
0841 
0842     mutex_unlock(&fifo->lock);
0843 }
0844 
0845 /* The notify function is called when new buffers are posted. */
0846 static bool mlxbf_tmfifo_virtio_notify(struct virtqueue *vq)
0847 {
0848     struct mlxbf_tmfifo_vring *vring = vq->priv;
0849     struct mlxbf_tmfifo_vdev *tm_vdev;
0850     struct mlxbf_tmfifo *fifo;
0851     unsigned long flags;
0852 
0853     fifo = vring->fifo;
0854 
0855     /*
0856      * Virtio maintains vrings in pairs, even number ring for Rx
0857      * and odd number ring for Tx.
0858      */
0859     if (vring->index & BIT(0)) {
0860         /*
0861          * Console could make blocking call with interrupts disabled.
0862          * In such case, the vring needs to be served right away. For
0863          * other cases, just set the TX LWM bit to start Tx in the
0864          * worker handler.
0865          */
0866         if (vring->vdev_id == VIRTIO_ID_CONSOLE) {
0867             spin_lock_irqsave(&fifo->spin_lock[0], flags);
0868             tm_vdev = fifo->vdev[VIRTIO_ID_CONSOLE];
0869             mlxbf_tmfifo_console_output(tm_vdev, vring);
0870             spin_unlock_irqrestore(&fifo->spin_lock[0], flags);
0871         } else if (test_and_set_bit(MLXBF_TM_TX_LWM_IRQ,
0872                         &fifo->pend_events)) {
0873             return true;
0874         }
0875     } else {
0876         if (test_and_set_bit(MLXBF_TM_RX_HWM_IRQ, &fifo->pend_events))
0877             return true;
0878     }
0879 
0880     schedule_work(&fifo->work);
0881 
0882     return true;
0883 }
0884 
0885 /* Get the array of feature bits for this device. */
0886 static u64 mlxbf_tmfifo_virtio_get_features(struct virtio_device *vdev)
0887 {
0888     struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
0889 
0890     return tm_vdev->features;
0891 }
0892 
0893 /* Confirm device features to use. */
0894 static int mlxbf_tmfifo_virtio_finalize_features(struct virtio_device *vdev)
0895 {
0896     struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
0897 
0898     tm_vdev->features = vdev->features;
0899 
0900     return 0;
0901 }
0902 
0903 /* Free virtqueues found by find_vqs(). */
0904 static void mlxbf_tmfifo_virtio_del_vqs(struct virtio_device *vdev)
0905 {
0906     struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
0907     struct mlxbf_tmfifo_vring *vring;
0908     struct virtqueue *vq;
0909     int i;
0910 
0911     for (i = 0; i < ARRAY_SIZE(tm_vdev->vrings); i++) {
0912         vring = &tm_vdev->vrings[i];
0913 
0914         /* Release the pending packet. */
0915         if (vring->desc)
0916             mlxbf_tmfifo_release_pending_pkt(vring);
0917         vq = vring->vq;
0918         if (vq) {
0919             vring->vq = NULL;
0920             vring_del_virtqueue(vq);
0921         }
0922     }
0923 }
0924 
0925 /* Create and initialize the virtual queues. */
0926 static int mlxbf_tmfifo_virtio_find_vqs(struct virtio_device *vdev,
0927                     unsigned int nvqs,
0928                     struct virtqueue *vqs[],
0929                     vq_callback_t *callbacks[],
0930                     const char * const names[],
0931                     const bool *ctx,
0932                     struct irq_affinity *desc)
0933 {
0934     struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
0935     struct mlxbf_tmfifo_vring *vring;
0936     struct virtqueue *vq;
0937     int i, ret, size;
0938 
0939     if (nvqs > ARRAY_SIZE(tm_vdev->vrings))
0940         return -EINVAL;
0941 
0942     for (i = 0; i < nvqs; ++i) {
0943         if (!names[i]) {
0944             ret = -EINVAL;
0945             goto error;
0946         }
0947         vring = &tm_vdev->vrings[i];
0948 
0949         /* zero vring */
0950         size = vring_size(vring->num, vring->align);
0951         memset(vring->va, 0, size);
0952         vq = vring_new_virtqueue(i, vring->num, vring->align, vdev,
0953                      false, false, vring->va,
0954                      mlxbf_tmfifo_virtio_notify,
0955                      callbacks[i], names[i]);
0956         if (!vq) {
0957             dev_err(&vdev->dev, "vring_new_virtqueue failed\n");
0958             ret = -ENOMEM;
0959             goto error;
0960         }
0961 
0962         vq->num_max = vring->num;
0963 
0964         vqs[i] = vq;
0965         vring->vq = vq;
0966         vq->priv = vring;
0967     }
0968 
0969     return 0;
0970 
0971 error:
0972     mlxbf_tmfifo_virtio_del_vqs(vdev);
0973     return ret;
0974 }
0975 
0976 /* Read the status byte. */
0977 static u8 mlxbf_tmfifo_virtio_get_status(struct virtio_device *vdev)
0978 {
0979     struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
0980 
0981     return tm_vdev->status;
0982 }
0983 
0984 /* Write the status byte. */
0985 static void mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
0986                        u8 status)
0987 {
0988     struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
0989 
0990     tm_vdev->status = status;
0991 }
0992 
0993 /* Reset the device. Not much here for now. */
0994 static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
0995 {
0996     struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
0997 
0998     tm_vdev->status = 0;
0999 }
1000 
1001 /* Read the value of a configuration field. */
1002 static void mlxbf_tmfifo_virtio_get(struct virtio_device *vdev,
1003                     unsigned int offset,
1004                     void *buf,
1005                     unsigned int len)
1006 {
1007     struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
1008 
1009     if ((u64)offset + len > sizeof(tm_vdev->config))
1010         return;
1011 
1012     memcpy(buf, (u8 *)&tm_vdev->config + offset, len);
1013 }
1014 
1015 /* Write the value of a configuration field. */
1016 static void mlxbf_tmfifo_virtio_set(struct virtio_device *vdev,
1017                     unsigned int offset,
1018                     const void *buf,
1019                     unsigned int len)
1020 {
1021     struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
1022 
1023     if ((u64)offset + len > sizeof(tm_vdev->config))
1024         return;
1025 
1026     memcpy((u8 *)&tm_vdev->config + offset, buf, len);
1027 }
1028 
1029 static void tmfifo_virtio_dev_release(struct device *device)
1030 {
1031     struct virtio_device *vdev =
1032             container_of(device, struct virtio_device, dev);
1033     struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
1034 
1035     kfree(tm_vdev);
1036 }
1037 
1038 /* Virtio config operations. */
1039 static const struct virtio_config_ops mlxbf_tmfifo_virtio_config_ops = {
1040     .get_features = mlxbf_tmfifo_virtio_get_features,
1041     .finalize_features = mlxbf_tmfifo_virtio_finalize_features,
1042     .find_vqs = mlxbf_tmfifo_virtio_find_vqs,
1043     .del_vqs = mlxbf_tmfifo_virtio_del_vqs,
1044     .reset = mlxbf_tmfifo_virtio_reset,
1045     .set_status = mlxbf_tmfifo_virtio_set_status,
1046     .get_status = mlxbf_tmfifo_virtio_get_status,
1047     .get = mlxbf_tmfifo_virtio_get,
1048     .set = mlxbf_tmfifo_virtio_set,
1049 };
1050 
1051 /* Create vdev for the FIFO. */
1052 static int mlxbf_tmfifo_create_vdev(struct device *dev,
1053                     struct mlxbf_tmfifo *fifo,
1054                     int vdev_id, u64 features,
1055                     void *config, u32 size)
1056 {
1057     struct mlxbf_tmfifo_vdev *tm_vdev, *reg_dev = NULL;
1058     int ret;
1059 
1060     mutex_lock(&fifo->lock);
1061 
1062     tm_vdev = fifo->vdev[vdev_id];
1063     if (tm_vdev) {
1064         dev_err(dev, "vdev %d already exists\n", vdev_id);
1065         ret = -EEXIST;
1066         goto fail;
1067     }
1068 
1069     tm_vdev = kzalloc(sizeof(*tm_vdev), GFP_KERNEL);
1070     if (!tm_vdev) {
1071         ret = -ENOMEM;
1072         goto fail;
1073     }
1074 
1075     tm_vdev->vdev.id.device = vdev_id;
1076     tm_vdev->vdev.config = &mlxbf_tmfifo_virtio_config_ops;
1077     tm_vdev->vdev.dev.parent = dev;
1078     tm_vdev->vdev.dev.release = tmfifo_virtio_dev_release;
1079     tm_vdev->features = features;
1080     if (config)
1081         memcpy(&tm_vdev->config, config, size);
1082 
1083     if (mlxbf_tmfifo_alloc_vrings(fifo, tm_vdev)) {
1084         dev_err(dev, "unable to allocate vring\n");
1085         ret = -ENOMEM;
1086         goto vdev_fail;
1087     }
1088 
1089     /* Allocate an output buffer for the console device. */
1090     if (vdev_id == VIRTIO_ID_CONSOLE)
1091         tm_vdev->tx_buf.buf = devm_kmalloc(dev,
1092                            MLXBF_TMFIFO_CON_TX_BUF_SIZE,
1093                            GFP_KERNEL);
1094     fifo->vdev[vdev_id] = tm_vdev;
1095 
1096     /* Register the virtio device. */
1097     ret = register_virtio_device(&tm_vdev->vdev);
1098     reg_dev = tm_vdev;
1099     if (ret) {
1100         dev_err(dev, "register_virtio_device failed\n");
1101         goto vdev_fail;
1102     }
1103 
1104     mutex_unlock(&fifo->lock);
1105     return 0;
1106 
1107 vdev_fail:
1108     mlxbf_tmfifo_free_vrings(fifo, tm_vdev);
1109     fifo->vdev[vdev_id] = NULL;
1110     if (reg_dev)
1111         put_device(&tm_vdev->vdev.dev);
1112     else
1113         kfree(tm_vdev);
1114 fail:
1115     mutex_unlock(&fifo->lock);
1116     return ret;
1117 }
1118 
1119 /* Delete vdev for the FIFO. */
1120 static int mlxbf_tmfifo_delete_vdev(struct mlxbf_tmfifo *fifo, int vdev_id)
1121 {
1122     struct mlxbf_tmfifo_vdev *tm_vdev;
1123 
1124     mutex_lock(&fifo->lock);
1125 
1126     /* Unregister vdev. */
1127     tm_vdev = fifo->vdev[vdev_id];
1128     if (tm_vdev) {
1129         unregister_virtio_device(&tm_vdev->vdev);
1130         mlxbf_tmfifo_free_vrings(fifo, tm_vdev);
1131         fifo->vdev[vdev_id] = NULL;
1132     }
1133 
1134     mutex_unlock(&fifo->lock);
1135 
1136     return 0;
1137 }
1138 
1139 /* Read the configured network MAC address from efi variable. */
1140 static void mlxbf_tmfifo_get_cfg_mac(u8 *mac)
1141 {
1142     efi_guid_t guid = EFI_GLOBAL_VARIABLE_GUID;
1143     unsigned long size = ETH_ALEN;
1144     u8 buf[ETH_ALEN];
1145     efi_status_t rc;
1146 
1147     rc = efi.get_variable(mlxbf_tmfifo_efi_name, &guid, NULL, &size, buf);
1148     if (rc == EFI_SUCCESS && size == ETH_ALEN)
1149         ether_addr_copy(mac, buf);
1150     else
1151         ether_addr_copy(mac, mlxbf_tmfifo_net_default_mac);
1152 }
1153 
1154 /* Set TmFifo thresolds which is used to trigger interrupts. */
1155 static void mlxbf_tmfifo_set_threshold(struct mlxbf_tmfifo *fifo)
1156 {
1157     u64 ctl;
1158 
1159     /* Get Tx FIFO size and set the low/high watermark. */
1160     ctl = readq(fifo->tx_base + MLXBF_TMFIFO_TX_CTL);
1161     fifo->tx_fifo_size =
1162         FIELD_GET(MLXBF_TMFIFO_TX_CTL__MAX_ENTRIES_MASK, ctl);
1163     ctl = (ctl & ~MLXBF_TMFIFO_TX_CTL__LWM_MASK) |
1164         FIELD_PREP(MLXBF_TMFIFO_TX_CTL__LWM_MASK,
1165                fifo->tx_fifo_size / 2);
1166     ctl = (ctl & ~MLXBF_TMFIFO_TX_CTL__HWM_MASK) |
1167         FIELD_PREP(MLXBF_TMFIFO_TX_CTL__HWM_MASK,
1168                fifo->tx_fifo_size - 1);
1169     writeq(ctl, fifo->tx_base + MLXBF_TMFIFO_TX_CTL);
1170 
1171     /* Get Rx FIFO size and set the low/high watermark. */
1172     ctl = readq(fifo->rx_base + MLXBF_TMFIFO_RX_CTL);
1173     fifo->rx_fifo_size =
1174         FIELD_GET(MLXBF_TMFIFO_RX_CTL__MAX_ENTRIES_MASK, ctl);
1175     ctl = (ctl & ~MLXBF_TMFIFO_RX_CTL__LWM_MASK) |
1176         FIELD_PREP(MLXBF_TMFIFO_RX_CTL__LWM_MASK, 0);
1177     ctl = (ctl & ~MLXBF_TMFIFO_RX_CTL__HWM_MASK) |
1178         FIELD_PREP(MLXBF_TMFIFO_RX_CTL__HWM_MASK, 1);
1179     writeq(ctl, fifo->rx_base + MLXBF_TMFIFO_RX_CTL);
1180 }
1181 
1182 static void mlxbf_tmfifo_cleanup(struct mlxbf_tmfifo *fifo)
1183 {
1184     int i;
1185 
1186     fifo->is_ready = false;
1187     del_timer_sync(&fifo->timer);
1188     mlxbf_tmfifo_disable_irqs(fifo);
1189     cancel_work_sync(&fifo->work);
1190     for (i = 0; i < MLXBF_TMFIFO_VDEV_MAX; i++)
1191         mlxbf_tmfifo_delete_vdev(fifo, i);
1192 }
1193 
1194 /* Probe the TMFIFO. */
1195 static int mlxbf_tmfifo_probe(struct platform_device *pdev)
1196 {
1197     struct virtio_net_config net_config;
1198     struct device *dev = &pdev->dev;
1199     struct mlxbf_tmfifo *fifo;
1200     int i, rc;
1201 
1202     fifo = devm_kzalloc(dev, sizeof(*fifo), GFP_KERNEL);
1203     if (!fifo)
1204         return -ENOMEM;
1205 
1206     spin_lock_init(&fifo->spin_lock[0]);
1207     spin_lock_init(&fifo->spin_lock[1]);
1208     INIT_WORK(&fifo->work, mlxbf_tmfifo_work_handler);
1209     mutex_init(&fifo->lock);
1210 
1211     /* Get the resource of the Rx FIFO. */
1212     fifo->rx_base = devm_platform_ioremap_resource(pdev, 0);
1213     if (IS_ERR(fifo->rx_base))
1214         return PTR_ERR(fifo->rx_base);
1215 
1216     /* Get the resource of the Tx FIFO. */
1217     fifo->tx_base = devm_platform_ioremap_resource(pdev, 1);
1218     if (IS_ERR(fifo->tx_base))
1219         return PTR_ERR(fifo->tx_base);
1220 
1221     platform_set_drvdata(pdev, fifo);
1222 
1223     timer_setup(&fifo->timer, mlxbf_tmfifo_timer, 0);
1224 
1225     for (i = 0; i < MLXBF_TM_MAX_IRQ; i++) {
1226         fifo->irq_info[i].index = i;
1227         fifo->irq_info[i].fifo = fifo;
1228         fifo->irq_info[i].irq = platform_get_irq(pdev, i);
1229         rc = devm_request_irq(dev, fifo->irq_info[i].irq,
1230                       mlxbf_tmfifo_irq_handler, 0,
1231                       "tmfifo", &fifo->irq_info[i]);
1232         if (rc) {
1233             dev_err(dev, "devm_request_irq failed\n");
1234             fifo->irq_info[i].irq = 0;
1235             return rc;
1236         }
1237     }
1238 
1239     mlxbf_tmfifo_set_threshold(fifo);
1240 
1241     /* Create the console vdev. */
1242     rc = mlxbf_tmfifo_create_vdev(dev, fifo, VIRTIO_ID_CONSOLE, 0, NULL, 0);
1243     if (rc)
1244         goto fail;
1245 
1246     /* Create the network vdev. */
1247     memset(&net_config, 0, sizeof(net_config));
1248 
1249     /* A legacy-only interface for now. */
1250     net_config.mtu = __cpu_to_virtio16(virtio_legacy_is_little_endian(),
1251                        ETH_DATA_LEN);
1252     net_config.status = __cpu_to_virtio16(virtio_legacy_is_little_endian(),
1253                           VIRTIO_NET_S_LINK_UP);
1254     mlxbf_tmfifo_get_cfg_mac(net_config.mac);
1255     rc = mlxbf_tmfifo_create_vdev(dev, fifo, VIRTIO_ID_NET,
1256                       MLXBF_TMFIFO_NET_FEATURES, &net_config,
1257                       sizeof(net_config));
1258     if (rc)
1259         goto fail;
1260 
1261     mod_timer(&fifo->timer, jiffies + MLXBF_TMFIFO_TIMER_INTERVAL);
1262 
1263     fifo->is_ready = true;
1264     return 0;
1265 
1266 fail:
1267     mlxbf_tmfifo_cleanup(fifo);
1268     return rc;
1269 }
1270 
1271 /* Device remove function. */
1272 static int mlxbf_tmfifo_remove(struct platform_device *pdev)
1273 {
1274     struct mlxbf_tmfifo *fifo = platform_get_drvdata(pdev);
1275 
1276     mlxbf_tmfifo_cleanup(fifo);
1277 
1278     return 0;
1279 }
1280 
1281 static const struct acpi_device_id mlxbf_tmfifo_acpi_match[] = {
1282     { "MLNXBF01", 0 },
1283     {}
1284 };
1285 MODULE_DEVICE_TABLE(acpi, mlxbf_tmfifo_acpi_match);
1286 
1287 static struct platform_driver mlxbf_tmfifo_driver = {
1288     .probe = mlxbf_tmfifo_probe,
1289     .remove = mlxbf_tmfifo_remove,
1290     .driver = {
1291         .name = "bf-tmfifo",
1292         .acpi_match_table = mlxbf_tmfifo_acpi_match,
1293     },
1294 };
1295 
1296 module_platform_driver(mlxbf_tmfifo_driver);
1297 
1298 MODULE_DESCRIPTION("Mellanox BlueField SoC TmFifo Driver");
1299 MODULE_LICENSE("GPL v2");
1300 MODULE_AUTHOR("Mellanox Technologies");
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!