amazon/ena/ena_eth_com.h

0001 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
0002 /*
0003  * Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
0004  */
0005
0006 #ifndef ENA_ETH_COM_H_
0007 #define ENA_ETH_COM_H_
0008
0009 #include "ena_com.h"
0010
0011 /* head update threshold in units of (queue size / ENA_COMP_HEAD_THRESH) */
0012 #define ENA_COMP_HEAD_THRESH 4
0013
0014 struct ena_com_tx_ctx {
0015     struct ena_com_tx_meta ena_meta;
0016     struct ena_com_buf *ena_bufs;
0017     /* For LLQ, header buffer - pushed to the device mem space */
0018     void *push_header;
0019
0020     enum ena_eth_io_l3_proto_index l3_proto;
0021     enum ena_eth_io_l4_proto_index l4_proto;
0022     u16 num_bufs;
0023     u16 req_id;
0024     /* For regular queue, indicate the size of the header
0025      * For LLQ, indicate the size of the pushed buffer
0026      */
0027     u16 header_len;
0028
0029     u8 meta_valid;
0030     u8 tso_enable;
0031     u8 l3_csum_enable;
0032     u8 l4_csum_enable;
0033     u8 l4_csum_partial;
0034     u8 df; /* Don't fragment */
0035 };
0036
0037 struct ena_com_rx_ctx {
0038     struct ena_com_rx_buf_info *ena_bufs;
0039     enum ena_eth_io_l3_proto_index l3_proto;
0040     enum ena_eth_io_l4_proto_index l4_proto;
0041     bool l3_csum_err;
0042     bool l4_csum_err;
0043     u8 l4_csum_checked;
0044     /* fragmented packet */
0045     bool frag;
0046     u32 hash;
0047     u16 descs;
0048     int max_bufs;
0049     u8 pkt_offset;
0050 };
0051
0052 int ena_com_prepare_tx(struct ena_com_io_sq *io_sq,
0053                struct ena_com_tx_ctx *ena_tx_ctx,
0054                int *nb_hw_desc);
0055
0056 int ena_com_rx_pkt(struct ena_com_io_cq *io_cq,
0057            struct ena_com_io_sq *io_sq,
0058            struct ena_com_rx_ctx *ena_rx_ctx);
0059
0060 int ena_com_add_single_rx_desc(struct ena_com_io_sq *io_sq,
0061                    struct ena_com_buf *ena_buf,
0062                    u16 req_id);
0063
0064 bool ena_com_cq_empty(struct ena_com_io_cq *io_cq);
0065
0066 static inline void ena_com_unmask_intr(struct ena_com_io_cq *io_cq,
0067                        struct ena_eth_io_intr_reg *intr_reg)
0068 {
0069     writel(intr_reg->intr_control, io_cq->unmask_reg);
0070 }
0071
0072 static inline int ena_com_free_q_entries(struct ena_com_io_sq *io_sq)
0073 {
0074     u16 tail, next_to_comp, cnt;
0075
0076     next_to_comp = io_sq->next_to_comp;
0077     tail = io_sq->tail;
0078     cnt = tail - next_to_comp;
0079
0080     return io_sq->q_depth - 1 - cnt;
0081 }
0082
0083 /* Check if the submission queue has enough space to hold required_buffers */
0084 static inline bool ena_com_sq_have_enough_space(struct ena_com_io_sq *io_sq,
0085                         u16 required_buffers)
0086 {
0087     int temp;
0088
0089     if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)
0090         return ena_com_free_q_entries(io_sq) >= required_buffers;
0091
0092     /* This calculation doesn't need to be 100% accurate. So to reduce
0093      * the calculation overhead just Subtract 2 lines from the free descs
0094      * (one for the header line and one to compensate the devision
0095      * down calculation.
0096      */
0097     temp = required_buffers / io_sq->llq_info.descs_per_entry + 2;
0098
0099     return ena_com_free_q_entries(io_sq) > temp;
0100 }
0101
0102 static inline bool ena_com_meta_desc_changed(struct ena_com_io_sq *io_sq,
0103                          struct ena_com_tx_ctx *ena_tx_ctx)
0104 {
0105     if (!ena_tx_ctx->meta_valid)
0106         return false;
0107
0108     return !!memcmp(&io_sq->cached_tx_meta,
0109             &ena_tx_ctx->ena_meta,
0110             sizeof(struct ena_com_tx_meta));
0111 }
0112
0113 static inline bool is_llq_max_tx_burst_exists(struct ena_com_io_sq *io_sq)
0114 {
0115     return (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) &&
0116            io_sq->llq_info.max_entries_in_tx_burst > 0;
0117 }
0118
0119 static inline bool ena_com_is_doorbell_needed(struct ena_com_io_sq *io_sq,
0120                           struct ena_com_tx_ctx *ena_tx_ctx)
0121 {
0122     struct ena_com_llq_info *llq_info;
0123     int descs_after_first_entry;
0124     int num_entries_needed = 1;
0125     u16 num_descs;
0126
0127     if (!is_llq_max_tx_burst_exists(io_sq))
0128         return false;
0129
0130     llq_info = &io_sq->llq_info;
0131     num_descs = ena_tx_ctx->num_bufs;
0132
0133     if (llq_info->disable_meta_caching ||
0134         unlikely(ena_com_meta_desc_changed(io_sq, ena_tx_ctx)))
0135         ++num_descs;
0136
0137     if (num_descs > llq_info->descs_num_before_header) {
0138         descs_after_first_entry = num_descs - llq_info->descs_num_before_header;
0139         num_entries_needed += DIV_ROUND_UP(descs_after_first_entry,
0140                            llq_info->descs_per_entry);
0141     }
0142
0143     netdev_dbg(ena_com_io_sq_to_ena_dev(io_sq)->net_device,
0144            "Queue: %d num_descs: %d num_entries_needed: %d\n",
0145            io_sq->qid, num_descs, num_entries_needed);
0146
0147     return num_entries_needed > io_sq->entries_in_tx_burst_left;
0148 }
0149
0150 static inline int ena_com_write_sq_doorbell(struct ena_com_io_sq *io_sq)
0151 {
0152     u16 max_entries_in_tx_burst = io_sq->llq_info.max_entries_in_tx_burst;
0153     u16 tail = io_sq->tail;
0154
0155     netdev_dbg(ena_com_io_sq_to_ena_dev(io_sq)->net_device,
0156            "Write submission queue doorbell for queue: %d tail: %d\n",
0157            io_sq->qid, tail);
0158
0159     writel(tail, io_sq->db_addr);
0160
0161     if (is_llq_max_tx_burst_exists(io_sq)) {
0162         netdev_dbg(ena_com_io_sq_to_ena_dev(io_sq)->net_device,
0163                "Reset available entries in tx burst for queue %d to %d\n",
0164                io_sq->qid, max_entries_in_tx_burst);
0165         io_sq->entries_in_tx_burst_left = max_entries_in_tx_burst;
0166     }
0167
0168     return 0;
0169 }
0170
0171 static inline int ena_com_update_dev_comp_head(struct ena_com_io_cq *io_cq)
0172 {
0173     u16 unreported_comp, head;
0174     bool need_update;
0175
0176     if (unlikely(io_cq->cq_head_db_reg)) {
0177         head = io_cq->head;
0178         unreported_comp = head - io_cq->last_head_update;
0179         need_update = unreported_comp > (io_cq->q_depth / ENA_COMP_HEAD_THRESH);
0180
0181         if (unlikely(need_update)) {
0182             netdev_dbg(ena_com_io_cq_to_ena_dev(io_cq)->net_device,
0183                    "Write completion queue doorbell for queue %d: head: %d\n",
0184                    io_cq->qid, head);
0185             writel(head, io_cq->cq_head_db_reg);
0186             io_cq->last_head_update = head;
0187         }
0188     }
0189
0190     return 0;
0191 }
0192
0193 static inline void ena_com_update_numa_node(struct ena_com_io_cq *io_cq,
0194                         u8 numa_node)
0195 {
0196     struct ena_eth_io_numa_node_cfg_reg numa_cfg;
0197
0198     if (!io_cq->numa_node_cfg_reg)
0199         return;
0200
0201     numa_cfg.numa_cfg = (numa_node & ENA_ETH_IO_NUMA_NODE_CFG_REG_NUMA_MASK)
0202         | ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_MASK;
0203
0204     writel(numa_cfg.numa_cfg, io_cq->numa_node_cfg_reg);
0205 }
0206
0207 static inline void ena_com_comp_ack(struct ena_com_io_sq *io_sq, u16 elem)
0208 {
0209     io_sq->next_to_comp += elem;
0210 }
0211
0212 static inline void ena_com_cq_inc_head(struct ena_com_io_cq *io_cq)
0213 {
0214     io_cq->head++;
0215
0216     /* Switch phase bit in case of wrap around */
0217     if (unlikely((io_cq->head & (io_cq->q_depth - 1)) == 0))
0218         io_cq->phase ^= 1;
0219 }
0220
0221 static inline int ena_com_tx_comp_req_id_get(struct ena_com_io_cq *io_cq,
0222                          u16 *req_id)
0223 {
0224     u8 expected_phase, cdesc_phase;
0225     struct ena_eth_io_tx_cdesc *cdesc;
0226     u16 masked_head;
0227
0228     masked_head = io_cq->head & (io_cq->q_depth - 1);
0229     expected_phase = io_cq->phase;
0230
0231     cdesc = (struct ena_eth_io_tx_cdesc *)
0232         ((uintptr_t)io_cq->cdesc_addr.virt_addr +
0233         (masked_head * io_cq->cdesc_entry_size_in_bytes));
0234
0235     /* When the current completion descriptor phase isn't the same as the
0236      * expected, it mean that the device still didn't update
0237      * this completion.
0238      */
0239     cdesc_phase = READ_ONCE(cdesc->flags) & ENA_ETH_IO_TX_CDESC_PHASE_MASK;
0240     if (cdesc_phase != expected_phase)
0241         return -EAGAIN;
0242
0243     dma_rmb();
0244
0245     *req_id = READ_ONCE(cdesc->req_id);
0246     if (unlikely(*req_id >= io_cq->q_depth)) {
0247         netdev_err(ena_com_io_cq_to_ena_dev(io_cq)->net_device,
0248                "Invalid req id %d\n", cdesc->req_id);
0249         return -EINVAL;
0250     }
0251
0252     ena_com_cq_inc_head(io_cq);
0253
0254     return 0;
0255 }
0256
0257 #endif /* ENA_ETH_COM_H_ */