amazon/ena/ena_com.c

0001 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
0002 /*
0003  * Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
0004  */
0005
0006 #include "ena_com.h"
0007
0008 /*****************************************************************************/
0009 /*****************************************************************************/
0010
0011 /* Timeout in micro-sec */
0012 #define ADMIN_CMD_TIMEOUT_US (3000000)
0013
0014 #define ENA_ASYNC_QUEUE_DEPTH 16
0015 #define ENA_ADMIN_QUEUE_DEPTH 32
0016
0017
0018 #define ENA_CTRL_MAJOR      0
0019 #define ENA_CTRL_MINOR      0
0020 #define ENA_CTRL_SUB_MINOR  1
0021
0022 #define MIN_ENA_CTRL_VER \
0023     (((ENA_CTRL_MAJOR) << \
0024     (ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT)) | \
0025     ((ENA_CTRL_MINOR) << \
0026     (ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT)) | \
0027     (ENA_CTRL_SUB_MINOR))
0028
0029 #define ENA_DMA_ADDR_TO_UINT32_LOW(x)   ((u32)((u64)(x)))
0030 #define ENA_DMA_ADDR_TO_UINT32_HIGH(x)  ((u32)(((u64)(x)) >> 32))
0031
0032 #define ENA_MMIO_READ_TIMEOUT 0xFFFFFFFF
0033
0034 #define ENA_COM_BOUNCE_BUFFER_CNTRL_CNT 4
0035
0036 #define ENA_REGS_ADMIN_INTR_MASK 1
0037
0038 #define ENA_MIN_ADMIN_POLL_US 100
0039
0040 #define ENA_MAX_ADMIN_POLL_US 5000
0041
0042 /*****************************************************************************/
0043 /*****************************************************************************/
0044 /*****************************************************************************/
0045
0046 enum ena_cmd_status {
0047     ENA_CMD_SUBMITTED,
0048     ENA_CMD_COMPLETED,
0049     /* Abort - canceled by the driver */
0050     ENA_CMD_ABORTED,
0051 };
0052
0053 struct ena_comp_ctx {
0054     struct completion wait_event;
0055     struct ena_admin_acq_entry *user_cqe;
0056     u32 comp_size;
0057     enum ena_cmd_status status;
0058     /* status from the device */
0059     u8 comp_status;
0060     u8 cmd_opcode;
0061     bool occupied;
0062 };
0063
0064 struct ena_com_stats_ctx {
0065     struct ena_admin_aq_get_stats_cmd get_cmd;
0066     struct ena_admin_acq_get_stats_resp get_resp;
0067 };
0068
0069 static int ena_com_mem_addr_set(struct ena_com_dev *ena_dev,
0070                        struct ena_common_mem_addr *ena_addr,
0071                        dma_addr_t addr)
0072 {
0073     if ((addr & GENMASK_ULL(ena_dev->dma_addr_bits - 1, 0)) != addr) {
0074         netdev_err(ena_dev->net_device,
0075                "DMA address has more bits that the device supports\n");
0076         return -EINVAL;
0077     }
0078
0079     ena_addr->mem_addr_low = lower_32_bits(addr);
0080     ena_addr->mem_addr_high = (u16)upper_32_bits(addr);
0081
0082     return 0;
0083 }
0084
0085 static int ena_com_admin_init_sq(struct ena_com_admin_queue *admin_queue)
0086 {
0087     struct ena_com_dev *ena_dev = admin_queue->ena_dev;
0088     struct ena_com_admin_sq *sq = &admin_queue->sq;
0089     u16 size = ADMIN_SQ_SIZE(admin_queue->q_depth);
0090
0091     sq->entries = dma_alloc_coherent(admin_queue->q_dmadev, size,
0092                      &sq->dma_addr, GFP_KERNEL);
0093
0094     if (!sq->entries) {
0095         netdev_err(ena_dev->net_device, "Memory allocation failed\n");
0096         return -ENOMEM;
0097     }
0098
0099     sq->head = 0;
0100     sq->tail = 0;
0101     sq->phase = 1;
0102
0103     sq->db_addr = NULL;
0104
0105     return 0;
0106 }
0107
0108 static int ena_com_admin_init_cq(struct ena_com_admin_queue *admin_queue)
0109 {
0110     struct ena_com_dev *ena_dev = admin_queue->ena_dev;
0111     struct ena_com_admin_cq *cq = &admin_queue->cq;
0112     u16 size = ADMIN_CQ_SIZE(admin_queue->q_depth);
0113
0114     cq->entries = dma_alloc_coherent(admin_queue->q_dmadev, size,
0115                      &cq->dma_addr, GFP_KERNEL);
0116
0117     if (!cq->entries) {
0118         netdev_err(ena_dev->net_device, "Memory allocation failed\n");
0119         return -ENOMEM;
0120     }
0121
0122     cq->head = 0;
0123     cq->phase = 1;
0124
0125     return 0;
0126 }
0127
0128 static int ena_com_admin_init_aenq(struct ena_com_dev *ena_dev,
0129                    struct ena_aenq_handlers *aenq_handlers)
0130 {
0131     struct ena_com_aenq *aenq = &ena_dev->aenq;
0132     u32 addr_low, addr_high, aenq_caps;
0133     u16 size;
0134
0135     ena_dev->aenq.q_depth = ENA_ASYNC_QUEUE_DEPTH;
0136     size = ADMIN_AENQ_SIZE(ENA_ASYNC_QUEUE_DEPTH);
0137     aenq->entries = dma_alloc_coherent(ena_dev->dmadev, size,
0138                        &aenq->dma_addr, GFP_KERNEL);
0139
0140     if (!aenq->entries) {
0141         netdev_err(ena_dev->net_device, "Memory allocation failed\n");
0142         return -ENOMEM;
0143     }
0144
0145     aenq->head = aenq->q_depth;
0146     aenq->phase = 1;
0147
0148     addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(aenq->dma_addr);
0149     addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(aenq->dma_addr);
0150
0151     writel(addr_low, ena_dev->reg_bar + ENA_REGS_AENQ_BASE_LO_OFF);
0152     writel(addr_high, ena_dev->reg_bar + ENA_REGS_AENQ_BASE_HI_OFF);
0153
0154     aenq_caps = 0;
0155     aenq_caps |= ena_dev->aenq.q_depth & ENA_REGS_AENQ_CAPS_AENQ_DEPTH_MASK;
0156     aenq_caps |= (sizeof(struct ena_admin_aenq_entry)
0157               << ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_SHIFT) &
0158              ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_MASK;
0159     writel(aenq_caps, ena_dev->reg_bar + ENA_REGS_AENQ_CAPS_OFF);
0160
0161     if (unlikely(!aenq_handlers)) {
0162         netdev_err(ena_dev->net_device,
0163                "AENQ handlers pointer is NULL\n");
0164         return -EINVAL;
0165     }
0166
0167     aenq->aenq_handlers = aenq_handlers;
0168
0169     return 0;
0170 }
0171
0172 static void comp_ctxt_release(struct ena_com_admin_queue *queue,
0173                      struct ena_comp_ctx *comp_ctx)
0174 {
0175     comp_ctx->occupied = false;
0176     atomic_dec(&queue->outstanding_cmds);
0177 }
0178
0179 static struct ena_comp_ctx *get_comp_ctxt(struct ena_com_admin_queue *admin_queue,
0180                       u16 command_id, bool capture)
0181 {
0182     if (unlikely(command_id >= admin_queue->q_depth)) {
0183         netdev_err(admin_queue->ena_dev->net_device,
0184                "Command id is larger than the queue size. cmd_id: %u queue size %d\n",
0185                command_id, admin_queue->q_depth);
0186         return NULL;
0187     }
0188
0189     if (unlikely(!admin_queue->comp_ctx)) {
0190         netdev_err(admin_queue->ena_dev->net_device,
0191                "Completion context is NULL\n");
0192         return NULL;
0193     }
0194
0195     if (unlikely(admin_queue->comp_ctx[command_id].occupied && capture)) {
0196         netdev_err(admin_queue->ena_dev->net_device,
0197                "Completion context is occupied\n");
0198         return NULL;
0199     }
0200
0201     if (capture) {
0202         atomic_inc(&admin_queue->outstanding_cmds);
0203         admin_queue->comp_ctx[command_id].occupied = true;
0204     }
0205
0206     return &admin_queue->comp_ctx[command_id];
0207 }
0208
0209 static struct ena_comp_ctx *__ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,
0210                                struct ena_admin_aq_entry *cmd,
0211                                size_t cmd_size_in_bytes,
0212                                struct ena_admin_acq_entry *comp,
0213                                size_t comp_size_in_bytes)
0214 {
0215     struct ena_comp_ctx *comp_ctx;
0216     u16 tail_masked, cmd_id;
0217     u16 queue_size_mask;
0218     u16 cnt;
0219
0220     queue_size_mask = admin_queue->q_depth - 1;
0221
0222     tail_masked = admin_queue->sq.tail & queue_size_mask;
0223
0224     /* In case of queue FULL */
0225     cnt = (u16)atomic_read(&admin_queue->outstanding_cmds);
0226     if (cnt >= admin_queue->q_depth) {
0227         netdev_dbg(admin_queue->ena_dev->net_device,
0228                "Admin queue is full.\n");
0229         admin_queue->stats.out_of_space++;
0230         return ERR_PTR(-ENOSPC);
0231     }
0232
0233     cmd_id = admin_queue->curr_cmd_id;
0234
0235     cmd->aq_common_descriptor.flags |= admin_queue->sq.phase &
0236         ENA_ADMIN_AQ_COMMON_DESC_PHASE_MASK;
0237
0238     cmd->aq_common_descriptor.command_id |= cmd_id &
0239         ENA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK;
0240
0241     comp_ctx = get_comp_ctxt(admin_queue, cmd_id, true);
0242     if (unlikely(!comp_ctx))
0243         return ERR_PTR(-EINVAL);
0244
0245     comp_ctx->status = ENA_CMD_SUBMITTED;
0246     comp_ctx->comp_size = (u32)comp_size_in_bytes;
0247     comp_ctx->user_cqe = comp;
0248     comp_ctx->cmd_opcode = cmd->aq_common_descriptor.opcode;
0249
0250     reinit_completion(&comp_ctx->wait_event);
0251
0252     memcpy(&admin_queue->sq.entries[tail_masked], cmd, cmd_size_in_bytes);
0253
0254     admin_queue->curr_cmd_id = (admin_queue->curr_cmd_id + 1) &
0255         queue_size_mask;
0256
0257     admin_queue->sq.tail++;
0258     admin_queue->stats.submitted_cmd++;
0259
0260     if (unlikely((admin_queue->sq.tail & queue_size_mask) == 0))
0261         admin_queue->sq.phase = !admin_queue->sq.phase;
0262
0263     writel(admin_queue->sq.tail, admin_queue->sq.db_addr);
0264
0265     return comp_ctx;
0266 }
0267
0268 static int ena_com_init_comp_ctxt(struct ena_com_admin_queue *admin_queue)
0269 {
0270     struct ena_com_dev *ena_dev = admin_queue->ena_dev;
0271     size_t size = admin_queue->q_depth * sizeof(struct ena_comp_ctx);
0272     struct ena_comp_ctx *comp_ctx;
0273     u16 i;
0274
0275     admin_queue->comp_ctx =
0276         devm_kzalloc(admin_queue->q_dmadev, size, GFP_KERNEL);
0277     if (unlikely(!admin_queue->comp_ctx)) {
0278         netdev_err(ena_dev->net_device, "Memory allocation failed\n");
0279         return -ENOMEM;
0280     }
0281
0282     for (i = 0; i < admin_queue->q_depth; i++) {
0283         comp_ctx = get_comp_ctxt(admin_queue, i, false);
0284         if (comp_ctx)
0285             init_completion(&comp_ctx->wait_event);
0286     }
0287
0288     return 0;
0289 }
0290
0291 static struct ena_comp_ctx *ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,
0292                              struct ena_admin_aq_entry *cmd,
0293                              size_t cmd_size_in_bytes,
0294                              struct ena_admin_acq_entry *comp,
0295                              size_t comp_size_in_bytes)
0296 {
0297     unsigned long flags = 0;
0298     struct ena_comp_ctx *comp_ctx;
0299
0300     spin_lock_irqsave(&admin_queue->q_lock, flags);
0301     if (unlikely(!admin_queue->running_state)) {
0302         spin_unlock_irqrestore(&admin_queue->q_lock, flags);
0303         return ERR_PTR(-ENODEV);
0304     }
0305     comp_ctx = __ena_com_submit_admin_cmd(admin_queue, cmd,
0306                           cmd_size_in_bytes,
0307                           comp,
0308                           comp_size_in_bytes);
0309     if (IS_ERR(comp_ctx))
0310         admin_queue->running_state = false;
0311     spin_unlock_irqrestore(&admin_queue->q_lock, flags);
0312
0313     return comp_ctx;
0314 }
0315
0316 static int ena_com_init_io_sq(struct ena_com_dev *ena_dev,
0317                   struct ena_com_create_io_ctx *ctx,
0318                   struct ena_com_io_sq *io_sq)
0319 {
0320     size_t size;
0321     int dev_node = 0;
0322
0323     memset(&io_sq->desc_addr, 0x0, sizeof(io_sq->desc_addr));
0324
0325     io_sq->dma_addr_bits = (u8)ena_dev->dma_addr_bits;
0326     io_sq->desc_entry_size =
0327         (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) ?
0328         sizeof(struct ena_eth_io_tx_desc) :
0329         sizeof(struct ena_eth_io_rx_desc);
0330
0331     size = io_sq->desc_entry_size * io_sq->q_depth;
0332
0333     if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) {
0334         dev_node = dev_to_node(ena_dev->dmadev);
0335         set_dev_node(ena_dev->dmadev, ctx->numa_node);
0336         io_sq->desc_addr.virt_addr =
0337             dma_alloc_coherent(ena_dev->dmadev, size,
0338                        &io_sq->desc_addr.phys_addr,
0339                        GFP_KERNEL);
0340         set_dev_node(ena_dev->dmadev, dev_node);
0341         if (!io_sq->desc_addr.virt_addr) {
0342             io_sq->desc_addr.virt_addr =
0343                 dma_alloc_coherent(ena_dev->dmadev, size,
0344                            &io_sq->desc_addr.phys_addr,
0345                            GFP_KERNEL);
0346         }
0347
0348         if (!io_sq->desc_addr.virt_addr) {
0349             netdev_err(ena_dev->net_device,
0350                    "Memory allocation failed\n");
0351             return -ENOMEM;
0352         }
0353     }
0354
0355     if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
0356         /* Allocate bounce buffers */
0357         io_sq->bounce_buf_ctrl.buffer_size =
0358             ena_dev->llq_info.desc_list_entry_size;
0359         io_sq->bounce_buf_ctrl.buffers_num =
0360             ENA_COM_BOUNCE_BUFFER_CNTRL_CNT;
0361         io_sq->bounce_buf_ctrl.next_to_use = 0;
0362
0363         size = io_sq->bounce_buf_ctrl.buffer_size *
0364             io_sq->bounce_buf_ctrl.buffers_num;
0365
0366         dev_node = dev_to_node(ena_dev->dmadev);
0367         set_dev_node(ena_dev->dmadev, ctx->numa_node);
0368         io_sq->bounce_buf_ctrl.base_buffer =
0369             devm_kzalloc(ena_dev->dmadev, size, GFP_KERNEL);
0370         set_dev_node(ena_dev->dmadev, dev_node);
0371         if (!io_sq->bounce_buf_ctrl.base_buffer)
0372             io_sq->bounce_buf_ctrl.base_buffer =
0373                 devm_kzalloc(ena_dev->dmadev, size, GFP_KERNEL);
0374
0375         if (!io_sq->bounce_buf_ctrl.base_buffer) {
0376             netdev_err(ena_dev->net_device,
0377                    "Bounce buffer memory allocation failed\n");
0378             return -ENOMEM;
0379         }
0380
0381         memcpy(&io_sq->llq_info, &ena_dev->llq_info,
0382                sizeof(io_sq->llq_info));
0383
0384         /* Initiate the first bounce buffer */
0385         io_sq->llq_buf_ctrl.curr_bounce_buf =
0386             ena_com_get_next_bounce_buffer(&io_sq->bounce_buf_ctrl);
0387         memset(io_sq->llq_buf_ctrl.curr_bounce_buf,
0388                0x0, io_sq->llq_info.desc_list_entry_size);
0389         io_sq->llq_buf_ctrl.descs_left_in_line =
0390             io_sq->llq_info.descs_num_before_header;
0391         io_sq->disable_meta_caching =
0392             io_sq->llq_info.disable_meta_caching;
0393
0394         if (io_sq->llq_info.max_entries_in_tx_burst > 0)
0395             io_sq->entries_in_tx_burst_left =
0396                 io_sq->llq_info.max_entries_in_tx_burst;
0397     }
0398
0399     io_sq->tail = 0;
0400     io_sq->next_to_comp = 0;
0401     io_sq->phase = 1;
0402
0403     return 0;
0404 }
0405
0406 static int ena_com_init_io_cq(struct ena_com_dev *ena_dev,
0407                   struct ena_com_create_io_ctx *ctx,
0408                   struct ena_com_io_cq *io_cq)
0409 {
0410     size_t size;
0411     int prev_node = 0;
0412
0413     memset(&io_cq->cdesc_addr, 0x0, sizeof(io_cq->cdesc_addr));
0414
0415     /* Use the basic completion descriptor for Rx */
0416     io_cq->cdesc_entry_size_in_bytes =
0417         (io_cq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) ?
0418         sizeof(struct ena_eth_io_tx_cdesc) :
0419         sizeof(struct ena_eth_io_rx_cdesc_base);
0420
0421     size = io_cq->cdesc_entry_size_in_bytes * io_cq->q_depth;
0422
0423     prev_node = dev_to_node(ena_dev->dmadev);
0424     set_dev_node(ena_dev->dmadev, ctx->numa_node);
0425     io_cq->cdesc_addr.virt_addr =
0426         dma_alloc_coherent(ena_dev->dmadev, size,
0427                    &io_cq->cdesc_addr.phys_addr, GFP_KERNEL);
0428     set_dev_node(ena_dev->dmadev, prev_node);
0429     if (!io_cq->cdesc_addr.virt_addr) {
0430         io_cq->cdesc_addr.virt_addr =
0431             dma_alloc_coherent(ena_dev->dmadev, size,
0432                        &io_cq->cdesc_addr.phys_addr,
0433                        GFP_KERNEL);
0434     }
0435
0436     if (!io_cq->cdesc_addr.virt_addr) {
0437         netdev_err(ena_dev->net_device, "Memory allocation failed\n");
0438         return -ENOMEM;
0439     }
0440
0441     io_cq->phase = 1;
0442     io_cq->head = 0;
0443
0444     return 0;
0445 }
0446
0447 static void ena_com_handle_single_admin_completion(struct ena_com_admin_queue *admin_queue,
0448                            struct ena_admin_acq_entry *cqe)
0449 {
0450     struct ena_comp_ctx *comp_ctx;
0451     u16 cmd_id;
0452
0453     cmd_id = cqe->acq_common_descriptor.command &
0454         ENA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK;
0455
0456     comp_ctx = get_comp_ctxt(admin_queue, cmd_id, false);
0457     if (unlikely(!comp_ctx)) {
0458         netdev_err(admin_queue->ena_dev->net_device,
0459                "comp_ctx is NULL. Changing the admin queue running state\n");
0460         admin_queue->running_state = false;
0461         return;
0462     }
0463
0464     comp_ctx->status = ENA_CMD_COMPLETED;
0465     comp_ctx->comp_status = cqe->acq_common_descriptor.status;
0466
0467     if (comp_ctx->user_cqe)
0468         memcpy(comp_ctx->user_cqe, (void *)cqe, comp_ctx->comp_size);
0469
0470     if (!admin_queue->polling)
0471         complete(&comp_ctx->wait_event);
0472 }
0473
0474 static void ena_com_handle_admin_completion(struct ena_com_admin_queue *admin_queue)
0475 {
0476     struct ena_admin_acq_entry *cqe = NULL;
0477     u16 comp_num = 0;
0478     u16 head_masked;
0479     u8 phase;
0480
0481     head_masked = admin_queue->cq.head & (admin_queue->q_depth - 1);
0482     phase = admin_queue->cq.phase;
0483
0484     cqe = &admin_queue->cq.entries[head_masked];
0485
0486     /* Go over all the completions */
0487     while ((READ_ONCE(cqe->acq_common_descriptor.flags) &
0488         ENA_ADMIN_ACQ_COMMON_DESC_PHASE_MASK) == phase) {
0489         /* Do not read the rest of the completion entry before the
0490          * phase bit was validated
0491          */
0492         dma_rmb();
0493         ena_com_handle_single_admin_completion(admin_queue, cqe);
0494
0495         head_masked++;
0496         comp_num++;
0497         if (unlikely(head_masked == admin_queue->q_depth)) {
0498             head_masked = 0;
0499             phase = !phase;
0500         }
0501
0502         cqe = &admin_queue->cq.entries[head_masked];
0503     }
0504
0505     admin_queue->cq.head += comp_num;
0506     admin_queue->cq.phase = phase;
0507     admin_queue->sq.head += comp_num;
0508     admin_queue->stats.completed_cmd += comp_num;
0509 }
0510
0511 static int ena_com_comp_status_to_errno(struct ena_com_admin_queue *admin_queue,
0512                     u8 comp_status)
0513 {
0514     if (unlikely(comp_status != 0))
0515         netdev_err(admin_queue->ena_dev->net_device,
0516                "Admin command failed[%u]\n", comp_status);
0517
0518     switch (comp_status) {
0519     case ENA_ADMIN_SUCCESS:
0520         return 0;
0521     case ENA_ADMIN_RESOURCE_ALLOCATION_FAILURE:
0522         return -ENOMEM;
0523     case ENA_ADMIN_UNSUPPORTED_OPCODE:
0524         return -EOPNOTSUPP;
0525     case ENA_ADMIN_BAD_OPCODE:
0526     case ENA_ADMIN_MALFORMED_REQUEST:
0527     case ENA_ADMIN_ILLEGAL_PARAMETER:
0528     case ENA_ADMIN_UNKNOWN_ERROR:
0529         return -EINVAL;
0530     case ENA_ADMIN_RESOURCE_BUSY:
0531         return -EAGAIN;
0532     }
0533
0534     return -EINVAL;
0535 }
0536
0537 static void ena_delay_exponential_backoff_us(u32 exp, u32 delay_us)
0538 {
0539     delay_us = max_t(u32, ENA_MIN_ADMIN_POLL_US, delay_us);
0540     delay_us = min_t(u32, delay_us * (1U << exp), ENA_MAX_ADMIN_POLL_US);
0541     usleep_range(delay_us, 2 * delay_us);
0542 }
0543
0544 static int ena_com_wait_and_process_admin_cq_polling(struct ena_comp_ctx *comp_ctx,
0545                              struct ena_com_admin_queue *admin_queue)
0546 {
0547     unsigned long flags = 0;
0548     unsigned long timeout;
0549     int ret;
0550     u32 exp = 0;
0551
0552     timeout = jiffies + usecs_to_jiffies(admin_queue->completion_timeout);
0553
0554     while (1) {
0555         spin_lock_irqsave(&admin_queue->q_lock, flags);
0556         ena_com_handle_admin_completion(admin_queue);
0557         spin_unlock_irqrestore(&admin_queue->q_lock, flags);
0558
0559         if (comp_ctx->status != ENA_CMD_SUBMITTED)
0560             break;
0561
0562         if (time_is_before_jiffies(timeout)) {
0563             netdev_err(admin_queue->ena_dev->net_device,
0564                    "Wait for completion (polling) timeout\n");
0565             /* ENA didn't have any completion */
0566             spin_lock_irqsave(&admin_queue->q_lock, flags);
0567             admin_queue->stats.no_completion++;
0568             admin_queue->running_state = false;
0569             spin_unlock_irqrestore(&admin_queue->q_lock, flags);
0570
0571             ret = -ETIME;
0572             goto err;
0573         }
0574
0575         ena_delay_exponential_backoff_us(exp++,
0576                          admin_queue->ena_dev->ena_min_poll_delay_us);
0577     }
0578
0579     if (unlikely(comp_ctx->status == ENA_CMD_ABORTED)) {
0580         netdev_err(admin_queue->ena_dev->net_device,
0581                "Command was aborted\n");
0582         spin_lock_irqsave(&admin_queue->q_lock, flags);
0583         admin_queue->stats.aborted_cmd++;
0584         spin_unlock_irqrestore(&admin_queue->q_lock, flags);
0585         ret = -ENODEV;
0586         goto err;
0587     }
0588
0589     WARN(comp_ctx->status != ENA_CMD_COMPLETED, "Invalid comp status %d\n",
0590          comp_ctx->status);
0591
0592     ret = ena_com_comp_status_to_errno(admin_queue, comp_ctx->comp_status);
0593 err:
0594     comp_ctxt_release(admin_queue, comp_ctx);
0595     return ret;
0596 }
0597
0598 /*
0599  * Set the LLQ configurations of the firmware
0600  *
0601  * The driver provides only the enabled feature values to the device,
0602  * which in turn, checks if they are supported.
0603  */
0604 static int ena_com_set_llq(struct ena_com_dev *ena_dev)
0605 {
0606     struct ena_com_admin_queue *admin_queue;
0607     struct ena_admin_set_feat_cmd cmd;
0608     struct ena_admin_set_feat_resp resp;
0609     struct ena_com_llq_info *llq_info = &ena_dev->llq_info;
0610     int ret;
0611
0612     memset(&cmd, 0x0, sizeof(cmd));
0613     admin_queue = &ena_dev->admin_queue;
0614
0615     cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
0616     cmd.feat_common.feature_id = ENA_ADMIN_LLQ;
0617
0618     cmd.u.llq.header_location_ctrl_enabled = llq_info->header_location_ctrl;
0619     cmd.u.llq.entry_size_ctrl_enabled = llq_info->desc_list_entry_size_ctrl;
0620     cmd.u.llq.desc_num_before_header_enabled = llq_info->descs_num_before_header;
0621     cmd.u.llq.descriptors_stride_ctrl_enabled = llq_info->desc_stride_ctrl;
0622
0623     cmd.u.llq.accel_mode.u.set.enabled_flags =
0624         BIT(ENA_ADMIN_DISABLE_META_CACHING) |
0625         BIT(ENA_ADMIN_LIMIT_TX_BURST);
0626
0627     ret = ena_com_execute_admin_command(admin_queue,
0628                         (struct ena_admin_aq_entry *)&cmd,
0629                         sizeof(cmd),
0630                         (struct ena_admin_acq_entry *)&resp,
0631                         sizeof(resp));
0632
0633     if (unlikely(ret))
0634         netdev_err(ena_dev->net_device,
0635                "Failed to set LLQ configurations: %d\n", ret);
0636
0637     return ret;
0638 }
0639
0640 static int ena_com_config_llq_info(struct ena_com_dev *ena_dev,
0641                    struct ena_admin_feature_llq_desc *llq_features,
0642                    struct ena_llq_configurations *llq_default_cfg)
0643 {
0644     struct ena_com_llq_info *llq_info = &ena_dev->llq_info;
0645     struct ena_admin_accel_mode_get llq_accel_mode_get;
0646     u16 supported_feat;
0647     int rc;
0648
0649     memset(llq_info, 0, sizeof(*llq_info));
0650
0651     supported_feat = llq_features->header_location_ctrl_supported;
0652
0653     if (likely(supported_feat & llq_default_cfg->llq_header_location)) {
0654         llq_info->header_location_ctrl =
0655             llq_default_cfg->llq_header_location;
0656     } else {
0657         netdev_err(ena_dev->net_device,
0658                "Invalid header location control, supported: 0x%x\n",
0659                supported_feat);
0660         return -EINVAL;
0661     }
0662
0663     if (likely(llq_info->header_location_ctrl == ENA_ADMIN_INLINE_HEADER)) {
0664         supported_feat = llq_features->descriptors_stride_ctrl_supported;
0665         if (likely(supported_feat & llq_default_cfg->llq_stride_ctrl)) {
0666             llq_info->desc_stride_ctrl = llq_default_cfg->llq_stride_ctrl;
0667         } else  {
0668             if (supported_feat & ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY) {
0669                 llq_info->desc_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY;
0670             } else if (supported_feat & ENA_ADMIN_SINGLE_DESC_PER_ENTRY) {
0671                 llq_info->desc_stride_ctrl = ENA_ADMIN_SINGLE_DESC_PER_ENTRY;
0672             } else {
0673                 netdev_err(ena_dev->net_device,
0674                        "Invalid desc_stride_ctrl, supported: 0x%x\n",
0675                        supported_feat);
0676                 return -EINVAL;
0677             }
0678
0679             netdev_err(ena_dev->net_device,
0680                    "Default llq stride ctrl is not supported, performing fallback, default: 0x%x, supported: 0x%x, used: 0x%x\n",
0681                    llq_default_cfg->llq_stride_ctrl,
0682                    supported_feat, llq_info->desc_stride_ctrl);
0683         }
0684     } else {
0685         llq_info->desc_stride_ctrl = 0;
0686     }
0687
0688     supported_feat = llq_features->entry_size_ctrl_supported;
0689     if (likely(supported_feat & llq_default_cfg->llq_ring_entry_size)) {
0690         llq_info->desc_list_entry_size_ctrl = llq_default_cfg->llq_ring_entry_size;
0691         llq_info->desc_list_entry_size = llq_default_cfg->llq_ring_entry_size_value;
0692     } else {
0693         if (supported_feat & ENA_ADMIN_LIST_ENTRY_SIZE_128B) {
0694             llq_info->desc_list_entry_size_ctrl = ENA_ADMIN_LIST_ENTRY_SIZE_128B;
0695             llq_info->desc_list_entry_size = 128;
0696         } else if (supported_feat & ENA_ADMIN_LIST_ENTRY_SIZE_192B) {
0697             llq_info->desc_list_entry_size_ctrl = ENA_ADMIN_LIST_ENTRY_SIZE_192B;
0698             llq_info->desc_list_entry_size = 192;
0699         } else if (supported_feat & ENA_ADMIN_LIST_ENTRY_SIZE_256B) {
0700             llq_info->desc_list_entry_size_ctrl = ENA_ADMIN_LIST_ENTRY_SIZE_256B;
0701             llq_info->desc_list_entry_size = 256;
0702         } else {
0703             netdev_err(ena_dev->net_device,
0704                    "Invalid entry_size_ctrl, supported: 0x%x\n",
0705                    supported_feat);
0706             return -EINVAL;
0707         }
0708
0709         netdev_err(ena_dev->net_device,
0710                "Default llq ring entry size is not supported, performing fallback, default: 0x%x, supported: 0x%x, used: 0x%x\n",
0711                llq_default_cfg->llq_ring_entry_size, supported_feat,
0712                llq_info->desc_list_entry_size);
0713     }
0714     if (unlikely(llq_info->desc_list_entry_size & 0x7)) {
0715         /* The desc list entry size should be whole multiply of 8
0716          * This requirement comes from __iowrite64_copy()
0717          */
0718         netdev_err(ena_dev->net_device, "Illegal entry size %d\n",
0719                llq_info->desc_list_entry_size);
0720         return -EINVAL;
0721     }
0722
0723     if (llq_info->desc_stride_ctrl == ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY)
0724         llq_info->descs_per_entry = llq_info->desc_list_entry_size /
0725             sizeof(struct ena_eth_io_tx_desc);
0726     else
0727         llq_info->descs_per_entry = 1;
0728
0729     supported_feat = llq_features->desc_num_before_header_supported;
0730     if (likely(supported_feat & llq_default_cfg->llq_num_decs_before_header)) {
0731         llq_info->descs_num_before_header = llq_default_cfg->llq_num_decs_before_header;
0732     } else {
0733         if (supported_feat & ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2) {
0734             llq_info->descs_num_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2;
0735         } else if (supported_feat & ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_1) {
0736             llq_info->descs_num_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_1;
0737         } else if (supported_feat & ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_4) {
0738             llq_info->descs_num_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_4;
0739         } else if (supported_feat & ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_8) {
0740             llq_info->descs_num_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_8;
0741         } else {
0742             netdev_err(ena_dev->net_device,
0743                    "Invalid descs_num_before_header, supported: 0x%x\n",
0744                    supported_feat);
0745             return -EINVAL;
0746         }
0747
0748         netdev_err(ena_dev->net_device,
0749                "Default llq num descs before header is not supported, performing fallback, default: 0x%x, supported: 0x%x, used: 0x%x\n",
0750                llq_default_cfg->llq_num_decs_before_header,
0751                supported_feat, llq_info->descs_num_before_header);
0752     }
0753     /* Check for accelerated queue supported */
0754     llq_accel_mode_get = llq_features->accel_mode.u.get;
0755
0756     llq_info->disable_meta_caching =
0757         !!(llq_accel_mode_get.supported_flags &
0758            BIT(ENA_ADMIN_DISABLE_META_CACHING));
0759
0760     if (llq_accel_mode_get.supported_flags & BIT(ENA_ADMIN_LIMIT_TX_BURST))
0761         llq_info->max_entries_in_tx_burst =
0762             llq_accel_mode_get.max_tx_burst_size /
0763             llq_default_cfg->llq_ring_entry_size_value;
0764
0765     rc = ena_com_set_llq(ena_dev);
0766     if (rc)
0767         netdev_err(ena_dev->net_device,
0768                "Cannot set LLQ configuration: %d\n", rc);
0769
0770     return rc;
0771 }
0772
0773 static int ena_com_wait_and_process_admin_cq_interrupts(struct ena_comp_ctx *comp_ctx,
0774                             struct ena_com_admin_queue *admin_queue)
0775 {
0776     unsigned long flags = 0;
0777     int ret;
0778
0779     wait_for_completion_timeout(&comp_ctx->wait_event,
0780                     usecs_to_jiffies(
0781                         admin_queue->completion_timeout));
0782
0783     /* In case the command wasn't completed find out the root cause.
0784      * There might be 2 kinds of errors
0785      * 1) No completion (timeout reached)
0786      * 2) There is completion but the device didn't get any msi-x interrupt.
0787      */
0788     if (unlikely(comp_ctx->status == ENA_CMD_SUBMITTED)) {
0789         spin_lock_irqsave(&admin_queue->q_lock, flags);
0790         ena_com_handle_admin_completion(admin_queue);
0791         admin_queue->stats.no_completion++;
0792         spin_unlock_irqrestore(&admin_queue->q_lock, flags);
0793
0794         if (comp_ctx->status == ENA_CMD_COMPLETED) {
0795             netdev_err(admin_queue->ena_dev->net_device,
0796                    "The ena device sent a completion but the driver didn't receive a MSI-X interrupt (cmd %d), autopolling mode is %s\n",
0797                    comp_ctx->cmd_opcode,
0798                    admin_queue->auto_polling ? "ON" : "OFF");
0799             /* Check if fallback to polling is enabled */
0800             if (admin_queue->auto_polling)
0801                 admin_queue->polling = true;
0802         } else {
0803             netdev_err(admin_queue->ena_dev->net_device,
0804                    "The ena device didn't send a completion for the admin cmd %d status %d\n",
0805                    comp_ctx->cmd_opcode, comp_ctx->status);
0806         }
0807         /* Check if shifted to polling mode.
0808          * This will happen if there is a completion without an interrupt
0809          * and autopolling mode is enabled. Continuing normal execution in such case
0810          */
0811         if (!admin_queue->polling) {
0812             admin_queue->running_state = false;
0813             ret = -ETIME;
0814             goto err;
0815         }
0816     }
0817
0818     ret = ena_com_comp_status_to_errno(admin_queue, comp_ctx->comp_status);
0819 err:
0820     comp_ctxt_release(admin_queue, comp_ctx);
0821     return ret;
0822 }
0823
0824 /* This method read the hardware device register through posting writes
0825  * and waiting for response
0826  * On timeout the function will return ENA_MMIO_READ_TIMEOUT
0827  */
0828 static u32 ena_com_reg_bar_read32(struct ena_com_dev *ena_dev, u16 offset)
0829 {
0830     struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
0831     volatile struct ena_admin_ena_mmio_req_read_less_resp *read_resp =
0832         mmio_read->read_resp;
0833     u32 mmio_read_reg, ret, i;
0834     unsigned long flags = 0;
0835     u32 timeout = mmio_read->reg_read_to;
0836
0837     might_sleep();
0838
0839     if (timeout == 0)
0840         timeout = ENA_REG_READ_TIMEOUT;
0841
0842     /* If readless is disabled, perform regular read */
0843     if (!mmio_read->readless_supported)
0844         return readl(ena_dev->reg_bar + offset);
0845
0846     spin_lock_irqsave(&mmio_read->lock, flags);
0847     mmio_read->seq_num++;
0848
0849     read_resp->req_id = mmio_read->seq_num + 0xDEAD;
0850     mmio_read_reg = (offset << ENA_REGS_MMIO_REG_READ_REG_OFF_SHIFT) &
0851             ENA_REGS_MMIO_REG_READ_REG_OFF_MASK;
0852     mmio_read_reg |= mmio_read->seq_num &
0853             ENA_REGS_MMIO_REG_READ_REQ_ID_MASK;
0854
0855     writel(mmio_read_reg, ena_dev->reg_bar + ENA_REGS_MMIO_REG_READ_OFF);
0856
0857     for (i = 0; i < timeout; i++) {
0858         if (READ_ONCE(read_resp->req_id) == mmio_read->seq_num)
0859             break;
0860
0861         udelay(1);
0862     }
0863
0864     if (unlikely(i == timeout)) {
0865         netdev_err(ena_dev->net_device,
0866                "Reading reg failed for timeout. expected: req id[%u] offset[%u] actual: req id[%u] offset[%u]\n",
0867                mmio_read->seq_num, offset, read_resp->req_id,
0868                read_resp->reg_off);
0869         ret = ENA_MMIO_READ_TIMEOUT;
0870         goto err;
0871     }
0872
0873     if (read_resp->reg_off != offset) {
0874         netdev_err(ena_dev->net_device,
0875                "Read failure: wrong offset provided\n");
0876         ret = ENA_MMIO_READ_TIMEOUT;
0877     } else {
0878         ret = read_resp->reg_val;
0879     }
0880 err:
0881     spin_unlock_irqrestore(&mmio_read->lock, flags);
0882
0883     return ret;
0884 }
0885
0886 /* There are two types to wait for completion.
0887  * Polling mode - wait until the completion is available.
0888  * Async mode - wait on wait queue until the completion is ready
0889  * (or the timeout expired).
0890  * It is expected that the IRQ called ena_com_handle_admin_completion
0891  * to mark the completions.
0892  */
0893 static int ena_com_wait_and_process_admin_cq(struct ena_comp_ctx *comp_ctx,
0894                          struct ena_com_admin_queue *admin_queue)
0895 {
0896     if (admin_queue->polling)
0897         return ena_com_wait_and_process_admin_cq_polling(comp_ctx,
0898                                  admin_queue);
0899
0900     return ena_com_wait_and_process_admin_cq_interrupts(comp_ctx,
0901                                 admin_queue);
0902 }
0903
0904 static int ena_com_destroy_io_sq(struct ena_com_dev *ena_dev,
0905                  struct ena_com_io_sq *io_sq)
0906 {
0907     struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
0908     struct ena_admin_aq_destroy_sq_cmd destroy_cmd;
0909     struct ena_admin_acq_destroy_sq_resp_desc destroy_resp;
0910     u8 direction;
0911     int ret;
0912
0913     memset(&destroy_cmd, 0x0, sizeof(destroy_cmd));
0914
0915     if (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX)
0916         direction = ENA_ADMIN_SQ_DIRECTION_TX;
0917     else
0918         direction = ENA_ADMIN_SQ_DIRECTION_RX;
0919
0920     destroy_cmd.sq.sq_identity |= (direction <<
0921         ENA_ADMIN_SQ_SQ_DIRECTION_SHIFT) &
0922         ENA_ADMIN_SQ_SQ_DIRECTION_MASK;
0923
0924     destroy_cmd.sq.sq_idx = io_sq->idx;
0925     destroy_cmd.aq_common_descriptor.opcode = ENA_ADMIN_DESTROY_SQ;
0926
0927     ret = ena_com_execute_admin_command(admin_queue,
0928                         (struct ena_admin_aq_entry *)&destroy_cmd,
0929                         sizeof(destroy_cmd),
0930                         (struct ena_admin_acq_entry *)&destroy_resp,
0931                         sizeof(destroy_resp));
0932
0933     if (unlikely(ret && (ret != -ENODEV)))
0934         netdev_err(ena_dev->net_device,
0935                "Failed to destroy io sq error: %d\n", ret);
0936
0937     return ret;
0938 }
0939
0940 static void ena_com_io_queue_free(struct ena_com_dev *ena_dev,
0941                   struct ena_com_io_sq *io_sq,
0942                   struct ena_com_io_cq *io_cq)
0943 {
0944     size_t size;
0945
0946     if (io_cq->cdesc_addr.virt_addr) {
0947         size = io_cq->cdesc_entry_size_in_bytes * io_cq->q_depth;
0948
0949         dma_free_coherent(ena_dev->dmadev, size,
0950                   io_cq->cdesc_addr.virt_addr,
0951                   io_cq->cdesc_addr.phys_addr);
0952
0953         io_cq->cdesc_addr.virt_addr = NULL;
0954     }
0955
0956     if (io_sq->desc_addr.virt_addr) {
0957         size = io_sq->desc_entry_size * io_sq->q_depth;
0958
0959         dma_free_coherent(ena_dev->dmadev, size,
0960                   io_sq->desc_addr.virt_addr,
0961                   io_sq->desc_addr.phys_addr);
0962
0963         io_sq->desc_addr.virt_addr = NULL;
0964     }
0965
0966     if (io_sq->bounce_buf_ctrl.base_buffer) {
0967         devm_kfree(ena_dev->dmadev, io_sq->bounce_buf_ctrl.base_buffer);
0968         io_sq->bounce_buf_ctrl.base_buffer = NULL;
0969     }
0970 }
0971
0972 static int wait_for_reset_state(struct ena_com_dev *ena_dev, u32 timeout,
0973                 u16 exp_state)
0974 {
0975     u32 val, exp = 0;
0976     unsigned long timeout_stamp;
0977
0978     /* Convert timeout from resolution of 100ms to us resolution. */
0979     timeout_stamp = jiffies + usecs_to_jiffies(100 * 1000 * timeout);
0980
0981     while (1) {
0982         val = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF);
0983
0984         if (unlikely(val == ENA_MMIO_READ_TIMEOUT)) {
0985             netdev_err(ena_dev->net_device,
0986                    "Reg read timeout occurred\n");
0987             return -ETIME;
0988         }
0989
0990         if ((val & ENA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK) ==
0991             exp_state)
0992             return 0;
0993
0994         if (time_is_before_jiffies(timeout_stamp))
0995             return -ETIME;
0996
0997         ena_delay_exponential_backoff_us(exp++, ena_dev->ena_min_poll_delay_us);
0998     }
0999 }
1000
1001 static bool ena_com_check_supported_feature_id(struct ena_com_dev *ena_dev,
1002                            enum ena_admin_aq_feature_id feature_id)
1003 {
1004     u32 feature_mask = 1 << feature_id;
1005
1006     /* Device attributes is always supported */
1007     if ((feature_id != ENA_ADMIN_DEVICE_ATTRIBUTES) &&
1008         !(ena_dev->supported_features & feature_mask))
1009         return false;
1010
1011     return true;
1012 }
1013
1014 static int ena_com_get_feature_ex(struct ena_com_dev *ena_dev,
1015                   struct ena_admin_get_feat_resp *get_resp,
1016                   enum ena_admin_aq_feature_id feature_id,
1017                   dma_addr_t control_buf_dma_addr,
1018                   u32 control_buff_size,
1019                   u8 feature_ver)
1020 {
1021     struct ena_com_admin_queue *admin_queue;
1022     struct ena_admin_get_feat_cmd get_cmd;
1023     int ret;
1024
1025     if (!ena_com_check_supported_feature_id(ena_dev, feature_id)) {
1026         netdev_dbg(ena_dev->net_device, "Feature %d isn't supported\n",
1027                feature_id);
1028         return -EOPNOTSUPP;
1029     }
1030
1031     memset(&get_cmd, 0x0, sizeof(get_cmd));
1032     admin_queue = &ena_dev->admin_queue;
1033
1034     get_cmd.aq_common_descriptor.opcode = ENA_ADMIN_GET_FEATURE;
1035
1036     if (control_buff_size)
1037         get_cmd.aq_common_descriptor.flags =
1038             ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
1039     else
1040         get_cmd.aq_common_descriptor.flags = 0;
1041
1042     ret = ena_com_mem_addr_set(ena_dev,
1043                    &get_cmd.control_buffer.address,
1044                    control_buf_dma_addr);
1045     if (unlikely(ret)) {
1046         netdev_err(ena_dev->net_device, "Memory address set failed\n");
1047         return ret;
1048     }
1049
1050     get_cmd.control_buffer.length = control_buff_size;
1051     get_cmd.feat_common.feature_version = feature_ver;
1052     get_cmd.feat_common.feature_id = feature_id;
1053
1054     ret = ena_com_execute_admin_command(admin_queue,
1055                         (struct ena_admin_aq_entry *)
1056                         &get_cmd,
1057                         sizeof(get_cmd),
1058                         (struct ena_admin_acq_entry *)
1059                         get_resp,
1060                         sizeof(*get_resp));
1061
1062     if (unlikely(ret))
1063         netdev_err(ena_dev->net_device,
1064                "Failed to submit get_feature command %d error: %d\n",
1065                feature_id, ret);
1066
1067     return ret;
1068 }
1069
1070 static int ena_com_get_feature(struct ena_com_dev *ena_dev,
1071                    struct ena_admin_get_feat_resp *get_resp,
1072                    enum ena_admin_aq_feature_id feature_id,
1073                    u8 feature_ver)
1074 {
1075     return ena_com_get_feature_ex(ena_dev,
1076                       get_resp,
1077                       feature_id,
1078                       0,
1079                       0,
1080                       feature_ver);
1081 }
1082
1083 int ena_com_get_current_hash_function(struct ena_com_dev *ena_dev)
1084 {
1085     return ena_dev->rss.hash_func;
1086 }
1087
1088 static void ena_com_hash_key_fill_default_key(struct ena_com_dev *ena_dev)
1089 {
1090     struct ena_admin_feature_rss_flow_hash_control *hash_key =
1091         (ena_dev->rss).hash_key;
1092
1093     netdev_rss_key_fill(&hash_key->key, sizeof(hash_key->key));
1094     /* The key buffer is stored in the device in an array of
1095      * uint32 elements.
1096      */
1097     hash_key->key_parts = ENA_ADMIN_RSS_KEY_PARTS;
1098 }
1099
1100 static int ena_com_hash_key_allocate(struct ena_com_dev *ena_dev)
1101 {
1102     struct ena_rss *rss = &ena_dev->rss;
1103
1104     if (!ena_com_check_supported_feature_id(ena_dev,
1105                         ENA_ADMIN_RSS_HASH_FUNCTION))
1106         return -EOPNOTSUPP;
1107
1108     rss->hash_key =
1109         dma_alloc_coherent(ena_dev->dmadev, sizeof(*rss->hash_key),
1110                    &rss->hash_key_dma_addr, GFP_KERNEL);
1111
1112     if (unlikely(!rss->hash_key))
1113         return -ENOMEM;
1114
1115     return 0;
1116 }
1117
1118 static void ena_com_hash_key_destroy(struct ena_com_dev *ena_dev)
1119 {
1120     struct ena_rss *rss = &ena_dev->rss;
1121
1122     if (rss->hash_key)
1123         dma_free_coherent(ena_dev->dmadev, sizeof(*rss->hash_key),
1124                   rss->hash_key, rss->hash_key_dma_addr);
1125     rss->hash_key = NULL;
1126 }
1127
1128 static int ena_com_hash_ctrl_init(struct ena_com_dev *ena_dev)
1129 {
1130     struct ena_rss *rss = &ena_dev->rss;
1131
1132     rss->hash_ctrl =
1133         dma_alloc_coherent(ena_dev->dmadev, sizeof(*rss->hash_ctrl),
1134                    &rss->hash_ctrl_dma_addr, GFP_KERNEL);
1135
1136     if (unlikely(!rss->hash_ctrl))
1137         return -ENOMEM;
1138
1139     return 0;
1140 }
1141
1142 static void ena_com_hash_ctrl_destroy(struct ena_com_dev *ena_dev)
1143 {
1144     struct ena_rss *rss = &ena_dev->rss;
1145
1146     if (rss->hash_ctrl)
1147         dma_free_coherent(ena_dev->dmadev, sizeof(*rss->hash_ctrl),
1148                   rss->hash_ctrl, rss->hash_ctrl_dma_addr);
1149     rss->hash_ctrl = NULL;
1150 }
1151
1152 static int ena_com_indirect_table_allocate(struct ena_com_dev *ena_dev,
1153                        u16 log_size)
1154 {
1155     struct ena_rss *rss = &ena_dev->rss;
1156     struct ena_admin_get_feat_resp get_resp;
1157     size_t tbl_size;
1158     int ret;
1159
1160     ret = ena_com_get_feature(ena_dev, &get_resp,
1161                   ENA_ADMIN_RSS_INDIRECTION_TABLE_CONFIG, 0);
1162     if (unlikely(ret))
1163         return ret;
1164
1165     if ((get_resp.u.ind_table.min_size > log_size) ||
1166         (get_resp.u.ind_table.max_size < log_size)) {
1167         netdev_err(ena_dev->net_device,
1168                "Indirect table size doesn't fit. requested size: %d while min is:%d and max %d\n",
1169                1 << log_size, 1 << get_resp.u.ind_table.min_size,
1170                1 << get_resp.u.ind_table.max_size);
1171         return -EINVAL;
1172     }
1173
1174     tbl_size = (1ULL << log_size) *
1175         sizeof(struct ena_admin_rss_ind_table_entry);
1176
1177     rss->rss_ind_tbl =
1178         dma_alloc_coherent(ena_dev->dmadev, tbl_size,
1179                    &rss->rss_ind_tbl_dma_addr, GFP_KERNEL);
1180     if (unlikely(!rss->rss_ind_tbl))
1181         goto mem_err1;
1182
1183     tbl_size = (1ULL << log_size) * sizeof(u16);
1184     rss->host_rss_ind_tbl =
1185         devm_kzalloc(ena_dev->dmadev, tbl_size, GFP_KERNEL);
1186     if (unlikely(!rss->host_rss_ind_tbl))
1187         goto mem_err2;
1188
1189     rss->tbl_log_size = log_size;
1190
1191     return 0;
1192
1193 mem_err2:
1194     tbl_size = (1ULL << log_size) *
1195         sizeof(struct ena_admin_rss_ind_table_entry);
1196
1197     dma_free_coherent(ena_dev->dmadev, tbl_size, rss->rss_ind_tbl,
1198               rss->rss_ind_tbl_dma_addr);
1199     rss->rss_ind_tbl = NULL;
1200 mem_err1:
1201     rss->tbl_log_size = 0;
1202     return -ENOMEM;
1203 }
1204
1205 static void ena_com_indirect_table_destroy(struct ena_com_dev *ena_dev)
1206 {
1207     struct ena_rss *rss = &ena_dev->rss;
1208     size_t tbl_size = (1ULL << rss->tbl_log_size) *
1209         sizeof(struct ena_admin_rss_ind_table_entry);
1210
1211     if (rss->rss_ind_tbl)
1212         dma_free_coherent(ena_dev->dmadev, tbl_size, rss->rss_ind_tbl,
1213                   rss->rss_ind_tbl_dma_addr);
1214     rss->rss_ind_tbl = NULL;
1215
1216     if (rss->host_rss_ind_tbl)
1217         devm_kfree(ena_dev->dmadev, rss->host_rss_ind_tbl);
1218     rss->host_rss_ind_tbl = NULL;
1219 }
1220
1221 static int ena_com_create_io_sq(struct ena_com_dev *ena_dev,
1222                 struct ena_com_io_sq *io_sq, u16 cq_idx)
1223 {
1224     struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1225     struct ena_admin_aq_create_sq_cmd create_cmd;
1226     struct ena_admin_acq_create_sq_resp_desc cmd_completion;
1227     u8 direction;
1228     int ret;
1229
1230     memset(&create_cmd, 0x0, sizeof(create_cmd));
1231
1232     create_cmd.aq_common_descriptor.opcode = ENA_ADMIN_CREATE_SQ;
1233
1234     if (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX)
1235         direction = ENA_ADMIN_SQ_DIRECTION_TX;
1236     else
1237         direction = ENA_ADMIN_SQ_DIRECTION_RX;
1238
1239     create_cmd.sq_identity |= (direction <<
1240         ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_SHIFT) &
1241         ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_MASK;
1242
1243     create_cmd.sq_caps_2 |= io_sq->mem_queue_type &
1244         ENA_ADMIN_AQ_CREATE_SQ_CMD_PLACEMENT_POLICY_MASK;
1245
1246     create_cmd.sq_caps_2 |= (ENA_ADMIN_COMPLETION_POLICY_DESC <<
1247         ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_SHIFT) &
1248         ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_MASK;
1249
1250     create_cmd.sq_caps_3 |=
1251         ENA_ADMIN_AQ_CREATE_SQ_CMD_IS_PHYSICALLY_CONTIGUOUS_MASK;
1252
1253     create_cmd.cq_idx = cq_idx;
1254     create_cmd.sq_depth = io_sq->q_depth;
1255
1256     if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) {
1257         ret = ena_com_mem_addr_set(ena_dev,
1258                        &create_cmd.sq_ba,
1259                        io_sq->desc_addr.phys_addr);
1260         if (unlikely(ret)) {
1261             netdev_err(ena_dev->net_device,
1262                    "Memory address set failed\n");
1263             return ret;
1264         }
1265     }
1266
1267     ret = ena_com_execute_admin_command(admin_queue,
1268                         (struct ena_admin_aq_entry *)&create_cmd,
1269                         sizeof(create_cmd),
1270                         (struct ena_admin_acq_entry *)&cmd_completion,
1271                         sizeof(cmd_completion));
1272     if (unlikely(ret)) {
1273         netdev_err(ena_dev->net_device,
1274                "Failed to create IO SQ. error: %d\n", ret);
1275         return ret;
1276     }
1277
1278     io_sq->idx = cmd_completion.sq_idx;
1279
1280     io_sq->db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1281         (uintptr_t)cmd_completion.sq_doorbell_offset);
1282
1283     if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
1284         io_sq->header_addr = (u8 __iomem *)((uintptr_t)ena_dev->mem_bar
1285                 + cmd_completion.llq_headers_offset);
1286
1287         io_sq->desc_addr.pbuf_dev_addr =
1288             (u8 __iomem *)((uintptr_t)ena_dev->mem_bar +
1289             cmd_completion.llq_descriptors_offset);
1290     }
1291
1292     netdev_dbg(ena_dev->net_device, "Created sq[%u], depth[%u]\n",
1293            io_sq->idx, io_sq->q_depth);
1294
1295     return ret;
1296 }
1297
1298 static int ena_com_ind_tbl_convert_to_device(struct ena_com_dev *ena_dev)
1299 {
1300     struct ena_rss *rss = &ena_dev->rss;
1301     struct ena_com_io_sq *io_sq;
1302     u16 qid;
1303     int i;
1304
1305     for (i = 0; i < 1 << rss->tbl_log_size; i++) {
1306         qid = rss->host_rss_ind_tbl[i];
1307         if (qid >= ENA_TOTAL_NUM_QUEUES)
1308             return -EINVAL;
1309
1310         io_sq = &ena_dev->io_sq_queues[qid];
1311
1312         if (io_sq->direction != ENA_COM_IO_QUEUE_DIRECTION_RX)
1313             return -EINVAL;
1314
1315         rss->rss_ind_tbl[i].cq_idx = io_sq->idx;
1316     }
1317
1318     return 0;
1319 }
1320
1321 static void ena_com_update_intr_delay_resolution(struct ena_com_dev *ena_dev,
1322                          u16 intr_delay_resolution)
1323 {
1324     u16 prev_intr_delay_resolution = ena_dev->intr_delay_resolution;
1325
1326     if (unlikely(!intr_delay_resolution)) {
1327         netdev_err(ena_dev->net_device,
1328                "Illegal intr_delay_resolution provided. Going to use default 1 usec resolution\n");
1329         intr_delay_resolution = ENA_DEFAULT_INTR_DELAY_RESOLUTION;
1330     }
1331
1332     /* update Rx */
1333     ena_dev->intr_moder_rx_interval =
1334         ena_dev->intr_moder_rx_interval *
1335         prev_intr_delay_resolution /
1336         intr_delay_resolution;
1337
1338     /* update Tx */
1339     ena_dev->intr_moder_tx_interval =
1340         ena_dev->intr_moder_tx_interval *
1341         prev_intr_delay_resolution /
1342         intr_delay_resolution;
1343
1344     ena_dev->intr_delay_resolution = intr_delay_resolution;
1345 }
1346
1347 /*****************************************************************************/
1348 /*******************************      API       ******************************/
1349 /*****************************************************************************/
1350
1351 int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue,
1352                   struct ena_admin_aq_entry *cmd,
1353                   size_t cmd_size,
1354                   struct ena_admin_acq_entry *comp,
1355                   size_t comp_size)
1356 {
1357     struct ena_comp_ctx *comp_ctx;
1358     int ret;
1359
1360     comp_ctx = ena_com_submit_admin_cmd(admin_queue, cmd, cmd_size,
1361                         comp, comp_size);
1362     if (IS_ERR(comp_ctx)) {
1363         ret = PTR_ERR(comp_ctx);
1364         if (ret == -ENODEV)
1365             netdev_dbg(admin_queue->ena_dev->net_device,
1366                    "Failed to submit command [%d]\n", ret);
1367         else
1368             netdev_err(admin_queue->ena_dev->net_device,
1369                    "Failed to submit command [%d]\n", ret);
1370
1371         return ret;
1372     }
1373
1374     ret = ena_com_wait_and_process_admin_cq(comp_ctx, admin_queue);
1375     if (unlikely(ret)) {
1376         if (admin_queue->running_state)
1377             netdev_err(admin_queue->ena_dev->net_device,
1378                    "Failed to process command. ret = %d\n", ret);
1379         else
1380             netdev_dbg(admin_queue->ena_dev->net_device,
1381                    "Failed to process command. ret = %d\n", ret);
1382     }
1383     return ret;
1384 }
1385
1386 int ena_com_create_io_cq(struct ena_com_dev *ena_dev,
1387              struct ena_com_io_cq *io_cq)
1388 {
1389     struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1390     struct ena_admin_aq_create_cq_cmd create_cmd;
1391     struct ena_admin_acq_create_cq_resp_desc cmd_completion;
1392     int ret;
1393
1394     memset(&create_cmd, 0x0, sizeof(create_cmd));
1395
1396     create_cmd.aq_common_descriptor.opcode = ENA_ADMIN_CREATE_CQ;
1397
1398     create_cmd.cq_caps_2 |= (io_cq->cdesc_entry_size_in_bytes / 4) &
1399         ENA_ADMIN_AQ_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK;
1400     create_cmd.cq_caps_1 |=
1401         ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_MASK;
1402
1403     create_cmd.msix_vector = io_cq->msix_vector;
1404     create_cmd.cq_depth = io_cq->q_depth;
1405
1406     ret = ena_com_mem_addr_set(ena_dev,
1407                    &create_cmd.cq_ba,
1408                    io_cq->cdesc_addr.phys_addr);
1409     if (unlikely(ret)) {
1410         netdev_err(ena_dev->net_device, "Memory address set failed\n");
1411         return ret;
1412     }
1413
1414     ret = ena_com_execute_admin_command(admin_queue,
1415                         (struct ena_admin_aq_entry *)&create_cmd,
1416                         sizeof(create_cmd),
1417                         (struct ena_admin_acq_entry *)&cmd_completion,
1418                         sizeof(cmd_completion));
1419     if (unlikely(ret)) {
1420         netdev_err(ena_dev->net_device,
1421                "Failed to create IO CQ. error: %d\n", ret);
1422         return ret;
1423     }
1424
1425     io_cq->idx = cmd_completion.cq_idx;
1426
1427     io_cq->unmask_reg = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1428         cmd_completion.cq_interrupt_unmask_register_offset);
1429
1430     if (cmd_completion.cq_head_db_register_offset)
1431         io_cq->cq_head_db_reg =
1432             (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1433             cmd_completion.cq_head_db_register_offset);
1434
1435     if (cmd_completion.numa_node_register_offset)
1436         io_cq->numa_node_cfg_reg =
1437             (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1438             cmd_completion.numa_node_register_offset);
1439
1440     netdev_dbg(ena_dev->net_device, "Created cq[%u], depth[%u]\n",
1441            io_cq->idx, io_cq->q_depth);
1442
1443     return ret;
1444 }
1445
1446 int ena_com_get_io_handlers(struct ena_com_dev *ena_dev, u16 qid,
1447                 struct ena_com_io_sq **io_sq,
1448                 struct ena_com_io_cq **io_cq)
1449 {
1450     if (qid >= ENA_TOTAL_NUM_QUEUES) {
1451         netdev_err(ena_dev->net_device,
1452                "Invalid queue number %d but the max is %d\n", qid,
1453                ENA_TOTAL_NUM_QUEUES);
1454         return -EINVAL;
1455     }
1456
1457     *io_sq = &ena_dev->io_sq_queues[qid];
1458     *io_cq = &ena_dev->io_cq_queues[qid];
1459
1460     return 0;
1461 }
1462
1463 void ena_com_abort_admin_commands(struct ena_com_dev *ena_dev)
1464 {
1465     struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1466     struct ena_comp_ctx *comp_ctx;
1467     u16 i;
1468
1469     if (!admin_queue->comp_ctx)
1470         return;
1471
1472     for (i = 0; i < admin_queue->q_depth; i++) {
1473         comp_ctx = get_comp_ctxt(admin_queue, i, false);
1474         if (unlikely(!comp_ctx))
1475             break;
1476
1477         comp_ctx->status = ENA_CMD_ABORTED;
1478
1479         complete(&comp_ctx->wait_event);
1480     }
1481 }
1482
1483 void ena_com_wait_for_abort_completion(struct ena_com_dev *ena_dev)
1484 {
1485     struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1486     unsigned long flags = 0;
1487     u32 exp = 0;
1488
1489     spin_lock_irqsave(&admin_queue->q_lock, flags);
1490     while (atomic_read(&admin_queue->outstanding_cmds) != 0) {
1491         spin_unlock_irqrestore(&admin_queue->q_lock, flags);
1492         ena_delay_exponential_backoff_us(exp++,
1493                          ena_dev->ena_min_poll_delay_us);
1494         spin_lock_irqsave(&admin_queue->q_lock, flags);
1495     }
1496     spin_unlock_irqrestore(&admin_queue->q_lock, flags);
1497 }
1498
1499 int ena_com_destroy_io_cq(struct ena_com_dev *ena_dev,
1500               struct ena_com_io_cq *io_cq)
1501 {
1502     struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1503     struct ena_admin_aq_destroy_cq_cmd destroy_cmd;
1504     struct ena_admin_acq_destroy_cq_resp_desc destroy_resp;
1505     int ret;
1506
1507     memset(&destroy_cmd, 0x0, sizeof(destroy_cmd));
1508
1509     destroy_cmd.cq_idx = io_cq->idx;
1510     destroy_cmd.aq_common_descriptor.opcode = ENA_ADMIN_DESTROY_CQ;
1511
1512     ret = ena_com_execute_admin_command(admin_queue,
1513                         (struct ena_admin_aq_entry *)&destroy_cmd,
1514                         sizeof(destroy_cmd),
1515                         (struct ena_admin_acq_entry *)&destroy_resp,
1516                         sizeof(destroy_resp));
1517
1518     if (unlikely(ret && (ret != -ENODEV)))
1519         netdev_err(ena_dev->net_device,
1520                "Failed to destroy IO CQ. error: %d\n", ret);
1521
1522     return ret;
1523 }
1524
1525 bool ena_com_get_admin_running_state(struct ena_com_dev *ena_dev)
1526 {
1527     return ena_dev->admin_queue.running_state;
1528 }
1529
1530 void ena_com_set_admin_running_state(struct ena_com_dev *ena_dev, bool state)
1531 {
1532     struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1533     unsigned long flags = 0;
1534
1535     spin_lock_irqsave(&admin_queue->q_lock, flags);
1536     ena_dev->admin_queue.running_state = state;
1537     spin_unlock_irqrestore(&admin_queue->q_lock, flags);
1538 }
1539
1540 void ena_com_admin_aenq_enable(struct ena_com_dev *ena_dev)
1541 {
1542     u16 depth = ena_dev->aenq.q_depth;
1543
1544     WARN(ena_dev->aenq.head != depth, "Invalid AENQ state\n");
1545
1546     /* Init head_db to mark that all entries in the queue
1547      * are initially available
1548      */
1549     writel(depth, ena_dev->reg_bar + ENA_REGS_AENQ_HEAD_DB_OFF);
1550 }
1551
1552 int ena_com_set_aenq_config(struct ena_com_dev *ena_dev, u32 groups_flag)
1553 {
1554     struct ena_com_admin_queue *admin_queue;
1555     struct ena_admin_set_feat_cmd cmd;
1556     struct ena_admin_set_feat_resp resp;
1557     struct ena_admin_get_feat_resp get_resp;
1558     int ret;
1559
1560     ret = ena_com_get_feature(ena_dev, &get_resp, ENA_ADMIN_AENQ_CONFIG, 0);
1561     if (ret) {
1562         dev_info(ena_dev->dmadev, "Can't get aenq configuration\n");
1563         return ret;
1564     }
1565
1566     if ((get_resp.u.aenq.supported_groups & groups_flag) != groups_flag) {
1567         netdev_warn(ena_dev->net_device,
1568                 "Trying to set unsupported aenq events. supported flag: 0x%x asked flag: 0x%x\n",
1569                 get_resp.u.aenq.supported_groups, groups_flag);
1570         return -EOPNOTSUPP;
1571     }
1572
1573     memset(&cmd, 0x0, sizeof(cmd));
1574     admin_queue = &ena_dev->admin_queue;
1575
1576     cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
1577     cmd.aq_common_descriptor.flags = 0;
1578     cmd.feat_common.feature_id = ENA_ADMIN_AENQ_CONFIG;
1579     cmd.u.aenq.enabled_groups = groups_flag;
1580
1581     ret = ena_com_execute_admin_command(admin_queue,
1582                         (struct ena_admin_aq_entry *)&cmd,
1583                         sizeof(cmd),
1584                         (struct ena_admin_acq_entry *)&resp,
1585                         sizeof(resp));
1586
1587     if (unlikely(ret))
1588         netdev_err(ena_dev->net_device,
1589                "Failed to config AENQ ret: %d\n", ret);
1590
1591     return ret;
1592 }
1593
1594 int ena_com_get_dma_width(struct ena_com_dev *ena_dev)
1595 {
1596     u32 caps = ena_com_reg_bar_read32(ena_dev, ENA_REGS_CAPS_OFF);
1597     u32 width;
1598
1599     if (unlikely(caps == ENA_MMIO_READ_TIMEOUT)) {
1600         netdev_err(ena_dev->net_device, "Reg read timeout occurred\n");
1601         return -ETIME;
1602     }
1603
1604     width = (caps & ENA_REGS_CAPS_DMA_ADDR_WIDTH_MASK) >>
1605         ENA_REGS_CAPS_DMA_ADDR_WIDTH_SHIFT;
1606
1607     netdev_dbg(ena_dev->net_device, "ENA dma width: %d\n", width);
1608
1609     if ((width < 32) || width > ENA_MAX_PHYS_ADDR_SIZE_BITS) {
1610         netdev_err(ena_dev->net_device, "DMA width illegal value: %d\n",
1611                width);
1612         return -EINVAL;
1613     }
1614
1615     ena_dev->dma_addr_bits = width;
1616
1617     return width;
1618 }
1619
1620 int ena_com_validate_version(struct ena_com_dev *ena_dev)
1621 {
1622     u32 ver;
1623     u32 ctrl_ver;
1624     u32 ctrl_ver_masked;
1625
1626     /* Make sure the ENA version and the controller version are at least
1627      * as the driver expects
1628      */
1629     ver = ena_com_reg_bar_read32(ena_dev, ENA_REGS_VERSION_OFF);
1630     ctrl_ver = ena_com_reg_bar_read32(ena_dev,
1631                       ENA_REGS_CONTROLLER_VERSION_OFF);
1632
1633     if (unlikely((ver == ENA_MMIO_READ_TIMEOUT) ||
1634              (ctrl_ver == ENA_MMIO_READ_TIMEOUT))) {
1635         netdev_err(ena_dev->net_device, "Reg read timeout occurred\n");
1636         return -ETIME;
1637     }
1638
1639     dev_info(ena_dev->dmadev, "ENA device version: %d.%d\n",
1640          (ver & ENA_REGS_VERSION_MAJOR_VERSION_MASK) >>
1641              ENA_REGS_VERSION_MAJOR_VERSION_SHIFT,
1642          ver & ENA_REGS_VERSION_MINOR_VERSION_MASK);
1643
1644     dev_info(ena_dev->dmadev,
1645          "ENA controller version: %d.%d.%d implementation version %d\n",
1646          (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK) >>
1647              ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT,
1648          (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK) >>
1649              ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT,
1650          (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK),
1651          (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_IMPL_ID_MASK) >>
1652              ENA_REGS_CONTROLLER_VERSION_IMPL_ID_SHIFT);
1653
1654     ctrl_ver_masked =
1655         (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK) |
1656         (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK) |
1657         (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK);
1658
1659     /* Validate the ctrl version without the implementation ID */
1660     if (ctrl_ver_masked < MIN_ENA_CTRL_VER) {
1661         netdev_err(ena_dev->net_device,
1662                "ENA ctrl version is lower than the minimal ctrl version the driver supports\n");
1663         return -1;
1664     }
1665
1666     return 0;
1667 }
1668
1669 static void
1670 ena_com_free_ena_admin_queue_comp_ctx(struct ena_com_dev *ena_dev,
1671                       struct ena_com_admin_queue *admin_queue)
1672
1673 {
1674     if (!admin_queue->comp_ctx)
1675         return;
1676
1677     devm_kfree(ena_dev->dmadev, admin_queue->comp_ctx);
1678
1679     admin_queue->comp_ctx = NULL;
1680 }
1681
1682 void ena_com_admin_destroy(struct ena_com_dev *ena_dev)
1683 {
1684     struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1685     struct ena_com_admin_cq *cq = &admin_queue->cq;
1686     struct ena_com_admin_sq *sq = &admin_queue->sq;
1687     struct ena_com_aenq *aenq = &ena_dev->aenq;
1688     u16 size;
1689
1690     ena_com_free_ena_admin_queue_comp_ctx(ena_dev, admin_queue);
1691
1692     size = ADMIN_SQ_SIZE(admin_queue->q_depth);
1693     if (sq->entries)
1694         dma_free_coherent(ena_dev->dmadev, size, sq->entries,
1695                   sq->dma_addr);
1696     sq->entries = NULL;
1697
1698     size = ADMIN_CQ_SIZE(admin_queue->q_depth);
1699     if (cq->entries)
1700         dma_free_coherent(ena_dev->dmadev, size, cq->entries,
1701                   cq->dma_addr);
1702     cq->entries = NULL;
1703
1704     size = ADMIN_AENQ_SIZE(aenq->q_depth);
1705     if (ena_dev->aenq.entries)
1706         dma_free_coherent(ena_dev->dmadev, size, aenq->entries,
1707                   aenq->dma_addr);
1708     aenq->entries = NULL;
1709 }
1710
1711 void ena_com_set_admin_polling_mode(struct ena_com_dev *ena_dev, bool polling)
1712 {
1713     u32 mask_value = 0;
1714
1715     if (polling)
1716         mask_value = ENA_REGS_ADMIN_INTR_MASK;
1717
1718     writel(mask_value, ena_dev->reg_bar + ENA_REGS_INTR_MASK_OFF);
1719     ena_dev->admin_queue.polling = polling;
1720 }
1721
1722 void ena_com_set_admin_auto_polling_mode(struct ena_com_dev *ena_dev,
1723                      bool polling)
1724 {
1725     ena_dev->admin_queue.auto_polling = polling;
1726 }
1727
1728 int ena_com_mmio_reg_read_request_init(struct ena_com_dev *ena_dev)
1729 {
1730     struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
1731
1732     spin_lock_init(&mmio_read->lock);
1733     mmio_read->read_resp =
1734         dma_alloc_coherent(ena_dev->dmadev,
1735                    sizeof(*mmio_read->read_resp),
1736                    &mmio_read->read_resp_dma_addr, GFP_KERNEL);
1737     if (unlikely(!mmio_read->read_resp))
1738         goto err;
1739
1740     ena_com_mmio_reg_read_request_write_dev_addr(ena_dev);
1741
1742     mmio_read->read_resp->req_id = 0x0;
1743     mmio_read->seq_num = 0x0;
1744     mmio_read->readless_supported = true;
1745
1746     return 0;
1747
1748 err:
1749
1750     return -ENOMEM;
1751 }
1752
1753 void ena_com_set_mmio_read_mode(struct ena_com_dev *ena_dev, bool readless_supported)
1754 {
1755     struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
1756
1757     mmio_read->readless_supported = readless_supported;
1758 }
1759
1760 void ena_com_mmio_reg_read_request_destroy(struct ena_com_dev *ena_dev)
1761 {
1762     struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
1763
1764     writel(0x0, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_LO_OFF);
1765     writel(0x0, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_HI_OFF);
1766
1767     dma_free_coherent(ena_dev->dmadev, sizeof(*mmio_read->read_resp),
1768               mmio_read->read_resp, mmio_read->read_resp_dma_addr);
1769
1770     mmio_read->read_resp = NULL;
1771 }
1772
1773 void ena_com_mmio_reg_read_request_write_dev_addr(struct ena_com_dev *ena_dev)
1774 {
1775     struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
1776     u32 addr_low, addr_high;
1777
1778     addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(mmio_read->read_resp_dma_addr);
1779     addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(mmio_read->read_resp_dma_addr);
1780
1781     writel(addr_low, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_LO_OFF);
1782     writel(addr_high, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_HI_OFF);
1783 }
1784
1785 int ena_com_admin_init(struct ena_com_dev *ena_dev,
1786                struct ena_aenq_handlers *aenq_handlers)
1787 {
1788     struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1789     u32 aq_caps, acq_caps, dev_sts, addr_low, addr_high;
1790     int ret;
1791
1792     dev_sts = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF);
1793
1794     if (unlikely(dev_sts == ENA_MMIO_READ_TIMEOUT)) {
1795         netdev_err(ena_dev->net_device, "Reg read timeout occurred\n");
1796         return -ETIME;
1797     }
1798
1799     if (!(dev_sts & ENA_REGS_DEV_STS_READY_MASK)) {
1800         netdev_err(ena_dev->net_device,
1801                "Device isn't ready, abort com init\n");
1802         return -ENODEV;
1803     }
1804
1805     admin_queue->q_depth = ENA_ADMIN_QUEUE_DEPTH;
1806
1807     admin_queue->q_dmadev = ena_dev->dmadev;
1808     admin_queue->polling = false;
1809     admin_queue->curr_cmd_id = 0;
1810
1811     atomic_set(&admin_queue->outstanding_cmds, 0);
1812
1813     spin_lock_init(&admin_queue->q_lock);
1814
1815     ret = ena_com_init_comp_ctxt(admin_queue);
1816     if (ret)
1817         goto error;
1818
1819     ret = ena_com_admin_init_sq(admin_queue);
1820     if (ret)
1821         goto error;
1822
1823     ret = ena_com_admin_init_cq(admin_queue);
1824     if (ret)
1825         goto error;
1826
1827     admin_queue->sq.db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1828         ENA_REGS_AQ_DB_OFF);
1829
1830     addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->sq.dma_addr);
1831     addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->sq.dma_addr);
1832
1833     writel(addr_low, ena_dev->reg_bar + ENA_REGS_AQ_BASE_LO_OFF);
1834     writel(addr_high, ena_dev->reg_bar + ENA_REGS_AQ_BASE_HI_OFF);
1835
1836     addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->cq.dma_addr);
1837     addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->cq.dma_addr);
1838
1839     writel(addr_low, ena_dev->reg_bar + ENA_REGS_ACQ_BASE_LO_OFF);
1840     writel(addr_high, ena_dev->reg_bar + ENA_REGS_ACQ_BASE_HI_OFF);
1841
1842     aq_caps = 0;
1843     aq_caps |= admin_queue->q_depth & ENA_REGS_AQ_CAPS_AQ_DEPTH_MASK;
1844     aq_caps |= (sizeof(struct ena_admin_aq_entry) <<
1845             ENA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_SHIFT) &
1846             ENA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_MASK;
1847
1848     acq_caps = 0;
1849     acq_caps |= admin_queue->q_depth & ENA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK;
1850     acq_caps |= (sizeof(struct ena_admin_acq_entry) <<
1851         ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_SHIFT) &
1852         ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_MASK;
1853
1854     writel(aq_caps, ena_dev->reg_bar + ENA_REGS_AQ_CAPS_OFF);
1855     writel(acq_caps, ena_dev->reg_bar + ENA_REGS_ACQ_CAPS_OFF);
1856     ret = ena_com_admin_init_aenq(ena_dev, aenq_handlers);
1857     if (ret)
1858         goto error;
1859
1860     admin_queue->ena_dev = ena_dev;
1861     admin_queue->running_state = true;
1862
1863     return 0;
1864 error:
1865     ena_com_admin_destroy(ena_dev);
1866
1867     return ret;
1868 }
1869
1870 int ena_com_create_io_queue(struct ena_com_dev *ena_dev,
1871                 struct ena_com_create_io_ctx *ctx)
1872 {
1873     struct ena_com_io_sq *io_sq;
1874     struct ena_com_io_cq *io_cq;
1875     int ret;
1876
1877     if (ctx->qid >= ENA_TOTAL_NUM_QUEUES) {
1878         netdev_err(ena_dev->net_device,
1879                "Qid (%d) is bigger than max num of queues (%d)\n",
1880                ctx->qid, ENA_TOTAL_NUM_QUEUES);
1881         return -EINVAL;
1882     }
1883
1884     io_sq = &ena_dev->io_sq_queues[ctx->qid];
1885     io_cq = &ena_dev->io_cq_queues[ctx->qid];
1886
1887     memset(io_sq, 0x0, sizeof(*io_sq));
1888     memset(io_cq, 0x0, sizeof(*io_cq));
1889
1890     /* Init CQ */
1891     io_cq->q_depth = ctx->queue_size;
1892     io_cq->direction = ctx->direction;
1893     io_cq->qid = ctx->qid;
1894
1895     io_cq->msix_vector = ctx->msix_vector;
1896
1897     io_sq->q_depth = ctx->queue_size;
1898     io_sq->direction = ctx->direction;
1899     io_sq->qid = ctx->qid;
1900
1901     io_sq->mem_queue_type = ctx->mem_queue_type;
1902
1903     if (ctx->direction == ENA_COM_IO_QUEUE_DIRECTION_TX)
1904         /* header length is limited to 8 bits */
1905         io_sq->tx_max_header_size =
1906             min_t(u32, ena_dev->tx_max_header_size, SZ_256);
1907
1908     ret = ena_com_init_io_sq(ena_dev, ctx, io_sq);
1909     if (ret)
1910         goto error;
1911     ret = ena_com_init_io_cq(ena_dev, ctx, io_cq);
1912     if (ret)
1913         goto error;
1914
1915     ret = ena_com_create_io_cq(ena_dev, io_cq);
1916     if (ret)
1917         goto error;
1918
1919     ret = ena_com_create_io_sq(ena_dev, io_sq, io_cq->idx);
1920     if (ret)
1921         goto destroy_io_cq;
1922
1923     return 0;
1924
1925 destroy_io_cq:
1926     ena_com_destroy_io_cq(ena_dev, io_cq);
1927 error:
1928     ena_com_io_queue_free(ena_dev, io_sq, io_cq);
1929     return ret;
1930 }
1931
1932 void ena_com_destroy_io_queue(struct ena_com_dev *ena_dev, u16 qid)
1933 {
1934     struct ena_com_io_sq *io_sq;
1935     struct ena_com_io_cq *io_cq;
1936
1937     if (qid >= ENA_TOTAL_NUM_QUEUES) {
1938         netdev_err(ena_dev->net_device,
1939                "Qid (%d) is bigger than max num of queues (%d)\n",
1940                qid, ENA_TOTAL_NUM_QUEUES);
1941         return;
1942     }
1943
1944     io_sq = &ena_dev->io_sq_queues[qid];
1945     io_cq = &ena_dev->io_cq_queues[qid];
1946
1947     ena_com_destroy_io_sq(ena_dev, io_sq);
1948     ena_com_destroy_io_cq(ena_dev, io_cq);
1949
1950     ena_com_io_queue_free(ena_dev, io_sq, io_cq);
1951 }
1952
1953 int ena_com_get_link_params(struct ena_com_dev *ena_dev,
1954                 struct ena_admin_get_feat_resp *resp)
1955 {
1956     return ena_com_get_feature(ena_dev, resp, ENA_ADMIN_LINK_CONFIG, 0);
1957 }
1958
1959 int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev,
1960                   struct ena_com_dev_get_features_ctx *get_feat_ctx)
1961 {
1962     struct ena_admin_get_feat_resp get_resp;
1963     int rc;
1964
1965     rc = ena_com_get_feature(ena_dev, &get_resp,
1966                  ENA_ADMIN_DEVICE_ATTRIBUTES, 0);
1967     if (rc)
1968         return rc;
1969
1970     memcpy(&get_feat_ctx->dev_attr, &get_resp.u.dev_attr,
1971            sizeof(get_resp.u.dev_attr));
1972
1973     ena_dev->supported_features = get_resp.u.dev_attr.supported_features;
1974     ena_dev->capabilities = get_resp.u.dev_attr.capabilities;
1975
1976     if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {
1977         rc = ena_com_get_feature(ena_dev, &get_resp,
1978                      ENA_ADMIN_MAX_QUEUES_EXT,
1979                      ENA_FEATURE_MAX_QUEUE_EXT_VER);
1980         if (rc)
1981             return rc;
1982
1983         if (get_resp.u.max_queue_ext.version !=
1984             ENA_FEATURE_MAX_QUEUE_EXT_VER)
1985             return -EINVAL;
1986
1987         memcpy(&get_feat_ctx->max_queue_ext, &get_resp.u.max_queue_ext,
1988                sizeof(get_resp.u.max_queue_ext));
1989         ena_dev->tx_max_header_size =
1990             get_resp.u.max_queue_ext.max_queue_ext.max_tx_header_size;
1991     } else {
1992         rc = ena_com_get_feature(ena_dev, &get_resp,
1993                      ENA_ADMIN_MAX_QUEUES_NUM, 0);
1994         memcpy(&get_feat_ctx->max_queues, &get_resp.u.max_queue,
1995                sizeof(get_resp.u.max_queue));
1996         ena_dev->tx_max_header_size =
1997             get_resp.u.max_queue.max_header_size;
1998
1999         if (rc)
2000             return rc;
2001     }
2002
2003     rc = ena_com_get_feature(ena_dev, &get_resp,
2004                  ENA_ADMIN_AENQ_CONFIG, 0);
2005     if (rc)
2006         return rc;
2007
2008     memcpy(&get_feat_ctx->aenq, &get_resp.u.aenq,
2009            sizeof(get_resp.u.aenq));
2010
2011     rc = ena_com_get_feature(ena_dev, &get_resp,
2012                  ENA_ADMIN_STATELESS_OFFLOAD_CONFIG, 0);
2013     if (rc)
2014         return rc;
2015
2016     memcpy(&get_feat_ctx->offload, &get_resp.u.offload,
2017            sizeof(get_resp.u.offload));
2018
2019     /* Driver hints isn't mandatory admin command. So in case the
2020      * command isn't supported set driver hints to 0
2021      */
2022     rc = ena_com_get_feature(ena_dev, &get_resp, ENA_ADMIN_HW_HINTS, 0);
2023
2024     if (!rc)
2025         memcpy(&get_feat_ctx->hw_hints, &get_resp.u.hw_hints,
2026                sizeof(get_resp.u.hw_hints));
2027     else if (rc == -EOPNOTSUPP)
2028         memset(&get_feat_ctx->hw_hints, 0x0,
2029                sizeof(get_feat_ctx->hw_hints));
2030     else
2031         return rc;
2032
2033     rc = ena_com_get_feature(ena_dev, &get_resp, ENA_ADMIN_LLQ, 0);
2034     if (!rc)
2035         memcpy(&get_feat_ctx->llq, &get_resp.u.llq,
2036                sizeof(get_resp.u.llq));
2037     else if (rc == -EOPNOTSUPP)
2038         memset(&get_feat_ctx->llq, 0x0, sizeof(get_feat_ctx->llq));
2039     else
2040         return rc;
2041
2042     return 0;
2043 }
2044
2045 void ena_com_admin_q_comp_intr_handler(struct ena_com_dev *ena_dev)
2046 {
2047     ena_com_handle_admin_completion(&ena_dev->admin_queue);
2048 }
2049
2050 /* ena_handle_specific_aenq_event:
2051  * return the handler that is relevant to the specific event group
2052  */
2053 static ena_aenq_handler ena_com_get_specific_aenq_cb(struct ena_com_dev *ena_dev,
2054                              u16 group)
2055 {
2056     struct ena_aenq_handlers *aenq_handlers = ena_dev->aenq.aenq_handlers;
2057
2058     if ((group < ENA_MAX_HANDLERS) && aenq_handlers->handlers[group])
2059         return aenq_handlers->handlers[group];
2060
2061     return aenq_handlers->unimplemented_handler;
2062 }
2063
2064 /* ena_aenq_intr_handler:
2065  * handles the aenq incoming events.
2066  * pop events from the queue and apply the specific handler
2067  */
2068 void ena_com_aenq_intr_handler(struct ena_com_dev *ena_dev, void *data)
2069 {
2070     struct ena_admin_aenq_entry *aenq_e;
2071     struct ena_admin_aenq_common_desc *aenq_common;
2072     struct ena_com_aenq *aenq  = &ena_dev->aenq;
2073     u64 timestamp;
2074     ena_aenq_handler handler_cb;
2075     u16 masked_head, processed = 0;
2076     u8 phase;
2077
2078     masked_head = aenq->head & (aenq->q_depth - 1);
2079     phase = aenq->phase;
2080     aenq_e = &aenq->entries[masked_head]; /* Get first entry */
2081     aenq_common = &aenq_e->aenq_common_desc;
2082
2083     /* Go over all the events */
2084     while ((READ_ONCE(aenq_common->flags) &
2085         ENA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK) == phase) {
2086         /* Make sure the phase bit (ownership) is as expected before
2087          * reading the rest of the descriptor.
2088          */
2089         dma_rmb();
2090
2091         timestamp = (u64)aenq_common->timestamp_low |
2092             ((u64)aenq_common->timestamp_high << 32);
2093
2094         netdev_dbg(ena_dev->net_device,
2095                "AENQ! Group[%x] Syndrome[%x] timestamp: [%llus]\n",
2096                aenq_common->group, aenq_common->syndrome, timestamp);
2097
2098         /* Handle specific event*/
2099         handler_cb = ena_com_get_specific_aenq_cb(ena_dev,
2100                               aenq_common->group);
2101         handler_cb(data, aenq_e); /* call the actual event handler*/
2102
2103         /* Get next event entry */
2104         masked_head++;
2105         processed++;
2106
2107         if (unlikely(masked_head == aenq->q_depth)) {
2108             masked_head = 0;
2109             phase = !phase;
2110         }
2111         aenq_e = &aenq->entries[masked_head];
2112         aenq_common = &aenq_e->aenq_common_desc;
2113     }
2114
2115     aenq->head += processed;
2116     aenq->phase = phase;
2117
2118     /* Don't update aenq doorbell if there weren't any processed events */
2119     if (!processed)
2120         return;
2121
2122     /* write the aenq doorbell after all AENQ descriptors were read */
2123     mb();
2124     writel_relaxed((u32)aenq->head,
2125                ena_dev->reg_bar + ENA_REGS_AENQ_HEAD_DB_OFF);
2126 }
2127
2128 int ena_com_dev_reset(struct ena_com_dev *ena_dev,
2129               enum ena_regs_reset_reason_types reset_reason)
2130 {
2131     u32 stat, timeout, cap, reset_val;
2132     int rc;
2133
2134     stat = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF);
2135     cap = ena_com_reg_bar_read32(ena_dev, ENA_REGS_CAPS_OFF);
2136
2137     if (unlikely((stat == ENA_MMIO_READ_TIMEOUT) ||
2138              (cap == ENA_MMIO_READ_TIMEOUT))) {
2139         netdev_err(ena_dev->net_device, "Reg read32 timeout occurred\n");
2140         return -ETIME;
2141     }
2142
2143     if ((stat & ENA_REGS_DEV_STS_READY_MASK) == 0) {
2144         netdev_err(ena_dev->net_device,
2145                "Device isn't ready, can't reset device\n");
2146         return -EINVAL;
2147     }
2148
2149     timeout = (cap & ENA_REGS_CAPS_RESET_TIMEOUT_MASK) >>
2150             ENA_REGS_CAPS_RESET_TIMEOUT_SHIFT;
2151     if (timeout == 0) {
2152         netdev_err(ena_dev->net_device, "Invalid timeout value\n");
2153         return -EINVAL;
2154     }
2155
2156     /* start reset */
2157     reset_val = ENA_REGS_DEV_CTL_DEV_RESET_MASK;
2158     reset_val |= (reset_reason << ENA_REGS_DEV_CTL_RESET_REASON_SHIFT) &
2159              ENA_REGS_DEV_CTL_RESET_REASON_MASK;
2160     writel(reset_val, ena_dev->reg_bar + ENA_REGS_DEV_CTL_OFF);
2161
2162     /* Write again the MMIO read request address */
2163     ena_com_mmio_reg_read_request_write_dev_addr(ena_dev);
2164
2165     rc = wait_for_reset_state(ena_dev, timeout,
2166                   ENA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK);
2167     if (rc != 0) {
2168         netdev_err(ena_dev->net_device,
2169                "Reset indication didn't turn on\n");
2170         return rc;
2171     }
2172
2173     /* reset done */
2174     writel(0, ena_dev->reg_bar + ENA_REGS_DEV_CTL_OFF);
2175     rc = wait_for_reset_state(ena_dev, timeout, 0);
2176     if (rc != 0) {
2177         netdev_err(ena_dev->net_device,
2178                "Reset indication didn't turn off\n");
2179         return rc;
2180     }
2181
2182     timeout = (cap & ENA_REGS_CAPS_ADMIN_CMD_TO_MASK) >>
2183         ENA_REGS_CAPS_ADMIN_CMD_TO_SHIFT;
2184     if (timeout)
2185         /* the resolution of timeout reg is 100ms */
2186         ena_dev->admin_queue.completion_timeout = timeout * 100000;
2187     else
2188         ena_dev->admin_queue.completion_timeout = ADMIN_CMD_TIMEOUT_US;
2189
2190     return 0;
2191 }
2192
2193 static int ena_get_dev_stats(struct ena_com_dev *ena_dev,
2194                  struct ena_com_stats_ctx *ctx,
2195                  enum ena_admin_get_stats_type type)
2196 {
2197     struct ena_admin_aq_get_stats_cmd *get_cmd = &ctx->get_cmd;
2198     struct ena_admin_acq_get_stats_resp *get_resp = &ctx->get_resp;
2199     struct ena_com_admin_queue *admin_queue;
2200     int ret;
2201
2202     admin_queue = &ena_dev->admin_queue;
2203
2204     get_cmd->aq_common_descriptor.opcode = ENA_ADMIN_GET_STATS;
2205     get_cmd->aq_common_descriptor.flags = 0;
2206     get_cmd->type = type;
2207
2208     ret =  ena_com_execute_admin_command(admin_queue,
2209                          (struct ena_admin_aq_entry *)get_cmd,
2210                          sizeof(*get_cmd),
2211                          (struct ena_admin_acq_entry *)get_resp,
2212                          sizeof(*get_resp));
2213
2214     if (unlikely(ret))
2215         netdev_err(ena_dev->net_device,
2216                "Failed to get stats. error: %d\n", ret);
2217
2218     return ret;
2219 }
2220
2221 int ena_com_get_eni_stats(struct ena_com_dev *ena_dev,
2222               struct ena_admin_eni_stats *stats)
2223 {
2224     struct ena_com_stats_ctx ctx;
2225     int ret;
2226
2227     if (!ena_com_get_cap(ena_dev, ENA_ADMIN_ENI_STATS)) {
2228         netdev_err(ena_dev->net_device,
2229                "Capability %d isn't supported\n",
2230                ENA_ADMIN_ENI_STATS);
2231         return -EOPNOTSUPP;
2232     }
2233
2234     memset(&ctx, 0x0, sizeof(ctx));
2235     ret = ena_get_dev_stats(ena_dev, &ctx, ENA_ADMIN_GET_STATS_TYPE_ENI);
2236     if (likely(ret == 0))
2237         memcpy(stats, &ctx.get_resp.u.eni_stats,
2238                sizeof(ctx.get_resp.u.eni_stats));
2239
2240     return ret;
2241 }
2242
2243 int ena_com_get_dev_basic_stats(struct ena_com_dev *ena_dev,
2244                 struct ena_admin_basic_stats *stats)
2245 {
2246     struct ena_com_stats_ctx ctx;
2247     int ret;
2248
2249     memset(&ctx, 0x0, sizeof(ctx));
2250     ret = ena_get_dev_stats(ena_dev, &ctx, ENA_ADMIN_GET_STATS_TYPE_BASIC);
2251     if (likely(ret == 0))
2252         memcpy(stats, &ctx.get_resp.u.basic_stats,
2253                sizeof(ctx.get_resp.u.basic_stats));
2254
2255     return ret;
2256 }
2257
2258 int ena_com_set_dev_mtu(struct ena_com_dev *ena_dev, u32 mtu)
2259 {
2260     struct ena_com_admin_queue *admin_queue;
2261     struct ena_admin_set_feat_cmd cmd;
2262     struct ena_admin_set_feat_resp resp;
2263     int ret;
2264
2265     if (!ena_com_check_supported_feature_id(ena_dev, ENA_ADMIN_MTU)) {
2266         netdev_dbg(ena_dev->net_device, "Feature %d isn't supported\n",
2267                ENA_ADMIN_MTU);
2268         return -EOPNOTSUPP;
2269     }
2270
2271     memset(&cmd, 0x0, sizeof(cmd));
2272     admin_queue = &ena_dev->admin_queue;
2273
2274     cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
2275     cmd.aq_common_descriptor.flags = 0;
2276     cmd.feat_common.feature_id = ENA_ADMIN_MTU;
2277     cmd.u.mtu.mtu = mtu;
2278
2279     ret = ena_com_execute_admin_command(admin_queue,
2280                         (struct ena_admin_aq_entry *)&cmd,
2281                         sizeof(cmd),
2282                         (struct ena_admin_acq_entry *)&resp,
2283                         sizeof(resp));
2284
2285     if (unlikely(ret))
2286         netdev_err(ena_dev->net_device,
2287                "Failed to set mtu %d. error: %d\n", mtu, ret);
2288
2289     return ret;
2290 }
2291
2292 int ena_com_get_offload_settings(struct ena_com_dev *ena_dev,
2293                  struct ena_admin_feature_offload_desc *offload)
2294 {
2295     int ret;
2296     struct ena_admin_get_feat_resp resp;
2297
2298     ret = ena_com_get_feature(ena_dev, &resp,
2299                   ENA_ADMIN_STATELESS_OFFLOAD_CONFIG, 0);
2300     if (unlikely(ret)) {
2301         netdev_err(ena_dev->net_device,
2302                "Failed to get offload capabilities %d\n", ret);
2303         return ret;
2304     }
2305
2306     memcpy(offload, &resp.u.offload, sizeof(resp.u.offload));
2307
2308     return 0;
2309 }
2310
2311 int ena_com_set_hash_function(struct ena_com_dev *ena_dev)
2312 {
2313     struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
2314     struct ena_rss *rss = &ena_dev->rss;
2315     struct ena_admin_set_feat_cmd cmd;
2316     struct ena_admin_set_feat_resp resp;
2317     struct ena_admin_get_feat_resp get_resp;
2318     int ret;
2319
2320     if (!ena_com_check_supported_feature_id(ena_dev,
2321                         ENA_ADMIN_RSS_HASH_FUNCTION)) {
2322         netdev_dbg(ena_dev->net_device, "Feature %d isn't supported\n",
2323                ENA_ADMIN_RSS_HASH_FUNCTION);
2324         return -EOPNOTSUPP;
2325     }
2326
2327     /* Validate hash function is supported */
2328     ret = ena_com_get_feature(ena_dev, &get_resp,
2329                   ENA_ADMIN_RSS_HASH_FUNCTION, 0);
2330     if (unlikely(ret))
2331         return ret;
2332
2333     if (!(get_resp.u.flow_hash_func.supported_func & BIT(rss->hash_func))) {
2334         netdev_err(ena_dev->net_device,
2335                "Func hash %d isn't supported by device, abort\n",
2336                rss->hash_func);
2337         return -EOPNOTSUPP;
2338     }
2339
2340     memset(&cmd, 0x0, sizeof(cmd));
2341
2342     cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
2343     cmd.aq_common_descriptor.flags =
2344         ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
2345     cmd.feat_common.feature_id = ENA_ADMIN_RSS_HASH_FUNCTION;
2346     cmd.u.flow_hash_func.init_val = rss->hash_init_val;
2347     cmd.u.flow_hash_func.selected_func = 1 << rss->hash_func;
2348
2349     ret = ena_com_mem_addr_set(ena_dev,
2350                    &cmd.control_buffer.address,
2351                    rss->hash_key_dma_addr);
2352     if (unlikely(ret)) {
2353         netdev_err(ena_dev->net_device, "Memory address set failed\n");
2354         return ret;
2355     }
2356
2357     cmd.control_buffer.length = sizeof(*rss->hash_key);
2358
2359     ret = ena_com_execute_admin_command(admin_queue,
2360                         (struct ena_admin_aq_entry *)&cmd,
2361                         sizeof(cmd),
2362                         (struct ena_admin_acq_entry *)&resp,
2363                         sizeof(resp));
2364     if (unlikely(ret)) {
2365         netdev_err(ena_dev->net_device,
2366                "Failed to set hash function %d. error: %d\n",
2367                rss->hash_func, ret);
2368         return -EINVAL;
2369     }
2370
2371     return 0;
2372 }
2373
2374 int ena_com_fill_hash_function(struct ena_com_dev *ena_dev,
2375                    enum ena_admin_hash_functions func,
2376                    const u8 *key, u16 key_len, u32 init_val)
2377 {
2378     struct ena_admin_feature_rss_flow_hash_control *hash_key;
2379     struct ena_admin_get_feat_resp get_resp;
2380     enum ena_admin_hash_functions old_func;
2381     struct ena_rss *rss = &ena_dev->rss;
2382     int rc;
2383
2384     hash_key = rss->hash_key;
2385
2386     /* Make sure size is a mult of DWs */
2387     if (unlikely(key_len & 0x3))
2388         return -EINVAL;
2389
2390     rc = ena_com_get_feature_ex(ena_dev, &get_resp,
2391                     ENA_ADMIN_RSS_HASH_FUNCTION,
2392                     rss->hash_key_dma_addr,
2393                     sizeof(*rss->hash_key), 0);
2394     if (unlikely(rc))
2395         return rc;
2396
2397     if (!(BIT(func) & get_resp.u.flow_hash_func.supported_func)) {
2398         netdev_err(ena_dev->net_device,
2399                "Flow hash function %d isn't supported\n", func);
2400         return -EOPNOTSUPP;
2401     }
2402
2403     switch (func) {
2404     case ENA_ADMIN_TOEPLITZ:
2405         if (key) {
2406             if (key_len != sizeof(hash_key->key)) {
2407                 netdev_err(ena_dev->net_device,
2408                        "key len (%u) doesn't equal the supported size (%zu)\n",
2409                        key_len, sizeof(hash_key->key));
2410                 return -EINVAL;
2411             }
2412             memcpy(hash_key->key, key, key_len);
2413             rss->hash_init_val = init_val;
2414             hash_key->key_parts = key_len / sizeof(hash_key->key[0]);
2415         }
2416         break;
2417     case ENA_ADMIN_CRC32:
2418         rss->hash_init_val = init_val;
2419         break;
2420     default:
2421         netdev_err(ena_dev->net_device, "Invalid hash function (%d)\n",
2422                func);
2423         return -EINVAL;
2424     }
2425
2426     old_func = rss->hash_func;
2427     rss->hash_func = func;
2428     rc = ena_com_set_hash_function(ena_dev);
2429
2430     /* Restore the old function */
2431     if (unlikely(rc))
2432         rss->hash_func = old_func;
2433
2434     return rc;
2435 }
2436
2437 int ena_com_get_hash_function(struct ena_com_dev *ena_dev,
2438                   enum ena_admin_hash_functions *func)
2439 {
2440     struct ena_rss *rss = &ena_dev->rss;
2441     struct ena_admin_get_feat_resp get_resp;
2442     int rc;
2443
2444     if (unlikely(!func))
2445         return -EINVAL;
2446
2447     rc = ena_com_get_feature_ex(ena_dev, &get_resp,
2448                     ENA_ADMIN_RSS_HASH_FUNCTION,
2449                     rss->hash_key_dma_addr,
2450                     sizeof(*rss->hash_key), 0);
2451     if (unlikely(rc))
2452         return rc;
2453
2454     /* ffs() returns 1 in case the lsb is set */
2455     rss->hash_func = ffs(get_resp.u.flow_hash_func.selected_func);
2456     if (rss->hash_func)
2457         rss->hash_func--;
2458
2459     *func = rss->hash_func;
2460
2461     return 0;
2462 }
2463
2464 int ena_com_get_hash_key(struct ena_com_dev *ena_dev, u8 *key)
2465 {
2466     struct ena_admin_feature_rss_flow_hash_control *hash_key =
2467         ena_dev->rss.hash_key;
2468
2469     if (key)
2470         memcpy(key, hash_key->key,
2471                (size_t)(hash_key->key_parts) * sizeof(hash_key->key[0]));
2472
2473     return 0;
2474 }
2475
2476 int ena_com_get_hash_ctrl(struct ena_com_dev *ena_dev,
2477               enum ena_admin_flow_hash_proto proto,
2478               u16 *fields)
2479 {
2480     struct ena_rss *rss = &ena_dev->rss;
2481     struct ena_admin_get_feat_resp get_resp;
2482     int rc;
2483
2484     rc = ena_com_get_feature_ex(ena_dev, &get_resp,
2485                     ENA_ADMIN_RSS_HASH_INPUT,
2486                     rss->hash_ctrl_dma_addr,
2487                     sizeof(*rss->hash_ctrl), 0);
2488     if (unlikely(rc))
2489         return rc;
2490
2491     if (fields)
2492         *fields = rss->hash_ctrl->selected_fields[proto].fields;
2493
2494     return 0;
2495 }
2496
2497 int ena_com_set_hash_ctrl(struct ena_com_dev *ena_dev)
2498 {
2499     struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
2500     struct ena_rss *rss = &ena_dev->rss;
2501     struct ena_admin_feature_rss_hash_control *hash_ctrl = rss->hash_ctrl;
2502     struct ena_admin_set_feat_cmd cmd;
2503     struct ena_admin_set_feat_resp resp;
2504     int ret;
2505
2506     if (!ena_com_check_supported_feature_id(ena_dev,
2507                         ENA_ADMIN_RSS_HASH_INPUT)) {
2508         netdev_dbg(ena_dev->net_device, "Feature %d isn't supported\n",
2509                ENA_ADMIN_RSS_HASH_INPUT);
2510         return -EOPNOTSUPP;
2511     }
2512
2513     memset(&cmd, 0x0, sizeof(cmd));
2514
2515     cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
2516     cmd.aq_common_descriptor.flags =
2517         ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
2518     cmd.feat_common.feature_id = ENA_ADMIN_RSS_HASH_INPUT;
2519     cmd.u.flow_hash_input.enabled_input_sort =
2520         ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_MASK |
2521         ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_MASK;
2522
2523     ret = ena_com_mem_addr_set(ena_dev,
2524                    &cmd.control_buffer.address,
2525                    rss->hash_ctrl_dma_addr);
2526     if (unlikely(ret)) {
2527         netdev_err(ena_dev->net_device, "Memory address set failed\n");
2528         return ret;
2529     }
2530     cmd.control_buffer.length = sizeof(*hash_ctrl);
2531
2532     ret = ena_com_execute_admin_command(admin_queue,
2533                         (struct ena_admin_aq_entry *)&cmd,
2534                         sizeof(cmd),
2535                         (struct ena_admin_acq_entry *)&resp,
2536                         sizeof(resp));
2537     if (unlikely(ret))
2538         netdev_err(ena_dev->net_device,
2539                "Failed to set hash input. error: %d\n", ret);
2540
2541     return ret;
2542 }
2543
2544 int ena_com_set_default_hash_ctrl(struct ena_com_dev *ena_dev)
2545 {
2546     struct ena_rss *rss = &ena_dev->rss;
2547     struct ena_admin_feature_rss_hash_control *hash_ctrl =
2548         rss->hash_ctrl;
2549     u16 available_fields = 0;
2550     int rc, i;
2551
2552     /* Get the supported hash input */
2553     rc = ena_com_get_hash_ctrl(ena_dev, 0, NULL);
2554     if (unlikely(rc))
2555         return rc;
2556
2557     hash_ctrl->selected_fields[ENA_ADMIN_RSS_TCP4].fields =
2558         ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA |
2559         ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP;
2560
2561     hash_ctrl->selected_fields[ENA_ADMIN_RSS_UDP4].fields =
2562         ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA |
2563         ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP;
2564
2565     hash_ctrl->selected_fields[ENA_ADMIN_RSS_TCP6].fields =
2566         ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA |
2567         ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP;
2568
2569     hash_ctrl->selected_fields[ENA_ADMIN_RSS_UDP6].fields =
2570         ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA |
2571         ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP;
2572
2573     hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP4].fields =
2574         ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA;
2575
2576     hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP6].fields =
2577         ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA;
2578
2579     hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP4_FRAG].fields =
2580         ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA;
2581
2582     hash_ctrl->selected_fields[ENA_ADMIN_RSS_NOT_IP].fields =
2583         ENA_ADMIN_RSS_L2_DA | ENA_ADMIN_RSS_L2_SA;
2584
2585     for (i = 0; i < ENA_ADMIN_RSS_PROTO_NUM; i++) {
2586         available_fields = hash_ctrl->selected_fields[i].fields &
2587                 hash_ctrl->supported_fields[i].fields;
2588         if (available_fields != hash_ctrl->selected_fields[i].fields) {
2589             netdev_err(ena_dev->net_device,
2590                    "Hash control doesn't support all the desire configuration. proto %x supported %x selected %x\n",
2591                    i, hash_ctrl->supported_fields[i].fields,
2592                    hash_ctrl->selected_fields[i].fields);
2593             return -EOPNOTSUPP;
2594         }
2595     }
2596
2597     rc = ena_com_set_hash_ctrl(ena_dev);
2598
2599     /* In case of failure, restore the old hash ctrl */
2600     if (unlikely(rc))
2601         ena_com_get_hash_ctrl(ena_dev, 0, NULL);
2602
2603     return rc;
2604 }
2605
2606 int ena_com_fill_hash_ctrl(struct ena_com_dev *ena_dev,
2607                enum ena_admin_flow_hash_proto proto,
2608                u16 hash_fields)
2609 {
2610     struct ena_rss *rss = &ena_dev->rss;
2611     struct ena_admin_feature_rss_hash_control *hash_ctrl = rss->hash_ctrl;
2612     u16 supported_fields;
2613     int rc;
2614
2615     if (proto >= ENA_ADMIN_RSS_PROTO_NUM) {
2616         netdev_err(ena_dev->net_device, "Invalid proto num (%u)\n",
2617                proto);
2618         return -EINVAL;
2619     }
2620
2621     /* Get the ctrl table */
2622     rc = ena_com_get_hash_ctrl(ena_dev, proto, NULL);
2623     if (unlikely(rc))
2624         return rc;
2625
2626     /* Make sure all the fields are supported */
2627     supported_fields = hash_ctrl->supported_fields[proto].fields;
2628     if ((hash_fields & supported_fields) != hash_fields) {
2629         netdev_err(ena_dev->net_device,
2630                "Proto %d doesn't support the required fields %x. supports only: %x\n",
2631                proto, hash_fields, supported_fields);
2632     }
2633
2634     hash_ctrl->selected_fields[proto].fields = hash_fields;
2635
2636     rc = ena_com_set_hash_ctrl(ena_dev);
2637
2638     /* In case of failure, restore the old hash ctrl */
2639     if (unlikely(rc))
2640         ena_com_get_hash_ctrl(ena_dev, 0, NULL);
2641
2642     return 0;
2643 }
2644
2645 int ena_com_indirect_table_fill_entry(struct ena_com_dev *ena_dev,
2646                       u16 entry_idx, u16 entry_value)
2647 {
2648     struct ena_rss *rss = &ena_dev->rss;
2649
2650     if (unlikely(entry_idx >= (1 << rss->tbl_log_size)))
2651         return -EINVAL;
2652
2653     if (unlikely((entry_value > ENA_TOTAL_NUM_QUEUES)))
2654         return -EINVAL;
2655
2656     rss->host_rss_ind_tbl[entry_idx] = entry_value;
2657
2658     return 0;
2659 }
2660
2661 int ena_com_indirect_table_set(struct ena_com_dev *ena_dev)
2662 {
2663     struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
2664     struct ena_rss *rss = &ena_dev->rss;
2665     struct ena_admin_set_feat_cmd cmd;
2666     struct ena_admin_set_feat_resp resp;
2667     int ret;
2668
2669     if (!ena_com_check_supported_feature_id(
2670             ena_dev, ENA_ADMIN_RSS_INDIRECTION_TABLE_CONFIG)) {
2671         netdev_dbg(ena_dev->net_device, "Feature %d isn't supported\n",
2672                ENA_ADMIN_RSS_INDIRECTION_TABLE_CONFIG);
2673         return -EOPNOTSUPP;
2674     }
2675
2676     ret = ena_com_ind_tbl_convert_to_device(ena_dev);
2677     if (ret) {
2678         netdev_err(ena_dev->net_device,
2679                "Failed to convert host indirection table to device table\n");
2680         return ret;
2681     }
2682
2683     memset(&cmd, 0x0, sizeof(cmd));
2684
2685     cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
2686     cmd.aq_common_descriptor.flags =
2687         ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
2688     cmd.feat_common.feature_id = ENA_ADMIN_RSS_INDIRECTION_TABLE_CONFIG;
2689     cmd.u.ind_table.size = rss->tbl_log_size;
2690     cmd.u.ind_table.inline_index = 0xFFFFFFFF;
2691
2692     ret = ena_com_mem_addr_set(ena_dev,
2693                    &cmd.control_buffer.address,
2694                    rss->rss_ind_tbl_dma_addr);
2695     if (unlikely(ret)) {
2696         netdev_err(ena_dev->net_device, "Memory address set failed\n");
2697         return ret;
2698     }
2699
2700     cmd.control_buffer.length = (1ULL << rss->tbl_log_size) *
2701         sizeof(struct ena_admin_rss_ind_table_entry);
2702
2703     ret = ena_com_execute_admin_command(admin_queue,
2704                         (struct ena_admin_aq_entry *)&cmd,
2705                         sizeof(cmd),
2706                         (struct ena_admin_acq_entry *)&resp,
2707                         sizeof(resp));
2708
2709     if (unlikely(ret))
2710         netdev_err(ena_dev->net_device,
2711                "Failed to set indirect table. error: %d\n", ret);
2712
2713     return ret;
2714 }
2715
2716 int ena_com_indirect_table_get(struct ena_com_dev *ena_dev, u32 *ind_tbl)
2717 {
2718     struct ena_rss *rss = &ena_dev->rss;
2719     struct ena_admin_get_feat_resp get_resp;
2720     u32 tbl_size;
2721     int i, rc;
2722
2723     tbl_size = (1ULL << rss->tbl_log_size) *
2724         sizeof(struct ena_admin_rss_ind_table_entry);
2725
2726     rc = ena_com_get_feature_ex(ena_dev, &get_resp,
2727                     ENA_ADMIN_RSS_INDIRECTION_TABLE_CONFIG,
2728                     rss->rss_ind_tbl_dma_addr,
2729                     tbl_size, 0);
2730     if (unlikely(rc))
2731         return rc;
2732
2733     if (!ind_tbl)
2734         return 0;
2735
2736     for (i = 0; i < (1 << rss->tbl_log_size); i++)
2737         ind_tbl[i] = rss->host_rss_ind_tbl[i];
2738
2739     return 0;
2740 }
2741
2742 int ena_com_rss_init(struct ena_com_dev *ena_dev, u16 indr_tbl_log_size)
2743 {
2744     int rc;
2745
2746     memset(&ena_dev->rss, 0x0, sizeof(ena_dev->rss));
2747
2748     rc = ena_com_indirect_table_allocate(ena_dev, indr_tbl_log_size);
2749     if (unlikely(rc))
2750         goto err_indr_tbl;
2751
2752     /* The following function might return unsupported in case the
2753      * device doesn't support setting the key / hash function. We can safely
2754      * ignore this error and have indirection table support only.
2755      */
2756     rc = ena_com_hash_key_allocate(ena_dev);
2757     if (likely(!rc))
2758         ena_com_hash_key_fill_default_key(ena_dev);
2759     else if (rc != -EOPNOTSUPP)
2760         goto err_hash_key;
2761
2762     rc = ena_com_hash_ctrl_init(ena_dev);
2763     if (unlikely(rc))
2764         goto err_hash_ctrl;
2765
2766     return 0;
2767
2768 err_hash_ctrl:
2769     ena_com_hash_key_destroy(ena_dev);
2770 err_hash_key:
2771     ena_com_indirect_table_destroy(ena_dev);
2772 err_indr_tbl:
2773
2774     return rc;
2775 }
2776
2777 void ena_com_rss_destroy(struct ena_com_dev *ena_dev)
2778 {
2779     ena_com_indirect_table_destroy(ena_dev);
2780     ena_com_hash_key_destroy(ena_dev);
2781     ena_com_hash_ctrl_destroy(ena_dev);
2782
2783     memset(&ena_dev->rss, 0x0, sizeof(ena_dev->rss));
2784 }
2785
2786 int ena_com_allocate_host_info(struct ena_com_dev *ena_dev)
2787 {
2788     struct ena_host_attribute *host_attr = &ena_dev->host_attr;
2789
2790     host_attr->host_info =
2791         dma_alloc_coherent(ena_dev->dmadev, SZ_4K,
2792                    &host_attr->host_info_dma_addr, GFP_KERNEL);
2793     if (unlikely(!host_attr->host_info))
2794         return -ENOMEM;
2795
2796     host_attr->host_info->ena_spec_version = ((ENA_COMMON_SPEC_VERSION_MAJOR <<
2797         ENA_REGS_VERSION_MAJOR_VERSION_SHIFT) |
2798         (ENA_COMMON_SPEC_VERSION_MINOR));
2799
2800     return 0;
2801 }
2802
2803 int ena_com_allocate_debug_area(struct ena_com_dev *ena_dev,
2804                 u32 debug_area_size)
2805 {
2806     struct ena_host_attribute *host_attr = &ena_dev->host_attr;
2807
2808     host_attr->debug_area_virt_addr =
2809         dma_alloc_coherent(ena_dev->dmadev, debug_area_size,
2810                    &host_attr->debug_area_dma_addr, GFP_KERNEL);
2811     if (unlikely(!host_attr->debug_area_virt_addr)) {
2812         host_attr->debug_area_size = 0;
2813         return -ENOMEM;
2814     }
2815
2816     host_attr->debug_area_size = debug_area_size;
2817
2818     return 0;
2819 }
2820
2821 void ena_com_delete_host_info(struct ena_com_dev *ena_dev)
2822 {
2823     struct ena_host_attribute *host_attr = &ena_dev->host_attr;
2824
2825     if (host_attr->host_info) {
2826         dma_free_coherent(ena_dev->dmadev, SZ_4K, host_attr->host_info,
2827                   host_attr->host_info_dma_addr);
2828         host_attr->host_info = NULL;
2829     }
2830 }
2831
2832 void ena_com_delete_debug_area(struct ena_com_dev *ena_dev)
2833 {
2834     struct ena_host_attribute *host_attr = &ena_dev->host_attr;
2835
2836     if (host_attr->debug_area_virt_addr) {
2837         dma_free_coherent(ena_dev->dmadev, host_attr->debug_area_size,
2838                   host_attr->debug_area_virt_addr,
2839                   host_attr->debug_area_dma_addr);
2840         host_attr->debug_area_virt_addr = NULL;
2841     }
2842 }
2843
2844 int ena_com_set_host_attributes(struct ena_com_dev *ena_dev)
2845 {
2846     struct ena_host_attribute *host_attr = &ena_dev->host_attr;
2847     struct ena_com_admin_queue *admin_queue;
2848     struct ena_admin_set_feat_cmd cmd;
2849     struct ena_admin_set_feat_resp resp;
2850
2851     int ret;
2852
2853     /* Host attribute config is called before ena_com_get_dev_attr_feat
2854      * so ena_com can't check if the feature is supported.
2855      */
2856
2857     memset(&cmd, 0x0, sizeof(cmd));
2858     admin_queue = &ena_dev->admin_queue;
2859
2860     cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
2861     cmd.feat_common.feature_id = ENA_ADMIN_HOST_ATTR_CONFIG;
2862
2863     ret = ena_com_mem_addr_set(ena_dev,
2864                    &cmd.u.host_attr.debug_ba,
2865                    host_attr->debug_area_dma_addr);
2866     if (unlikely(ret)) {
2867         netdev_err(ena_dev->net_device, "Memory address set failed\n");
2868         return ret;
2869     }
2870
2871     ret = ena_com_mem_addr_set(ena_dev,
2872                    &cmd.u.host_attr.os_info_ba,
2873                    host_attr->host_info_dma_addr);
2874     if (unlikely(ret)) {
2875         netdev_err(ena_dev->net_device, "Memory address set failed\n");
2876         return ret;
2877     }
2878
2879     cmd.u.host_attr.debug_area_size = host_attr->debug_area_size;
2880
2881     ret = ena_com_execute_admin_command(admin_queue,
2882                         (struct ena_admin_aq_entry *)&cmd,
2883                         sizeof(cmd),
2884                         (struct ena_admin_acq_entry *)&resp,
2885                         sizeof(resp));
2886
2887     if (unlikely(ret))
2888         netdev_err(ena_dev->net_device,
2889                "Failed to set host attributes: %d\n", ret);
2890
2891     return ret;
2892 }
2893
2894 /* Interrupt moderation */
2895 bool ena_com_interrupt_moderation_supported(struct ena_com_dev *ena_dev)
2896 {
2897     return ena_com_check_supported_feature_id(ena_dev,
2898                           ENA_ADMIN_INTERRUPT_MODERATION);
2899 }
2900
2901 static int ena_com_update_nonadaptive_moderation_interval(struct ena_com_dev *ena_dev,
2902                               u32 coalesce_usecs,
2903                               u32 intr_delay_resolution,
2904                               u32 *intr_moder_interval)
2905 {
2906     if (!intr_delay_resolution) {
2907         netdev_err(ena_dev->net_device,
2908                "Illegal interrupt delay granularity value\n");
2909         return -EFAULT;
2910     }
2911
2912     *intr_moder_interval = coalesce_usecs / intr_delay_resolution;
2913
2914     return 0;
2915 }
2916
2917 int ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev,
2918                               u32 tx_coalesce_usecs)
2919 {
2920     return ena_com_update_nonadaptive_moderation_interval(ena_dev,
2921                                   tx_coalesce_usecs,
2922                                   ena_dev->intr_delay_resolution,
2923                                   &ena_dev->intr_moder_tx_interval);
2924 }
2925
2926 int ena_com_update_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev,
2927                               u32 rx_coalesce_usecs)
2928 {
2929     return ena_com_update_nonadaptive_moderation_interval(ena_dev,
2930                                   rx_coalesce_usecs,
2931                                   ena_dev->intr_delay_resolution,
2932                                   &ena_dev->intr_moder_rx_interval);
2933 }
2934
2935 int ena_com_init_interrupt_moderation(struct ena_com_dev *ena_dev)
2936 {
2937     struct ena_admin_get_feat_resp get_resp;
2938     u16 delay_resolution;
2939     int rc;
2940
2941     rc = ena_com_get_feature(ena_dev, &get_resp,
2942                  ENA_ADMIN_INTERRUPT_MODERATION, 0);
2943
2944     if (rc) {
2945         if (rc == -EOPNOTSUPP) {
2946             netdev_dbg(ena_dev->net_device,
2947                    "Feature %d isn't supported\n",
2948                    ENA_ADMIN_INTERRUPT_MODERATION);
2949             rc = 0;
2950         } else {
2951             netdev_err(ena_dev->net_device,
2952                    "Failed to get interrupt moderation admin cmd. rc: %d\n",
2953                    rc);
2954         }
2955
2956         /* no moderation supported, disable adaptive support */
2957         ena_com_disable_adaptive_moderation(ena_dev);
2958         return rc;
2959     }
2960
2961     /* if moderation is supported by device we set adaptive moderation */
2962     delay_resolution = get_resp.u.intr_moderation.intr_delay_resolution;
2963     ena_com_update_intr_delay_resolution(ena_dev, delay_resolution);
2964
2965     /* Disable adaptive moderation by default - can be enabled later */
2966     ena_com_disable_adaptive_moderation(ena_dev);
2967
2968     return 0;
2969 }
2970
2971 unsigned int ena_com_get_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev)
2972 {
2973     return ena_dev->intr_moder_tx_interval;
2974 }
2975
2976 unsigned int ena_com_get_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev)
2977 {
2978     return ena_dev->intr_moder_rx_interval;
2979 }
2980
2981 int ena_com_config_dev_mode(struct ena_com_dev *ena_dev,
2982                 struct ena_admin_feature_llq_desc *llq_features,
2983                 struct ena_llq_configurations *llq_default_cfg)
2984 {
2985     struct ena_com_llq_info *llq_info = &ena_dev->llq_info;
2986     int rc;
2987
2988     if (!llq_features->max_llq_num) {
2989         ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
2990         return 0;
2991     }
2992
2993     rc = ena_com_config_llq_info(ena_dev, llq_features, llq_default_cfg);
2994     if (rc)
2995         return rc;
2996
2997     ena_dev->tx_max_header_size = llq_info->desc_list_entry_size -
2998         (llq_info->descs_num_before_header * sizeof(struct ena_eth_io_tx_desc));
2999
3000     if (unlikely(ena_dev->tx_max_header_size == 0)) {
3001         netdev_err(ena_dev->net_device,
3002                "The size of the LLQ entry is smaller than needed\n");
3003         return -EINVAL;
3004     }
3005
3006     ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_DEV;
3007
3008     return 0;
3009 }