Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
0002 /* Google virtual Ethernet (gve) driver
0003  *
0004  * Copyright (C) 2015-2021 Google, Inc.
0005  */
0006 
0007 #include <linux/etherdevice.h>
0008 #include <linux/pci.h>
0009 #include "gve.h"
0010 #include "gve_adminq.h"
0011 #include "gve_register.h"
0012 
0013 #define GVE_MAX_ADMINQ_RELEASE_CHECK    500
0014 #define GVE_ADMINQ_SLEEP_LEN        20
0015 #define GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK  100
0016 
0017 #define GVE_DEVICE_OPTION_ERROR_FMT "%s option error:\n" \
0018 "Expected: length=%d, feature_mask=%x.\n" \
0019 "Actual: length=%d, feature_mask=%x.\n"
0020 
0021 #define GVE_DEVICE_OPTION_TOO_BIG_FMT "Length of %s option larger than expected. Possible older version of guest driver.\n"
0022 
0023 static
0024 struct gve_device_option *gve_get_next_option(struct gve_device_descriptor *descriptor,
0025                           struct gve_device_option *option)
0026 {
0027     void *option_end, *descriptor_end;
0028 
0029     option_end = (void *)(option + 1) + be16_to_cpu(option->option_length);
0030     descriptor_end = (void *)descriptor + be16_to_cpu(descriptor->total_length);
0031 
0032     return option_end > descriptor_end ? NULL : (struct gve_device_option *)option_end;
0033 }
0034 
0035 static
0036 void gve_parse_device_option(struct gve_priv *priv,
0037                  struct gve_device_descriptor *device_descriptor,
0038                  struct gve_device_option *option,
0039                  struct gve_device_option_gqi_rda **dev_op_gqi_rda,
0040                  struct gve_device_option_gqi_qpl **dev_op_gqi_qpl,
0041                  struct gve_device_option_dqo_rda **dev_op_dqo_rda,
0042                  struct gve_device_option_jumbo_frames **dev_op_jumbo_frames)
0043 {
0044     u32 req_feat_mask = be32_to_cpu(option->required_features_mask);
0045     u16 option_length = be16_to_cpu(option->option_length);
0046     u16 option_id = be16_to_cpu(option->option_id);
0047 
0048     /* If the length or feature mask doesn't match, continue without
0049      * enabling the feature.
0050      */
0051     switch (option_id) {
0052     case GVE_DEV_OPT_ID_GQI_RAW_ADDRESSING:
0053         if (option_length != GVE_DEV_OPT_LEN_GQI_RAW_ADDRESSING ||
0054             req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RAW_ADDRESSING) {
0055             dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
0056                  "Raw Addressing",
0057                  GVE_DEV_OPT_LEN_GQI_RAW_ADDRESSING,
0058                  GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RAW_ADDRESSING,
0059                  option_length, req_feat_mask);
0060             break;
0061         }
0062 
0063         dev_info(&priv->pdev->dev,
0064              "Gqi raw addressing device option enabled.\n");
0065         priv->queue_format = GVE_GQI_RDA_FORMAT;
0066         break;
0067     case GVE_DEV_OPT_ID_GQI_RDA:
0068         if (option_length < sizeof(**dev_op_gqi_rda) ||
0069             req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RDA) {
0070             dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
0071                  "GQI RDA", (int)sizeof(**dev_op_gqi_rda),
0072                  GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RDA,
0073                  option_length, req_feat_mask);
0074             break;
0075         }
0076 
0077         if (option_length > sizeof(**dev_op_gqi_rda)) {
0078             dev_warn(&priv->pdev->dev,
0079                  GVE_DEVICE_OPTION_TOO_BIG_FMT, "GQI RDA");
0080         }
0081         *dev_op_gqi_rda = (void *)(option + 1);
0082         break;
0083     case GVE_DEV_OPT_ID_GQI_QPL:
0084         if (option_length < sizeof(**dev_op_gqi_qpl) ||
0085             req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL) {
0086             dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
0087                  "GQI QPL", (int)sizeof(**dev_op_gqi_qpl),
0088                  GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL,
0089                  option_length, req_feat_mask);
0090             break;
0091         }
0092 
0093         if (option_length > sizeof(**dev_op_gqi_qpl)) {
0094             dev_warn(&priv->pdev->dev,
0095                  GVE_DEVICE_OPTION_TOO_BIG_FMT, "GQI QPL");
0096         }
0097         *dev_op_gqi_qpl = (void *)(option + 1);
0098         break;
0099     case GVE_DEV_OPT_ID_DQO_RDA:
0100         if (option_length < sizeof(**dev_op_dqo_rda) ||
0101             req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA) {
0102             dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
0103                  "DQO RDA", (int)sizeof(**dev_op_dqo_rda),
0104                  GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA,
0105                  option_length, req_feat_mask);
0106             break;
0107         }
0108 
0109         if (option_length > sizeof(**dev_op_dqo_rda)) {
0110             dev_warn(&priv->pdev->dev,
0111                  GVE_DEVICE_OPTION_TOO_BIG_FMT, "DQO RDA");
0112         }
0113         *dev_op_dqo_rda = (void *)(option + 1);
0114         break;
0115     case GVE_DEV_OPT_ID_JUMBO_FRAMES:
0116         if (option_length < sizeof(**dev_op_jumbo_frames) ||
0117             req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES) {
0118             dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
0119                  "Jumbo Frames",
0120                  (int)sizeof(**dev_op_jumbo_frames),
0121                  GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES,
0122                  option_length, req_feat_mask);
0123             break;
0124         }
0125 
0126         if (option_length > sizeof(**dev_op_jumbo_frames)) {
0127             dev_warn(&priv->pdev->dev,
0128                  GVE_DEVICE_OPTION_TOO_BIG_FMT,
0129                  "Jumbo Frames");
0130         }
0131         *dev_op_jumbo_frames = (void *)(option + 1);
0132         break;
0133     default:
0134         /* If we don't recognize the option just continue
0135          * without doing anything.
0136          */
0137         dev_dbg(&priv->pdev->dev, "Unrecognized device option 0x%hx not enabled.\n",
0138             option_id);
0139     }
0140 }
0141 
0142 /* Process all device options for a given describe device call. */
0143 static int
0144 gve_process_device_options(struct gve_priv *priv,
0145                struct gve_device_descriptor *descriptor,
0146                struct gve_device_option_gqi_rda **dev_op_gqi_rda,
0147                struct gve_device_option_gqi_qpl **dev_op_gqi_qpl,
0148                struct gve_device_option_dqo_rda **dev_op_dqo_rda,
0149                struct gve_device_option_jumbo_frames **dev_op_jumbo_frames)
0150 {
0151     const int num_options = be16_to_cpu(descriptor->num_device_options);
0152     struct gve_device_option *dev_opt;
0153     int i;
0154 
0155     /* The options struct directly follows the device descriptor. */
0156     dev_opt = (void *)(descriptor + 1);
0157     for (i = 0; i < num_options; i++) {
0158         struct gve_device_option *next_opt;
0159 
0160         next_opt = gve_get_next_option(descriptor, dev_opt);
0161         if (!next_opt) {
0162             dev_err(&priv->dev->dev,
0163                 "options exceed device_descriptor's total length.\n");
0164             return -EINVAL;
0165         }
0166 
0167         gve_parse_device_option(priv, descriptor, dev_opt,
0168                     dev_op_gqi_rda, dev_op_gqi_qpl,
0169                     dev_op_dqo_rda, dev_op_jumbo_frames);
0170         dev_opt = next_opt;
0171     }
0172 
0173     return 0;
0174 }
0175 
0176 int gve_adminq_alloc(struct device *dev, struct gve_priv *priv)
0177 {
0178     priv->adminq = dma_alloc_coherent(dev, PAGE_SIZE,
0179                       &priv->adminq_bus_addr, GFP_KERNEL);
0180     if (unlikely(!priv->adminq))
0181         return -ENOMEM;
0182 
0183     priv->adminq_mask = (PAGE_SIZE / sizeof(union gve_adminq_command)) - 1;
0184     priv->adminq_prod_cnt = 0;
0185     priv->adminq_cmd_fail = 0;
0186     priv->adminq_timeouts = 0;
0187     priv->adminq_describe_device_cnt = 0;
0188     priv->adminq_cfg_device_resources_cnt = 0;
0189     priv->adminq_register_page_list_cnt = 0;
0190     priv->adminq_unregister_page_list_cnt = 0;
0191     priv->adminq_create_tx_queue_cnt = 0;
0192     priv->adminq_create_rx_queue_cnt = 0;
0193     priv->adminq_destroy_tx_queue_cnt = 0;
0194     priv->adminq_destroy_rx_queue_cnt = 0;
0195     priv->adminq_dcfg_device_resources_cnt = 0;
0196     priv->adminq_set_driver_parameter_cnt = 0;
0197     priv->adminq_report_stats_cnt = 0;
0198     priv->adminq_report_link_speed_cnt = 0;
0199     priv->adminq_get_ptype_map_cnt = 0;
0200 
0201     /* Setup Admin queue with the device */
0202     iowrite32be(priv->adminq_bus_addr / PAGE_SIZE,
0203             &priv->reg_bar0->adminq_pfn);
0204 
0205     gve_set_admin_queue_ok(priv);
0206     return 0;
0207 }
0208 
0209 void gve_adminq_release(struct gve_priv *priv)
0210 {
0211     int i = 0;
0212 
0213     /* Tell the device the adminq is leaving */
0214     iowrite32be(0x0, &priv->reg_bar0->adminq_pfn);
0215     while (ioread32be(&priv->reg_bar0->adminq_pfn)) {
0216         /* If this is reached the device is unrecoverable and still
0217          * holding memory. Continue looping to avoid memory corruption,
0218          * but WARN so it is visible what is going on.
0219          */
0220         if (i == GVE_MAX_ADMINQ_RELEASE_CHECK)
0221             WARN(1, "Unrecoverable platform error!");
0222         i++;
0223         msleep(GVE_ADMINQ_SLEEP_LEN);
0224     }
0225     gve_clear_device_rings_ok(priv);
0226     gve_clear_device_resources_ok(priv);
0227     gve_clear_admin_queue_ok(priv);
0228 }
0229 
0230 void gve_adminq_free(struct device *dev, struct gve_priv *priv)
0231 {
0232     if (!gve_get_admin_queue_ok(priv))
0233         return;
0234     gve_adminq_release(priv);
0235     dma_free_coherent(dev, PAGE_SIZE, priv->adminq, priv->adminq_bus_addr);
0236     gve_clear_admin_queue_ok(priv);
0237 }
0238 
0239 static void gve_adminq_kick_cmd(struct gve_priv *priv, u32 prod_cnt)
0240 {
0241     iowrite32be(prod_cnt, &priv->reg_bar0->adminq_doorbell);
0242 }
0243 
0244 static bool gve_adminq_wait_for_cmd(struct gve_priv *priv, u32 prod_cnt)
0245 {
0246     int i;
0247 
0248     for (i = 0; i < GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK; i++) {
0249         if (ioread32be(&priv->reg_bar0->adminq_event_counter)
0250             == prod_cnt)
0251             return true;
0252         msleep(GVE_ADMINQ_SLEEP_LEN);
0253     }
0254 
0255     return false;
0256 }
0257 
0258 static int gve_adminq_parse_err(struct gve_priv *priv, u32 status)
0259 {
0260     if (status != GVE_ADMINQ_COMMAND_PASSED &&
0261         status != GVE_ADMINQ_COMMAND_UNSET) {
0262         dev_err(&priv->pdev->dev, "AQ command failed with status %d\n", status);
0263         priv->adminq_cmd_fail++;
0264     }
0265     switch (status) {
0266     case GVE_ADMINQ_COMMAND_PASSED:
0267         return 0;
0268     case GVE_ADMINQ_COMMAND_UNSET:
0269         dev_err(&priv->pdev->dev, "parse_aq_err: err and status both unset, this should not be possible.\n");
0270         return -EINVAL;
0271     case GVE_ADMINQ_COMMAND_ERROR_ABORTED:
0272     case GVE_ADMINQ_COMMAND_ERROR_CANCELLED:
0273     case GVE_ADMINQ_COMMAND_ERROR_DATALOSS:
0274     case GVE_ADMINQ_COMMAND_ERROR_FAILED_PRECONDITION:
0275     case GVE_ADMINQ_COMMAND_ERROR_UNAVAILABLE:
0276         return -EAGAIN;
0277     case GVE_ADMINQ_COMMAND_ERROR_ALREADY_EXISTS:
0278     case GVE_ADMINQ_COMMAND_ERROR_INTERNAL_ERROR:
0279     case GVE_ADMINQ_COMMAND_ERROR_INVALID_ARGUMENT:
0280     case GVE_ADMINQ_COMMAND_ERROR_NOT_FOUND:
0281     case GVE_ADMINQ_COMMAND_ERROR_OUT_OF_RANGE:
0282     case GVE_ADMINQ_COMMAND_ERROR_UNKNOWN_ERROR:
0283         return -EINVAL;
0284     case GVE_ADMINQ_COMMAND_ERROR_DEADLINE_EXCEEDED:
0285         return -ETIME;
0286     case GVE_ADMINQ_COMMAND_ERROR_PERMISSION_DENIED:
0287     case GVE_ADMINQ_COMMAND_ERROR_UNAUTHENTICATED:
0288         return -EACCES;
0289     case GVE_ADMINQ_COMMAND_ERROR_RESOURCE_EXHAUSTED:
0290         return -ENOMEM;
0291     case GVE_ADMINQ_COMMAND_ERROR_UNIMPLEMENTED:
0292         return -ENOTSUPP;
0293     default:
0294         dev_err(&priv->pdev->dev, "parse_aq_err: unknown status code %d\n", status);
0295         return -EINVAL;
0296     }
0297 }
0298 
0299 /* Flushes all AQ commands currently queued and waits for them to complete.
0300  * If there are failures, it will return the first error.
0301  */
0302 static int gve_adminq_kick_and_wait(struct gve_priv *priv)
0303 {
0304     int tail, head;
0305     int i;
0306 
0307     tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
0308     head = priv->adminq_prod_cnt;
0309 
0310     gve_adminq_kick_cmd(priv, head);
0311     if (!gve_adminq_wait_for_cmd(priv, head)) {
0312         dev_err(&priv->pdev->dev, "AQ commands timed out, need to reset AQ\n");
0313         priv->adminq_timeouts++;
0314         return -ENOTRECOVERABLE;
0315     }
0316 
0317     for (i = tail; i < head; i++) {
0318         union gve_adminq_command *cmd;
0319         u32 status, err;
0320 
0321         cmd = &priv->adminq[i & priv->adminq_mask];
0322         status = be32_to_cpu(READ_ONCE(cmd->status));
0323         err = gve_adminq_parse_err(priv, status);
0324         if (err)
0325             // Return the first error if we failed.
0326             return err;
0327     }
0328 
0329     return 0;
0330 }
0331 
0332 /* This function is not threadsafe - the caller is responsible for any
0333  * necessary locks.
0334  */
0335 static int gve_adminq_issue_cmd(struct gve_priv *priv,
0336                 union gve_adminq_command *cmd_orig)
0337 {
0338     union gve_adminq_command *cmd;
0339     u32 opcode;
0340     u32 tail;
0341 
0342     tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
0343 
0344     // Check if next command will overflow the buffer.
0345     if (((priv->adminq_prod_cnt + 1) & priv->adminq_mask) ==
0346         (tail & priv->adminq_mask)) {
0347         int err;
0348 
0349         // Flush existing commands to make room.
0350         err = gve_adminq_kick_and_wait(priv);
0351         if (err)
0352             return err;
0353 
0354         // Retry.
0355         tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
0356         if (((priv->adminq_prod_cnt + 1) & priv->adminq_mask) ==
0357             (tail & priv->adminq_mask)) {
0358             // This should never happen. We just flushed the
0359             // command queue so there should be enough space.
0360             return -ENOMEM;
0361         }
0362     }
0363 
0364     cmd = &priv->adminq[priv->adminq_prod_cnt & priv->adminq_mask];
0365     priv->adminq_prod_cnt++;
0366 
0367     memcpy(cmd, cmd_orig, sizeof(*cmd_orig));
0368     opcode = be32_to_cpu(READ_ONCE(cmd->opcode));
0369 
0370     switch (opcode) {
0371     case GVE_ADMINQ_DESCRIBE_DEVICE:
0372         priv->adminq_describe_device_cnt++;
0373         break;
0374     case GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES:
0375         priv->adminq_cfg_device_resources_cnt++;
0376         break;
0377     case GVE_ADMINQ_REGISTER_PAGE_LIST:
0378         priv->adminq_register_page_list_cnt++;
0379         break;
0380     case GVE_ADMINQ_UNREGISTER_PAGE_LIST:
0381         priv->adminq_unregister_page_list_cnt++;
0382         break;
0383     case GVE_ADMINQ_CREATE_TX_QUEUE:
0384         priv->adminq_create_tx_queue_cnt++;
0385         break;
0386     case GVE_ADMINQ_CREATE_RX_QUEUE:
0387         priv->adminq_create_rx_queue_cnt++;
0388         break;
0389     case GVE_ADMINQ_DESTROY_TX_QUEUE:
0390         priv->adminq_destroy_tx_queue_cnt++;
0391         break;
0392     case GVE_ADMINQ_DESTROY_RX_QUEUE:
0393         priv->adminq_destroy_rx_queue_cnt++;
0394         break;
0395     case GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES:
0396         priv->adminq_dcfg_device_resources_cnt++;
0397         break;
0398     case GVE_ADMINQ_SET_DRIVER_PARAMETER:
0399         priv->adminq_set_driver_parameter_cnt++;
0400         break;
0401     case GVE_ADMINQ_REPORT_STATS:
0402         priv->adminq_report_stats_cnt++;
0403         break;
0404     case GVE_ADMINQ_REPORT_LINK_SPEED:
0405         priv->adminq_report_link_speed_cnt++;
0406         break;
0407     case GVE_ADMINQ_GET_PTYPE_MAP:
0408         priv->adminq_get_ptype_map_cnt++;
0409         break;
0410     default:
0411         dev_err(&priv->pdev->dev, "unknown AQ command opcode %d\n", opcode);
0412     }
0413 
0414     return 0;
0415 }
0416 
0417 /* This function is not threadsafe - the caller is responsible for any
0418  * necessary locks.
0419  * The caller is also responsible for making sure there are no commands
0420  * waiting to be executed.
0421  */
0422 static int gve_adminq_execute_cmd(struct gve_priv *priv,
0423                   union gve_adminq_command *cmd_orig)
0424 {
0425     u32 tail, head;
0426     int err;
0427 
0428     tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
0429     head = priv->adminq_prod_cnt;
0430     if (tail != head)
0431         // This is not a valid path
0432         return -EINVAL;
0433 
0434     err = gve_adminq_issue_cmd(priv, cmd_orig);
0435     if (err)
0436         return err;
0437 
0438     return gve_adminq_kick_and_wait(priv);
0439 }
0440 
0441 /* The device specifies that the management vector can either be the first irq
0442  * or the last irq. ntfy_blk_msix_base_idx indicates the first irq assigned to
0443  * the ntfy blks. It if is 0 then the management vector is last, if it is 1 then
0444  * the management vector is first.
0445  *
0446  * gve arranges the msix vectors so that the management vector is last.
0447  */
0448 #define GVE_NTFY_BLK_BASE_MSIX_IDX  0
0449 int gve_adminq_configure_device_resources(struct gve_priv *priv,
0450                       dma_addr_t counter_array_bus_addr,
0451                       u32 num_counters,
0452                       dma_addr_t db_array_bus_addr,
0453                       u32 num_ntfy_blks)
0454 {
0455     union gve_adminq_command cmd;
0456 
0457     memset(&cmd, 0, sizeof(cmd));
0458     cmd.opcode = cpu_to_be32(GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES);
0459     cmd.configure_device_resources =
0460         (struct gve_adminq_configure_device_resources) {
0461         .counter_array = cpu_to_be64(counter_array_bus_addr),
0462         .num_counters = cpu_to_be32(num_counters),
0463         .irq_db_addr = cpu_to_be64(db_array_bus_addr),
0464         .num_irq_dbs = cpu_to_be32(num_ntfy_blks),
0465         .irq_db_stride = cpu_to_be32(sizeof(*priv->irq_db_indices)),
0466         .ntfy_blk_msix_base_idx =
0467                     cpu_to_be32(GVE_NTFY_BLK_BASE_MSIX_IDX),
0468         .queue_format = priv->queue_format,
0469     };
0470 
0471     return gve_adminq_execute_cmd(priv, &cmd);
0472 }
0473 
0474 int gve_adminq_deconfigure_device_resources(struct gve_priv *priv)
0475 {
0476     union gve_adminq_command cmd;
0477 
0478     memset(&cmd, 0, sizeof(cmd));
0479     cmd.opcode = cpu_to_be32(GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES);
0480 
0481     return gve_adminq_execute_cmd(priv, &cmd);
0482 }
0483 
0484 static int gve_adminq_create_tx_queue(struct gve_priv *priv, u32 queue_index)
0485 {
0486     struct gve_tx_ring *tx = &priv->tx[queue_index];
0487     union gve_adminq_command cmd;
0488 
0489     memset(&cmd, 0, sizeof(cmd));
0490     cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_TX_QUEUE);
0491     cmd.create_tx_queue = (struct gve_adminq_create_tx_queue) {
0492         .queue_id = cpu_to_be32(queue_index),
0493         .queue_resources_addr =
0494             cpu_to_be64(tx->q_resources_bus),
0495         .tx_ring_addr = cpu_to_be64(tx->bus),
0496         .ntfy_id = cpu_to_be32(tx->ntfy_id),
0497     };
0498 
0499     if (gve_is_gqi(priv)) {
0500         u32 qpl_id = priv->queue_format == GVE_GQI_RDA_FORMAT ?
0501             GVE_RAW_ADDRESSING_QPL_ID : tx->tx_fifo.qpl->id;
0502 
0503         cmd.create_tx_queue.queue_page_list_id = cpu_to_be32(qpl_id);
0504     } else {
0505         cmd.create_tx_queue.tx_ring_size =
0506             cpu_to_be16(priv->tx_desc_cnt);
0507         cmd.create_tx_queue.tx_comp_ring_addr =
0508             cpu_to_be64(tx->complq_bus_dqo);
0509         cmd.create_tx_queue.tx_comp_ring_size =
0510             cpu_to_be16(priv->options_dqo_rda.tx_comp_ring_entries);
0511     }
0512 
0513     return gve_adminq_issue_cmd(priv, &cmd);
0514 }
0515 
0516 int gve_adminq_create_tx_queues(struct gve_priv *priv, u32 num_queues)
0517 {
0518     int err;
0519     int i;
0520 
0521     for (i = 0; i < num_queues; i++) {
0522         err = gve_adminq_create_tx_queue(priv, i);
0523         if (err)
0524             return err;
0525     }
0526 
0527     return gve_adminq_kick_and_wait(priv);
0528 }
0529 
0530 static int gve_adminq_create_rx_queue(struct gve_priv *priv, u32 queue_index)
0531 {
0532     struct gve_rx_ring *rx = &priv->rx[queue_index];
0533     union gve_adminq_command cmd;
0534 
0535     memset(&cmd, 0, sizeof(cmd));
0536     cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_RX_QUEUE);
0537     cmd.create_rx_queue = (struct gve_adminq_create_rx_queue) {
0538         .queue_id = cpu_to_be32(queue_index),
0539         .ntfy_id = cpu_to_be32(rx->ntfy_id),
0540         .queue_resources_addr = cpu_to_be64(rx->q_resources_bus),
0541     };
0542 
0543     if (gve_is_gqi(priv)) {
0544         u32 qpl_id = priv->queue_format == GVE_GQI_RDA_FORMAT ?
0545             GVE_RAW_ADDRESSING_QPL_ID : rx->data.qpl->id;
0546 
0547         cmd.create_rx_queue.rx_desc_ring_addr =
0548             cpu_to_be64(rx->desc.bus),
0549         cmd.create_rx_queue.rx_data_ring_addr =
0550             cpu_to_be64(rx->data.data_bus),
0551         cmd.create_rx_queue.index = cpu_to_be32(queue_index);
0552         cmd.create_rx_queue.queue_page_list_id = cpu_to_be32(qpl_id);
0553         cmd.create_rx_queue.packet_buffer_size = cpu_to_be16(rx->packet_buffer_size);
0554     } else {
0555         cmd.create_rx_queue.rx_ring_size =
0556             cpu_to_be16(priv->rx_desc_cnt);
0557         cmd.create_rx_queue.rx_desc_ring_addr =
0558             cpu_to_be64(rx->dqo.complq.bus);
0559         cmd.create_rx_queue.rx_data_ring_addr =
0560             cpu_to_be64(rx->dqo.bufq.bus);
0561         cmd.create_rx_queue.packet_buffer_size =
0562             cpu_to_be16(priv->data_buffer_size_dqo);
0563         cmd.create_rx_queue.rx_buff_ring_size =
0564             cpu_to_be16(priv->options_dqo_rda.rx_buff_ring_entries);
0565         cmd.create_rx_queue.enable_rsc =
0566             !!(priv->dev->features & NETIF_F_LRO);
0567     }
0568 
0569     return gve_adminq_issue_cmd(priv, &cmd);
0570 }
0571 
0572 int gve_adminq_create_rx_queues(struct gve_priv *priv, u32 num_queues)
0573 {
0574     int err;
0575     int i;
0576 
0577     for (i = 0; i < num_queues; i++) {
0578         err = gve_adminq_create_rx_queue(priv, i);
0579         if (err)
0580             return err;
0581     }
0582 
0583     return gve_adminq_kick_and_wait(priv);
0584 }
0585 
0586 static int gve_adminq_destroy_tx_queue(struct gve_priv *priv, u32 queue_index)
0587 {
0588     union gve_adminq_command cmd;
0589     int err;
0590 
0591     memset(&cmd, 0, sizeof(cmd));
0592     cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_TX_QUEUE);
0593     cmd.destroy_tx_queue = (struct gve_adminq_destroy_tx_queue) {
0594         .queue_id = cpu_to_be32(queue_index),
0595     };
0596 
0597     err = gve_adminq_issue_cmd(priv, &cmd);
0598     if (err)
0599         return err;
0600 
0601     return 0;
0602 }
0603 
0604 int gve_adminq_destroy_tx_queues(struct gve_priv *priv, u32 num_queues)
0605 {
0606     int err;
0607     int i;
0608 
0609     for (i = 0; i < num_queues; i++) {
0610         err = gve_adminq_destroy_tx_queue(priv, i);
0611         if (err)
0612             return err;
0613     }
0614 
0615     return gve_adminq_kick_and_wait(priv);
0616 }
0617 
0618 static int gve_adminq_destroy_rx_queue(struct gve_priv *priv, u32 queue_index)
0619 {
0620     union gve_adminq_command cmd;
0621     int err;
0622 
0623     memset(&cmd, 0, sizeof(cmd));
0624     cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_RX_QUEUE);
0625     cmd.destroy_rx_queue = (struct gve_adminq_destroy_rx_queue) {
0626         .queue_id = cpu_to_be32(queue_index),
0627     };
0628 
0629     err = gve_adminq_issue_cmd(priv, &cmd);
0630     if (err)
0631         return err;
0632 
0633     return 0;
0634 }
0635 
0636 int gve_adminq_destroy_rx_queues(struct gve_priv *priv, u32 num_queues)
0637 {
0638     int err;
0639     int i;
0640 
0641     for (i = 0; i < num_queues; i++) {
0642         err = gve_adminq_destroy_rx_queue(priv, i);
0643         if (err)
0644             return err;
0645     }
0646 
0647     return gve_adminq_kick_and_wait(priv);
0648 }
0649 
0650 static int gve_set_desc_cnt(struct gve_priv *priv,
0651                 struct gve_device_descriptor *descriptor)
0652 {
0653     priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries);
0654     if (priv->tx_desc_cnt * sizeof(priv->tx->desc[0]) < PAGE_SIZE) {
0655         dev_err(&priv->pdev->dev, "Tx desc count %d too low\n",
0656             priv->tx_desc_cnt);
0657         return -EINVAL;
0658     }
0659     priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries);
0660     if (priv->rx_desc_cnt * sizeof(priv->rx->desc.desc_ring[0])
0661         < PAGE_SIZE) {
0662         dev_err(&priv->pdev->dev, "Rx desc count %d too low\n",
0663             priv->rx_desc_cnt);
0664         return -EINVAL;
0665     }
0666     return 0;
0667 }
0668 
0669 static int
0670 gve_set_desc_cnt_dqo(struct gve_priv *priv,
0671              const struct gve_device_descriptor *descriptor,
0672              const struct gve_device_option_dqo_rda *dev_op_dqo_rda)
0673 {
0674     priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries);
0675     priv->options_dqo_rda.tx_comp_ring_entries =
0676         be16_to_cpu(dev_op_dqo_rda->tx_comp_ring_entries);
0677     priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries);
0678     priv->options_dqo_rda.rx_buff_ring_entries =
0679         be16_to_cpu(dev_op_dqo_rda->rx_buff_ring_entries);
0680 
0681     return 0;
0682 }
0683 
0684 static void gve_enable_supported_features(struct gve_priv *priv,
0685                       u32 supported_features_mask,
0686                       const struct gve_device_option_jumbo_frames
0687                           *dev_op_jumbo_frames)
0688 {
0689     /* Before control reaches this point, the page-size-capped max MTU from
0690      * the gve_device_descriptor field has already been stored in
0691      * priv->dev->max_mtu. We overwrite it with the true max MTU below.
0692      */
0693     if (dev_op_jumbo_frames &&
0694         (supported_features_mask & GVE_SUP_JUMBO_FRAMES_MASK)) {
0695         dev_info(&priv->pdev->dev,
0696              "JUMBO FRAMES device option enabled.\n");
0697         priv->dev->max_mtu = be16_to_cpu(dev_op_jumbo_frames->max_mtu);
0698     }
0699 }
0700 
0701 int gve_adminq_describe_device(struct gve_priv *priv)
0702 {
0703     struct gve_device_option_jumbo_frames *dev_op_jumbo_frames = NULL;
0704     struct gve_device_option_gqi_rda *dev_op_gqi_rda = NULL;
0705     struct gve_device_option_gqi_qpl *dev_op_gqi_qpl = NULL;
0706     struct gve_device_option_dqo_rda *dev_op_dqo_rda = NULL;
0707     struct gve_device_descriptor *descriptor;
0708     u32 supported_features_mask = 0;
0709     union gve_adminq_command cmd;
0710     dma_addr_t descriptor_bus;
0711     int err = 0;
0712     u8 *mac;
0713     u16 mtu;
0714 
0715     memset(&cmd, 0, sizeof(cmd));
0716     descriptor = dma_alloc_coherent(&priv->pdev->dev, PAGE_SIZE,
0717                     &descriptor_bus, GFP_KERNEL);
0718     if (!descriptor)
0719         return -ENOMEM;
0720     cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESCRIBE_DEVICE);
0721     cmd.describe_device.device_descriptor_addr =
0722                         cpu_to_be64(descriptor_bus);
0723     cmd.describe_device.device_descriptor_version =
0724             cpu_to_be32(GVE_ADMINQ_DEVICE_DESCRIPTOR_VERSION);
0725     cmd.describe_device.available_length = cpu_to_be32(PAGE_SIZE);
0726 
0727     err = gve_adminq_execute_cmd(priv, &cmd);
0728     if (err)
0729         goto free_device_descriptor;
0730 
0731     err = gve_process_device_options(priv, descriptor, &dev_op_gqi_rda,
0732                      &dev_op_gqi_qpl, &dev_op_dqo_rda,
0733                      &dev_op_jumbo_frames);
0734     if (err)
0735         goto free_device_descriptor;
0736 
0737     /* If the GQI_RAW_ADDRESSING option is not enabled and the queue format
0738      * is not set to GqiRda, choose the queue format in a priority order:
0739      * DqoRda, GqiRda, GqiQpl. Use GqiQpl as default.
0740      */
0741     if (dev_op_dqo_rda) {
0742         priv->queue_format = GVE_DQO_RDA_FORMAT;
0743         dev_info(&priv->pdev->dev,
0744              "Driver is running with DQO RDA queue format.\n");
0745         supported_features_mask =
0746             be32_to_cpu(dev_op_dqo_rda->supported_features_mask);
0747     } else if (dev_op_gqi_rda) {
0748         priv->queue_format = GVE_GQI_RDA_FORMAT;
0749         dev_info(&priv->pdev->dev,
0750              "Driver is running with GQI RDA queue format.\n");
0751         supported_features_mask =
0752             be32_to_cpu(dev_op_gqi_rda->supported_features_mask);
0753     } else if (priv->queue_format == GVE_GQI_RDA_FORMAT) {
0754         dev_info(&priv->pdev->dev,
0755              "Driver is running with GQI RDA queue format.\n");
0756     } else {
0757         priv->queue_format = GVE_GQI_QPL_FORMAT;
0758         if (dev_op_gqi_qpl)
0759             supported_features_mask =
0760                 be32_to_cpu(dev_op_gqi_qpl->supported_features_mask);
0761         dev_info(&priv->pdev->dev,
0762              "Driver is running with GQI QPL queue format.\n");
0763     }
0764     if (gve_is_gqi(priv)) {
0765         err = gve_set_desc_cnt(priv, descriptor);
0766     } else {
0767         /* DQO supports LRO. */
0768         priv->dev->hw_features |= NETIF_F_LRO;
0769         err = gve_set_desc_cnt_dqo(priv, descriptor, dev_op_dqo_rda);
0770     }
0771     if (err)
0772         goto free_device_descriptor;
0773 
0774     priv->max_registered_pages =
0775                 be64_to_cpu(descriptor->max_registered_pages);
0776     mtu = be16_to_cpu(descriptor->mtu);
0777     if (mtu < ETH_MIN_MTU) {
0778         dev_err(&priv->pdev->dev, "MTU %d below minimum MTU\n", mtu);
0779         err = -EINVAL;
0780         goto free_device_descriptor;
0781     }
0782     priv->dev->max_mtu = mtu;
0783     priv->num_event_counters = be16_to_cpu(descriptor->counters);
0784     eth_hw_addr_set(priv->dev, descriptor->mac);
0785     mac = descriptor->mac;
0786     dev_info(&priv->pdev->dev, "MAC addr: %pM\n", mac);
0787     priv->tx_pages_per_qpl = be16_to_cpu(descriptor->tx_pages_per_qpl);
0788     priv->rx_data_slot_cnt = be16_to_cpu(descriptor->rx_pages_per_qpl);
0789 
0790     if (gve_is_gqi(priv) && priv->rx_data_slot_cnt < priv->rx_desc_cnt) {
0791         dev_err(&priv->pdev->dev, "rx_data_slot_cnt cannot be smaller than rx_desc_cnt, setting rx_desc_cnt down to %d.\n",
0792             priv->rx_data_slot_cnt);
0793         priv->rx_desc_cnt = priv->rx_data_slot_cnt;
0794     }
0795     priv->default_num_queues = be16_to_cpu(descriptor->default_num_queues);
0796 
0797     gve_enable_supported_features(priv, supported_features_mask,
0798                       dev_op_jumbo_frames);
0799 
0800 free_device_descriptor:
0801     dma_free_coherent(&priv->pdev->dev, PAGE_SIZE, descriptor,
0802               descriptor_bus);
0803     return err;
0804 }
0805 
0806 int gve_adminq_register_page_list(struct gve_priv *priv,
0807                   struct gve_queue_page_list *qpl)
0808 {
0809     struct device *hdev = &priv->pdev->dev;
0810     u32 num_entries = qpl->num_entries;
0811     u32 size = num_entries * sizeof(qpl->page_buses[0]);
0812     union gve_adminq_command cmd;
0813     dma_addr_t page_list_bus;
0814     __be64 *page_list;
0815     int err;
0816     int i;
0817 
0818     memset(&cmd, 0, sizeof(cmd));
0819     page_list = dma_alloc_coherent(hdev, size, &page_list_bus, GFP_KERNEL);
0820     if (!page_list)
0821         return -ENOMEM;
0822 
0823     for (i = 0; i < num_entries; i++)
0824         page_list[i] = cpu_to_be64(qpl->page_buses[i]);
0825 
0826     cmd.opcode = cpu_to_be32(GVE_ADMINQ_REGISTER_PAGE_LIST);
0827     cmd.reg_page_list = (struct gve_adminq_register_page_list) {
0828         .page_list_id = cpu_to_be32(qpl->id),
0829         .num_pages = cpu_to_be32(num_entries),
0830         .page_address_list_addr = cpu_to_be64(page_list_bus),
0831     };
0832 
0833     err = gve_adminq_execute_cmd(priv, &cmd);
0834     dma_free_coherent(hdev, size, page_list, page_list_bus);
0835     return err;
0836 }
0837 
0838 int gve_adminq_unregister_page_list(struct gve_priv *priv, u32 page_list_id)
0839 {
0840     union gve_adminq_command cmd;
0841 
0842     memset(&cmd, 0, sizeof(cmd));
0843     cmd.opcode = cpu_to_be32(GVE_ADMINQ_UNREGISTER_PAGE_LIST);
0844     cmd.unreg_page_list = (struct gve_adminq_unregister_page_list) {
0845         .page_list_id = cpu_to_be32(page_list_id),
0846     };
0847 
0848     return gve_adminq_execute_cmd(priv, &cmd);
0849 }
0850 
0851 int gve_adminq_set_mtu(struct gve_priv *priv, u64 mtu)
0852 {
0853     union gve_adminq_command cmd;
0854 
0855     memset(&cmd, 0, sizeof(cmd));
0856     cmd.opcode = cpu_to_be32(GVE_ADMINQ_SET_DRIVER_PARAMETER);
0857     cmd.set_driver_param = (struct gve_adminq_set_driver_parameter) {
0858         .parameter_type = cpu_to_be32(GVE_SET_PARAM_MTU),
0859         .parameter_value = cpu_to_be64(mtu),
0860     };
0861 
0862     return gve_adminq_execute_cmd(priv, &cmd);
0863 }
0864 
0865 int gve_adminq_report_stats(struct gve_priv *priv, u64 stats_report_len,
0866                 dma_addr_t stats_report_addr, u64 interval)
0867 {
0868     union gve_adminq_command cmd;
0869 
0870     memset(&cmd, 0, sizeof(cmd));
0871     cmd.opcode = cpu_to_be32(GVE_ADMINQ_REPORT_STATS);
0872     cmd.report_stats = (struct gve_adminq_report_stats) {
0873         .stats_report_len = cpu_to_be64(stats_report_len),
0874         .stats_report_addr = cpu_to_be64(stats_report_addr),
0875         .interval = cpu_to_be64(interval),
0876     };
0877 
0878     return gve_adminq_execute_cmd(priv, &cmd);
0879 }
0880 
0881 int gve_adminq_report_link_speed(struct gve_priv *priv)
0882 {
0883     union gve_adminq_command gvnic_cmd;
0884     dma_addr_t link_speed_region_bus;
0885     __be64 *link_speed_region;
0886     int err;
0887 
0888     link_speed_region =
0889         dma_alloc_coherent(&priv->pdev->dev, sizeof(*link_speed_region),
0890                    &link_speed_region_bus, GFP_KERNEL);
0891 
0892     if (!link_speed_region)
0893         return -ENOMEM;
0894 
0895     memset(&gvnic_cmd, 0, sizeof(gvnic_cmd));
0896     gvnic_cmd.opcode = cpu_to_be32(GVE_ADMINQ_REPORT_LINK_SPEED);
0897     gvnic_cmd.report_link_speed.link_speed_address =
0898         cpu_to_be64(link_speed_region_bus);
0899 
0900     err = gve_adminq_execute_cmd(priv, &gvnic_cmd);
0901 
0902     priv->link_speed = be64_to_cpu(*link_speed_region);
0903     dma_free_coherent(&priv->pdev->dev, sizeof(*link_speed_region), link_speed_region,
0904               link_speed_region_bus);
0905     return err;
0906 }
0907 
0908 int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv,
0909                  struct gve_ptype_lut *ptype_lut)
0910 {
0911     struct gve_ptype_map *ptype_map;
0912     union gve_adminq_command cmd;
0913     dma_addr_t ptype_map_bus;
0914     int err = 0;
0915     int i;
0916 
0917     memset(&cmd, 0, sizeof(cmd));
0918     ptype_map = dma_alloc_coherent(&priv->pdev->dev, sizeof(*ptype_map),
0919                        &ptype_map_bus, GFP_KERNEL);
0920     if (!ptype_map)
0921         return -ENOMEM;
0922 
0923     cmd.opcode = cpu_to_be32(GVE_ADMINQ_GET_PTYPE_MAP);
0924     cmd.get_ptype_map = (struct gve_adminq_get_ptype_map) {
0925         .ptype_map_len = cpu_to_be64(sizeof(*ptype_map)),
0926         .ptype_map_addr = cpu_to_be64(ptype_map_bus),
0927     };
0928 
0929     err = gve_adminq_execute_cmd(priv, &cmd);
0930     if (err)
0931         goto err;
0932 
0933     /* Populate ptype_lut. */
0934     for (i = 0; i < GVE_NUM_PTYPES; i++) {
0935         ptype_lut->ptypes[i].l3_type =
0936             ptype_map->ptypes[i].l3_type;
0937         ptype_lut->ptypes[i].l4_type =
0938             ptype_map->ptypes[i].l4_type;
0939     }
0940 err:
0941     dma_free_coherent(&priv->pdev->dev, sizeof(*ptype_map), ptype_map,
0942               ptype_map_bus);
0943     return err;
0944 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!