Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
0002 /*
0003  * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved
0004  */
0005 
0006 #include "cmd.h"
0007 
0008 static int mlx5vf_cmd_get_vhca_id(struct mlx5_core_dev *mdev, u16 function_id,
0009                   u16 *vhca_id);
0010 
0011 int mlx5vf_cmd_suspend_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod)
0012 {
0013     u32 out[MLX5_ST_SZ_DW(suspend_vhca_out)] = {};
0014     u32 in[MLX5_ST_SZ_DW(suspend_vhca_in)] = {};
0015 
0016     lockdep_assert_held(&mvdev->state_mutex);
0017     if (mvdev->mdev_detach)
0018         return -ENOTCONN;
0019 
0020     MLX5_SET(suspend_vhca_in, in, opcode, MLX5_CMD_OP_SUSPEND_VHCA);
0021     MLX5_SET(suspend_vhca_in, in, vhca_id, mvdev->vhca_id);
0022     MLX5_SET(suspend_vhca_in, in, op_mod, op_mod);
0023 
0024     return mlx5_cmd_exec_inout(mvdev->mdev, suspend_vhca, in, out);
0025 }
0026 
0027 int mlx5vf_cmd_resume_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod)
0028 {
0029     u32 out[MLX5_ST_SZ_DW(resume_vhca_out)] = {};
0030     u32 in[MLX5_ST_SZ_DW(resume_vhca_in)] = {};
0031 
0032     lockdep_assert_held(&mvdev->state_mutex);
0033     if (mvdev->mdev_detach)
0034         return -ENOTCONN;
0035 
0036     MLX5_SET(resume_vhca_in, in, opcode, MLX5_CMD_OP_RESUME_VHCA);
0037     MLX5_SET(resume_vhca_in, in, vhca_id, mvdev->vhca_id);
0038     MLX5_SET(resume_vhca_in, in, op_mod, op_mod);
0039 
0040     return mlx5_cmd_exec_inout(mvdev->mdev, resume_vhca, in, out);
0041 }
0042 
0043 int mlx5vf_cmd_query_vhca_migration_state(struct mlx5vf_pci_core_device *mvdev,
0044                       size_t *state_size)
0045 {
0046     u32 out[MLX5_ST_SZ_DW(query_vhca_migration_state_out)] = {};
0047     u32 in[MLX5_ST_SZ_DW(query_vhca_migration_state_in)] = {};
0048     int ret;
0049 
0050     lockdep_assert_held(&mvdev->state_mutex);
0051     if (mvdev->mdev_detach)
0052         return -ENOTCONN;
0053 
0054     MLX5_SET(query_vhca_migration_state_in, in, opcode,
0055          MLX5_CMD_OP_QUERY_VHCA_MIGRATION_STATE);
0056     MLX5_SET(query_vhca_migration_state_in, in, vhca_id, mvdev->vhca_id);
0057     MLX5_SET(query_vhca_migration_state_in, in, op_mod, 0);
0058 
0059     ret = mlx5_cmd_exec_inout(mvdev->mdev, query_vhca_migration_state, in,
0060                   out);
0061     if (ret)
0062         return ret;
0063 
0064     *state_size = MLX5_GET(query_vhca_migration_state_out, out,
0065                    required_umem_size);
0066     return 0;
0067 }
0068 
0069 static int mlx5fv_vf_event(struct notifier_block *nb,
0070                unsigned long event, void *data)
0071 {
0072     struct mlx5vf_pci_core_device *mvdev =
0073         container_of(nb, struct mlx5vf_pci_core_device, nb);
0074 
0075     mutex_lock(&mvdev->state_mutex);
0076     switch (event) {
0077     case MLX5_PF_NOTIFY_ENABLE_VF:
0078         mvdev->mdev_detach = false;
0079         break;
0080     case MLX5_PF_NOTIFY_DISABLE_VF:
0081         mlx5vf_disable_fds(mvdev);
0082         mvdev->mdev_detach = true;
0083         break;
0084     default:
0085         break;
0086     }
0087     mlx5vf_state_mutex_unlock(mvdev);
0088     return 0;
0089 }
0090 
0091 void mlx5vf_cmd_close_migratable(struct mlx5vf_pci_core_device *mvdev)
0092 {
0093     if (!mvdev->migrate_cap)
0094         return;
0095 
0096     mutex_lock(&mvdev->state_mutex);
0097     mlx5vf_disable_fds(mvdev);
0098     mlx5vf_state_mutex_unlock(mvdev);
0099 }
0100 
0101 void mlx5vf_cmd_remove_migratable(struct mlx5vf_pci_core_device *mvdev)
0102 {
0103     if (!mvdev->migrate_cap)
0104         return;
0105 
0106     mlx5_sriov_blocking_notifier_unregister(mvdev->mdev, mvdev->vf_id,
0107                         &mvdev->nb);
0108     destroy_workqueue(mvdev->cb_wq);
0109 }
0110 
0111 void mlx5vf_cmd_set_migratable(struct mlx5vf_pci_core_device *mvdev,
0112                    const struct vfio_migration_ops *mig_ops)
0113 {
0114     struct pci_dev *pdev = mvdev->core_device.pdev;
0115     int ret;
0116 
0117     if (!pdev->is_virtfn)
0118         return;
0119 
0120     mvdev->mdev = mlx5_vf_get_core_dev(pdev);
0121     if (!mvdev->mdev)
0122         return;
0123 
0124     if (!MLX5_CAP_GEN(mvdev->mdev, migration))
0125         goto end;
0126 
0127     mvdev->vf_id = pci_iov_vf_id(pdev);
0128     if (mvdev->vf_id < 0)
0129         goto end;
0130 
0131     if (mlx5vf_cmd_get_vhca_id(mvdev->mdev, mvdev->vf_id + 1,
0132                    &mvdev->vhca_id))
0133         goto end;
0134 
0135     mvdev->cb_wq = alloc_ordered_workqueue("mlx5vf_wq", 0);
0136     if (!mvdev->cb_wq)
0137         goto end;
0138 
0139     mutex_init(&mvdev->state_mutex);
0140     spin_lock_init(&mvdev->reset_lock);
0141     mvdev->nb.notifier_call = mlx5fv_vf_event;
0142     ret = mlx5_sriov_blocking_notifier_register(mvdev->mdev, mvdev->vf_id,
0143                             &mvdev->nb);
0144     if (ret) {
0145         destroy_workqueue(mvdev->cb_wq);
0146         goto end;
0147     }
0148 
0149     mvdev->migrate_cap = 1;
0150     mvdev->core_device.vdev.migration_flags =
0151         VFIO_MIGRATION_STOP_COPY |
0152         VFIO_MIGRATION_P2P;
0153     mvdev->core_device.vdev.mig_ops = mig_ops;
0154 
0155 end:
0156     mlx5_vf_put_core_dev(mvdev->mdev);
0157 }
0158 
0159 static int mlx5vf_cmd_get_vhca_id(struct mlx5_core_dev *mdev, u16 function_id,
0160                   u16 *vhca_id)
0161 {
0162     u32 in[MLX5_ST_SZ_DW(query_hca_cap_in)] = {};
0163     int out_size;
0164     void *out;
0165     int ret;
0166 
0167     out_size = MLX5_ST_SZ_BYTES(query_hca_cap_out);
0168     out = kzalloc(out_size, GFP_KERNEL);
0169     if (!out)
0170         return -ENOMEM;
0171 
0172     MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);
0173     MLX5_SET(query_hca_cap_in, in, other_function, 1);
0174     MLX5_SET(query_hca_cap_in, in, function_id, function_id);
0175     MLX5_SET(query_hca_cap_in, in, op_mod,
0176          MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE << 1 |
0177          HCA_CAP_OPMOD_GET_CUR);
0178 
0179     ret = mlx5_cmd_exec_inout(mdev, query_hca_cap, in, out);
0180     if (ret)
0181         goto err_exec;
0182 
0183     *vhca_id = MLX5_GET(query_hca_cap_out, out,
0184                 capability.cmd_hca_cap.vhca_id);
0185 
0186 err_exec:
0187     kfree(out);
0188     return ret;
0189 }
0190 
0191 static int _create_state_mkey(struct mlx5_core_dev *mdev, u32 pdn,
0192                   struct mlx5_vf_migration_file *migf, u32 *mkey)
0193 {
0194     size_t npages = DIV_ROUND_UP(migf->total_length, PAGE_SIZE);
0195     struct sg_dma_page_iter dma_iter;
0196     int err = 0, inlen;
0197     __be64 *mtt;
0198     void *mkc;
0199     u32 *in;
0200 
0201     inlen = MLX5_ST_SZ_BYTES(create_mkey_in) +
0202         sizeof(*mtt) * round_up(npages, 2);
0203 
0204     in = kvzalloc(inlen, GFP_KERNEL);
0205     if (!in)
0206         return -ENOMEM;
0207 
0208     MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
0209          DIV_ROUND_UP(npages, 2));
0210     mtt = (__be64 *)MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
0211 
0212     for_each_sgtable_dma_page(&migf->table.sgt, &dma_iter, 0)
0213         *mtt++ = cpu_to_be64(sg_page_iter_dma_address(&dma_iter));
0214 
0215     mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
0216     MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);
0217     MLX5_SET(mkc, mkc, lr, 1);
0218     MLX5_SET(mkc, mkc, lw, 1);
0219     MLX5_SET(mkc, mkc, rr, 1);
0220     MLX5_SET(mkc, mkc, rw, 1);
0221     MLX5_SET(mkc, mkc, pd, pdn);
0222     MLX5_SET(mkc, mkc, bsf_octword_size, 0);
0223     MLX5_SET(mkc, mkc, qpn, 0xffffff);
0224     MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
0225     MLX5_SET(mkc, mkc, translations_octword_size, DIV_ROUND_UP(npages, 2));
0226     MLX5_SET64(mkc, mkc, len, migf->total_length);
0227     err = mlx5_core_create_mkey(mdev, mkey, in, inlen);
0228     kvfree(in);
0229     return err;
0230 }
0231 
0232 void mlx5vf_mig_file_cleanup_cb(struct work_struct *_work)
0233 {
0234     struct mlx5vf_async_data *async_data = container_of(_work,
0235         struct mlx5vf_async_data, work);
0236     struct mlx5_vf_migration_file *migf = container_of(async_data,
0237         struct mlx5_vf_migration_file, async_data);
0238     struct mlx5_core_dev *mdev = migf->mvdev->mdev;
0239 
0240     mutex_lock(&migf->lock);
0241     if (async_data->status) {
0242         migf->is_err = true;
0243         wake_up_interruptible(&migf->poll_wait);
0244     }
0245     mutex_unlock(&migf->lock);
0246 
0247     mlx5_core_destroy_mkey(mdev, async_data->mkey);
0248     dma_unmap_sgtable(mdev->device, &migf->table.sgt, DMA_FROM_DEVICE, 0);
0249     mlx5_core_dealloc_pd(mdev, async_data->pdn);
0250     kvfree(async_data->out);
0251     fput(migf->filp);
0252 }
0253 
0254 static void mlx5vf_save_callback(int status, struct mlx5_async_work *context)
0255 {
0256     struct mlx5vf_async_data *async_data = container_of(context,
0257             struct mlx5vf_async_data, cb_work);
0258     struct mlx5_vf_migration_file *migf = container_of(async_data,
0259             struct mlx5_vf_migration_file, async_data);
0260 
0261     if (!status) {
0262         WRITE_ONCE(migf->total_length,
0263                MLX5_GET(save_vhca_state_out, async_data->out,
0264                     actual_image_size));
0265         wake_up_interruptible(&migf->poll_wait);
0266     }
0267 
0268     /*
0269      * The error and the cleanup flows can't run from an
0270      * interrupt context
0271      */
0272     async_data->status = status;
0273     queue_work(migf->mvdev->cb_wq, &async_data->work);
0274 }
0275 
0276 int mlx5vf_cmd_save_vhca_state(struct mlx5vf_pci_core_device *mvdev,
0277                    struct mlx5_vf_migration_file *migf)
0278 {
0279     u32 out_size = MLX5_ST_SZ_BYTES(save_vhca_state_out);
0280     u32 in[MLX5_ST_SZ_DW(save_vhca_state_in)] = {};
0281     struct mlx5vf_async_data *async_data;
0282     struct mlx5_core_dev *mdev;
0283     u32 pdn, mkey;
0284     int err;
0285 
0286     lockdep_assert_held(&mvdev->state_mutex);
0287     if (mvdev->mdev_detach)
0288         return -ENOTCONN;
0289 
0290     mdev = mvdev->mdev;
0291     err = mlx5_core_alloc_pd(mdev, &pdn);
0292     if (err)
0293         return err;
0294 
0295     err = dma_map_sgtable(mdev->device, &migf->table.sgt, DMA_FROM_DEVICE,
0296                   0);
0297     if (err)
0298         goto err_dma_map;
0299 
0300     err = _create_state_mkey(mdev, pdn, migf, &mkey);
0301     if (err)
0302         goto err_create_mkey;
0303 
0304     MLX5_SET(save_vhca_state_in, in, opcode,
0305          MLX5_CMD_OP_SAVE_VHCA_STATE);
0306     MLX5_SET(save_vhca_state_in, in, op_mod, 0);
0307     MLX5_SET(save_vhca_state_in, in, vhca_id, mvdev->vhca_id);
0308     MLX5_SET(save_vhca_state_in, in, mkey, mkey);
0309     MLX5_SET(save_vhca_state_in, in, size, migf->total_length);
0310 
0311     async_data = &migf->async_data;
0312     async_data->out = kvzalloc(out_size, GFP_KERNEL);
0313     if (!async_data->out) {
0314         err = -ENOMEM;
0315         goto err_out;
0316     }
0317 
0318     /* no data exists till the callback comes back */
0319     migf->total_length = 0;
0320     get_file(migf->filp);
0321     async_data->mkey = mkey;
0322     async_data->pdn = pdn;
0323     err = mlx5_cmd_exec_cb(&migf->async_ctx, in, sizeof(in),
0324                    async_data->out,
0325                    out_size, mlx5vf_save_callback,
0326                    &async_data->cb_work);
0327     if (err)
0328         goto err_exec;
0329 
0330     return 0;
0331 
0332 err_exec:
0333     fput(migf->filp);
0334     kvfree(async_data->out);
0335 err_out:
0336     mlx5_core_destroy_mkey(mdev, mkey);
0337 err_create_mkey:
0338     dma_unmap_sgtable(mdev->device, &migf->table.sgt, DMA_FROM_DEVICE, 0);
0339 err_dma_map:
0340     mlx5_core_dealloc_pd(mdev, pdn);
0341     return err;
0342 }
0343 
0344 int mlx5vf_cmd_load_vhca_state(struct mlx5vf_pci_core_device *mvdev,
0345                    struct mlx5_vf_migration_file *migf)
0346 {
0347     struct mlx5_core_dev *mdev;
0348     u32 out[MLX5_ST_SZ_DW(save_vhca_state_out)] = {};
0349     u32 in[MLX5_ST_SZ_DW(save_vhca_state_in)] = {};
0350     u32 pdn, mkey;
0351     int err;
0352 
0353     lockdep_assert_held(&mvdev->state_mutex);
0354     if (mvdev->mdev_detach)
0355         return -ENOTCONN;
0356 
0357     mutex_lock(&migf->lock);
0358     if (!migf->total_length) {
0359         err = -EINVAL;
0360         goto end;
0361     }
0362 
0363     mdev = mvdev->mdev;
0364     err = mlx5_core_alloc_pd(mdev, &pdn);
0365     if (err)
0366         goto end;
0367 
0368     err = dma_map_sgtable(mdev->device, &migf->table.sgt, DMA_TO_DEVICE, 0);
0369     if (err)
0370         goto err_reg;
0371 
0372     err = _create_state_mkey(mdev, pdn, migf, &mkey);
0373     if (err)
0374         goto err_mkey;
0375 
0376     MLX5_SET(load_vhca_state_in, in, opcode,
0377          MLX5_CMD_OP_LOAD_VHCA_STATE);
0378     MLX5_SET(load_vhca_state_in, in, op_mod, 0);
0379     MLX5_SET(load_vhca_state_in, in, vhca_id, mvdev->vhca_id);
0380     MLX5_SET(load_vhca_state_in, in, mkey, mkey);
0381     MLX5_SET(load_vhca_state_in, in, size, migf->total_length);
0382 
0383     err = mlx5_cmd_exec_inout(mdev, load_vhca_state, in, out);
0384 
0385     mlx5_core_destroy_mkey(mdev, mkey);
0386 err_mkey:
0387     dma_unmap_sgtable(mdev->device, &migf->table.sgt, DMA_TO_DEVICE, 0);
0388 err_reg:
0389     mlx5_core_dealloc_pd(mdev, pdn);
0390 end:
0391     mutex_unlock(&migf->lock);
0392     return err;
0393 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!