Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0 OR MIT
0002 /*
0003  * Copyright 2014-2022 Advanced Micro Devices, Inc.
0004  *
0005  * Permission is hereby granted, free of charge, to any person obtaining a
0006  * copy of this software and associated documentation files (the "Software"),
0007  * to deal in the Software without restriction, including without limitation
0008  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0009  * and/or sell copies of the Software, and to permit persons to whom the
0010  * Software is furnished to do so, subject to the following conditions:
0011  *
0012  * The above copyright notice and this permission notice shall be included in
0013  * all copies or substantial portions of the Software.
0014  *
0015  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0016  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0017  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0018  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
0019  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
0020  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
0021  * OTHER DEALINGS IN THE SOFTWARE.
0022  *
0023  */
0024 
0025 #include <linux/slab.h>
0026 #include <linux/list.h>
0027 #include "kfd_device_queue_manager.h"
0028 #include "kfd_priv.h"
0029 #include "kfd_kernel_queue.h"
0030 #include "amdgpu_amdkfd.h"
0031 
0032 static inline struct process_queue_node *get_queue_by_qid(
0033             struct process_queue_manager *pqm, unsigned int qid)
0034 {
0035     struct process_queue_node *pqn;
0036 
0037     list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
0038         if ((pqn->q && pqn->q->properties.queue_id == qid) ||
0039             (pqn->kq && pqn->kq->queue->properties.queue_id == qid))
0040             return pqn;
0041     }
0042 
0043     return NULL;
0044 }
0045 
0046 static int assign_queue_slot_by_qid(struct process_queue_manager *pqm,
0047                     unsigned int qid)
0048 {
0049     if (qid >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)
0050         return -EINVAL;
0051 
0052     if (__test_and_set_bit(qid, pqm->queue_slot_bitmap)) {
0053         pr_err("Cannot create new queue because requested qid(%u) is in use\n", qid);
0054         return -ENOSPC;
0055     }
0056 
0057     return 0;
0058 }
0059 
0060 static int find_available_queue_slot(struct process_queue_manager *pqm,
0061                     unsigned int *qid)
0062 {
0063     unsigned long found;
0064 
0065     found = find_first_zero_bit(pqm->queue_slot_bitmap,
0066             KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
0067 
0068     pr_debug("The new slot id %lu\n", found);
0069 
0070     if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) {
0071         pr_info("Cannot open more queues for process with pasid 0x%x\n",
0072                 pqm->process->pasid);
0073         return -ENOMEM;
0074     }
0075 
0076     set_bit(found, pqm->queue_slot_bitmap);
0077     *qid = found;
0078 
0079     return 0;
0080 }
0081 
0082 void kfd_process_dequeue_from_device(struct kfd_process_device *pdd)
0083 {
0084     struct kfd_dev *dev = pdd->dev;
0085 
0086     if (pdd->already_dequeued)
0087         return;
0088 
0089     dev->dqm->ops.process_termination(dev->dqm, &pdd->qpd);
0090     pdd->already_dequeued = true;
0091 }
0092 
0093 int pqm_set_gws(struct process_queue_manager *pqm, unsigned int qid,
0094             void *gws)
0095 {
0096     struct kfd_dev *dev = NULL;
0097     struct process_queue_node *pqn;
0098     struct kfd_process_device *pdd;
0099     struct kgd_mem *mem = NULL;
0100     int ret;
0101 
0102     pqn = get_queue_by_qid(pqm, qid);
0103     if (!pqn) {
0104         pr_err("Queue id does not match any known queue\n");
0105         return -EINVAL;
0106     }
0107 
0108     if (pqn->q)
0109         dev = pqn->q->device;
0110     if (WARN_ON(!dev))
0111         return -ENODEV;
0112 
0113     pdd = kfd_get_process_device_data(dev, pqm->process);
0114     if (!pdd) {
0115         pr_err("Process device data doesn't exist\n");
0116         return -EINVAL;
0117     }
0118 
0119     /* Only allow one queue per process can have GWS assigned */
0120     if (gws && pdd->qpd.num_gws)
0121         return -EBUSY;
0122 
0123     if (!gws && pdd->qpd.num_gws == 0)
0124         return -EINVAL;
0125 
0126     if (gws)
0127         ret = amdgpu_amdkfd_add_gws_to_process(pdd->process->kgd_process_info,
0128             gws, &mem);
0129     else
0130         ret = amdgpu_amdkfd_remove_gws_from_process(pdd->process->kgd_process_info,
0131             pqn->q->gws);
0132     if (unlikely(ret))
0133         return ret;
0134 
0135     pqn->q->gws = mem;
0136     pdd->qpd.num_gws = gws ? dev->adev->gds.gws_size : 0;
0137 
0138     return pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,
0139                             pqn->q, NULL);
0140 }
0141 
0142 void kfd_process_dequeue_from_all_devices(struct kfd_process *p)
0143 {
0144     int i;
0145 
0146     for (i = 0; i < p->n_pdds; i++)
0147         kfd_process_dequeue_from_device(p->pdds[i]);
0148 }
0149 
0150 int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p)
0151 {
0152     INIT_LIST_HEAD(&pqm->queues);
0153     pqm->queue_slot_bitmap = bitmap_zalloc(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,
0154                            GFP_KERNEL);
0155     if (!pqm->queue_slot_bitmap)
0156         return -ENOMEM;
0157     pqm->process = p;
0158 
0159     return 0;
0160 }
0161 
0162 void pqm_uninit(struct process_queue_manager *pqm)
0163 {
0164     struct process_queue_node *pqn, *next;
0165 
0166     list_for_each_entry_safe(pqn, next, &pqm->queues, process_queue_list) {
0167         if (pqn->q && pqn->q->gws)
0168             amdgpu_amdkfd_remove_gws_from_process(pqm->process->kgd_process_info,
0169                 pqn->q->gws);
0170         kfd_procfs_del_queue(pqn->q);
0171         uninit_queue(pqn->q);
0172         list_del(&pqn->process_queue_list);
0173         kfree(pqn);
0174     }
0175 
0176     bitmap_free(pqm->queue_slot_bitmap);
0177     pqm->queue_slot_bitmap = NULL;
0178 }
0179 
0180 static int init_user_queue(struct process_queue_manager *pqm,
0181                 struct kfd_dev *dev, struct queue **q,
0182                 struct queue_properties *q_properties,
0183                 struct file *f, struct amdgpu_bo *wptr_bo,
0184                 unsigned int qid)
0185 {
0186     int retval;
0187 
0188     /* Doorbell initialized in user space*/
0189     q_properties->doorbell_ptr = NULL;
0190 
0191     /* let DQM handle it*/
0192     q_properties->vmid = 0;
0193     q_properties->queue_id = qid;
0194 
0195     retval = init_queue(q, q_properties);
0196     if (retval != 0)
0197         return retval;
0198 
0199     (*q)->device = dev;
0200     (*q)->process = pqm->process;
0201 
0202     if (dev->shared_resources.enable_mes) {
0203         retval = amdgpu_amdkfd_alloc_gtt_mem(dev->adev,
0204                         AMDGPU_MES_GANG_CTX_SIZE,
0205                         &(*q)->gang_ctx_bo,
0206                         &(*q)->gang_ctx_gpu_addr,
0207                         &(*q)->gang_ctx_cpu_ptr,
0208                         false);
0209         if (retval) {
0210             pr_err("failed to allocate gang context bo\n");
0211             goto cleanup;
0212         }
0213         memset((*q)->gang_ctx_cpu_ptr, 0, AMDGPU_MES_GANG_CTX_SIZE);
0214         (*q)->wptr_bo = wptr_bo;
0215     }
0216 
0217     pr_debug("PQM After init queue");
0218     return 0;
0219 
0220 cleanup:
0221     if (dev->shared_resources.enable_mes)
0222         uninit_queue(*q);
0223     return retval;
0224 }
0225 
0226 int pqm_create_queue(struct process_queue_manager *pqm,
0227                 struct kfd_dev *dev,
0228                 struct file *f,
0229                 struct queue_properties *properties,
0230                 unsigned int *qid,
0231                 struct amdgpu_bo *wptr_bo,
0232                 const struct kfd_criu_queue_priv_data *q_data,
0233                 const void *restore_mqd,
0234                 const void *restore_ctl_stack,
0235                 uint32_t *p_doorbell_offset_in_process)
0236 {
0237     int retval;
0238     struct kfd_process_device *pdd;
0239     struct queue *q;
0240     struct process_queue_node *pqn;
0241     struct kernel_queue *kq;
0242     enum kfd_queue_type type = properties->type;
0243     unsigned int max_queues = 127; /* HWS limit */
0244 
0245     q = NULL;
0246     kq = NULL;
0247 
0248     pdd = kfd_get_process_device_data(dev, pqm->process);
0249     if (!pdd) {
0250         pr_err("Process device data doesn't exist\n");
0251         return -1;
0252     }
0253 
0254     /*
0255      * for debug process, verify that it is within the static queues limit
0256      * currently limit is set to half of the total avail HQD slots
0257      * If we are just about to create DIQ, the is_debug flag is not set yet
0258      * Hence we also check the type as well
0259      */
0260     if ((pdd->qpd.is_debug) || (type == KFD_QUEUE_TYPE_DIQ))
0261         max_queues = dev->device_info.max_no_of_hqd/2;
0262 
0263     if (pdd->qpd.queue_count >= max_queues)
0264         return -ENOSPC;
0265 
0266     if (q_data) {
0267         retval = assign_queue_slot_by_qid(pqm, q_data->q_id);
0268         *qid = q_data->q_id;
0269     } else
0270         retval = find_available_queue_slot(pqm, qid);
0271 
0272     if (retval != 0)
0273         return retval;
0274 
0275     if (list_empty(&pdd->qpd.queues_list) &&
0276         list_empty(&pdd->qpd.priv_queue_list))
0277         dev->dqm->ops.register_process(dev->dqm, &pdd->qpd);
0278 
0279     pqn = kzalloc(sizeof(*pqn), GFP_KERNEL);
0280     if (!pqn) {
0281         retval = -ENOMEM;
0282         goto err_allocate_pqn;
0283     }
0284 
0285     switch (type) {
0286     case KFD_QUEUE_TYPE_SDMA:
0287     case KFD_QUEUE_TYPE_SDMA_XGMI:
0288         /* SDMA queues are always allocated statically no matter
0289          * which scheduler mode is used. We also do not need to
0290          * check whether a SDMA queue can be allocated here, because
0291          * allocate_sdma_queue() in create_queue() has the
0292          * corresponding check logic.
0293          */
0294         retval = init_user_queue(pqm, dev, &q, properties, f, wptr_bo, *qid);
0295         if (retval != 0)
0296             goto err_create_queue;
0297         pqn->q = q;
0298         pqn->kq = NULL;
0299         retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data,
0300                             restore_mqd, restore_ctl_stack);
0301         print_queue(q);
0302         break;
0303 
0304     case KFD_QUEUE_TYPE_COMPUTE:
0305         /* check if there is over subscription */
0306         if ((dev->dqm->sched_policy ==
0307              KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION) &&
0308         ((dev->dqm->processes_count >= dev->vm_info.vmid_num_kfd) ||
0309         (dev->dqm->active_queue_count >= get_cp_queues_num(dev->dqm)))) {
0310             pr_debug("Over-subscription is not allowed when amdkfd.sched_policy == 1\n");
0311             retval = -EPERM;
0312             goto err_create_queue;
0313         }
0314 
0315         retval = init_user_queue(pqm, dev, &q, properties, f, wptr_bo, *qid);
0316         if (retval != 0)
0317             goto err_create_queue;
0318         pqn->q = q;
0319         pqn->kq = NULL;
0320         retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data,
0321                             restore_mqd, restore_ctl_stack);
0322         print_queue(q);
0323         break;
0324     case KFD_QUEUE_TYPE_DIQ:
0325         kq = kernel_queue_init(dev, KFD_QUEUE_TYPE_DIQ);
0326         if (!kq) {
0327             retval = -ENOMEM;
0328             goto err_create_queue;
0329         }
0330         kq->queue->properties.queue_id = *qid;
0331         pqn->kq = kq;
0332         pqn->q = NULL;
0333         retval = dev->dqm->ops.create_kernel_queue(dev->dqm,
0334                             kq, &pdd->qpd);
0335         break;
0336     default:
0337         WARN(1, "Invalid queue type %d", type);
0338         retval = -EINVAL;
0339     }
0340 
0341     if (retval != 0) {
0342         pr_err("Pasid 0x%x DQM create queue type %d failed. ret %d\n",
0343             pqm->process->pasid, type, retval);
0344         goto err_create_queue;
0345     }
0346 
0347     if (q && p_doorbell_offset_in_process)
0348         /* Return the doorbell offset within the doorbell page
0349          * to the caller so it can be passed up to user mode
0350          * (in bytes).
0351          * There are always 1024 doorbells per process, so in case
0352          * of 8-byte doorbells, there are two doorbell pages per
0353          * process.
0354          */
0355         *p_doorbell_offset_in_process =
0356             (q->properties.doorbell_off * sizeof(uint32_t)) &
0357             (kfd_doorbell_process_slice(dev) - 1);
0358 
0359     pr_debug("PQM After DQM create queue\n");
0360 
0361     list_add(&pqn->process_queue_list, &pqm->queues);
0362 
0363     if (q) {
0364         pr_debug("PQM done creating queue\n");
0365         kfd_procfs_add_queue(q);
0366         print_queue_properties(&q->properties);
0367     }
0368 
0369     return retval;
0370 
0371 err_create_queue:
0372     uninit_queue(q);
0373     if (kq)
0374         kernel_queue_uninit(kq, false);
0375     kfree(pqn);
0376 err_allocate_pqn:
0377     /* check if queues list is empty unregister process from device */
0378     clear_bit(*qid, pqm->queue_slot_bitmap);
0379     if (list_empty(&pdd->qpd.queues_list) &&
0380         list_empty(&pdd->qpd.priv_queue_list))
0381         dev->dqm->ops.unregister_process(dev->dqm, &pdd->qpd);
0382     return retval;
0383 }
0384 
0385 int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid)
0386 {
0387     struct process_queue_node *pqn;
0388     struct kfd_process_device *pdd;
0389     struct device_queue_manager *dqm;
0390     struct kfd_dev *dev;
0391     int retval;
0392 
0393     dqm = NULL;
0394 
0395     retval = 0;
0396 
0397     pqn = get_queue_by_qid(pqm, qid);
0398     if (!pqn) {
0399         pr_err("Queue id does not match any known queue\n");
0400         return -EINVAL;
0401     }
0402 
0403     dev = NULL;
0404     if (pqn->kq)
0405         dev = pqn->kq->dev;
0406     if (pqn->q)
0407         dev = pqn->q->device;
0408     if (WARN_ON(!dev))
0409         return -ENODEV;
0410 
0411     pdd = kfd_get_process_device_data(dev, pqm->process);
0412     if (!pdd) {
0413         pr_err("Process device data doesn't exist\n");
0414         return -1;
0415     }
0416 
0417     if (pqn->kq) {
0418         /* destroy kernel queue (DIQ) */
0419         dqm = pqn->kq->dev->dqm;
0420         dqm->ops.destroy_kernel_queue(dqm, pqn->kq, &pdd->qpd);
0421         kernel_queue_uninit(pqn->kq, false);
0422     }
0423 
0424     if (pqn->q) {
0425         kfd_procfs_del_queue(pqn->q);
0426         dqm = pqn->q->device->dqm;
0427         retval = dqm->ops.destroy_queue(dqm, &pdd->qpd, pqn->q);
0428         if (retval) {
0429             pr_err("Pasid 0x%x destroy queue %d failed, ret %d\n",
0430                 pqm->process->pasid,
0431                 pqn->q->properties.queue_id, retval);
0432             if (retval != -ETIME)
0433                 goto err_destroy_queue;
0434         }
0435 
0436         if (pqn->q->gws) {
0437             amdgpu_amdkfd_remove_gws_from_process(pqm->process->kgd_process_info,
0438                 pqn->q->gws);
0439             pdd->qpd.num_gws = 0;
0440         }
0441 
0442         if (dev->shared_resources.enable_mes) {
0443             amdgpu_amdkfd_free_gtt_mem(dev->adev,
0444                            pqn->q->gang_ctx_bo);
0445             if (pqn->q->wptr_bo)
0446                 amdgpu_amdkfd_free_gtt_mem(dev->adev, pqn->q->wptr_bo);
0447 
0448         }
0449         uninit_queue(pqn->q);
0450     }
0451 
0452     list_del(&pqn->process_queue_list);
0453     kfree(pqn);
0454     clear_bit(qid, pqm->queue_slot_bitmap);
0455 
0456     if (list_empty(&pdd->qpd.queues_list) &&
0457         list_empty(&pdd->qpd.priv_queue_list))
0458         dqm->ops.unregister_process(dqm, &pdd->qpd);
0459 
0460 err_destroy_queue:
0461     return retval;
0462 }
0463 
0464 int pqm_update_queue_properties(struct process_queue_manager *pqm,
0465                 unsigned int qid, struct queue_properties *p)
0466 {
0467     int retval;
0468     struct process_queue_node *pqn;
0469 
0470     pqn = get_queue_by_qid(pqm, qid);
0471     if (!pqn) {
0472         pr_debug("No queue %d exists for update operation\n", qid);
0473         return -EFAULT;
0474     }
0475 
0476     pqn->q->properties.queue_address = p->queue_address;
0477     pqn->q->properties.queue_size = p->queue_size;
0478     pqn->q->properties.queue_percent = p->queue_percent;
0479     pqn->q->properties.priority = p->priority;
0480 
0481     retval = pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,
0482                             pqn->q, NULL);
0483     if (retval != 0)
0484         return retval;
0485 
0486     return 0;
0487 }
0488 
0489 int pqm_update_mqd(struct process_queue_manager *pqm,
0490                 unsigned int qid, struct mqd_update_info *minfo)
0491 {
0492     int retval;
0493     struct process_queue_node *pqn;
0494 
0495     pqn = get_queue_by_qid(pqm, qid);
0496     if (!pqn) {
0497         pr_debug("No queue %d exists for update operation\n", qid);
0498         return -EFAULT;
0499     }
0500 
0501     /* ASICs that have WGPs must enforce pairwise enabled mask checks. */
0502     if (minfo && minfo->update_flag == UPDATE_FLAG_CU_MASK && minfo->cu_mask.ptr &&
0503             KFD_GC_VERSION(pqn->q->device) >= IP_VERSION(10, 0, 0)) {
0504         int i;
0505 
0506         for (i = 0; i < minfo->cu_mask.count; i += 2) {
0507             uint32_t cu_pair = (minfo->cu_mask.ptr[i / 32] >> (i % 32)) & 0x3;
0508 
0509             if (cu_pair && cu_pair != 0x3) {
0510                 pr_debug("CUs must be adjacent pairwise enabled.\n");
0511                 return -EINVAL;
0512             }
0513         }
0514     }
0515 
0516     retval = pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,
0517                             pqn->q, minfo);
0518     if (retval != 0)
0519         return retval;
0520 
0521     return 0;
0522 }
0523 
0524 struct kernel_queue *pqm_get_kernel_queue(
0525                     struct process_queue_manager *pqm,
0526                     unsigned int qid)
0527 {
0528     struct process_queue_node *pqn;
0529 
0530     pqn = get_queue_by_qid(pqm, qid);
0531     if (pqn && pqn->kq)
0532         return pqn->kq;
0533 
0534     return NULL;
0535 }
0536 
0537 struct queue *pqm_get_user_queue(struct process_queue_manager *pqm,
0538                     unsigned int qid)
0539 {
0540     struct process_queue_node *pqn;
0541 
0542     pqn = get_queue_by_qid(pqm, qid);
0543     return pqn ? pqn->q : NULL;
0544 }
0545 
0546 int pqm_get_wave_state(struct process_queue_manager *pqm,
0547                unsigned int qid,
0548                void __user *ctl_stack,
0549                u32 *ctl_stack_used_size,
0550                u32 *save_area_used_size)
0551 {
0552     struct process_queue_node *pqn;
0553 
0554     pqn = get_queue_by_qid(pqm, qid);
0555     if (!pqn) {
0556         pr_debug("amdkfd: No queue %d exists for operation\n",
0557              qid);
0558         return -EFAULT;
0559     }
0560 
0561     return pqn->q->device->dqm->ops.get_wave_state(pqn->q->device->dqm,
0562                                pqn->q,
0563                                ctl_stack,
0564                                ctl_stack_used_size,
0565                                save_area_used_size);
0566 }
0567 
0568 static int get_queue_data_sizes(struct kfd_process_device *pdd,
0569                 struct queue *q,
0570                 uint32_t *mqd_size,
0571                 uint32_t *ctl_stack_size)
0572 {
0573     int ret;
0574 
0575     ret = pqm_get_queue_checkpoint_info(&pdd->process->pqm,
0576                         q->properties.queue_id,
0577                         mqd_size,
0578                         ctl_stack_size);
0579     if (ret)
0580         pr_err("Failed to get queue dump info (%d)\n", ret);
0581 
0582     return ret;
0583 }
0584 
0585 int kfd_process_get_queue_info(struct kfd_process *p,
0586                    uint32_t *num_queues,
0587                    uint64_t *priv_data_sizes)
0588 {
0589     uint32_t extra_data_sizes = 0;
0590     struct queue *q;
0591     int i;
0592     int ret;
0593 
0594     *num_queues = 0;
0595 
0596     /* Run over all PDDs of the process */
0597     for (i = 0; i < p->n_pdds; i++) {
0598         struct kfd_process_device *pdd = p->pdds[i];
0599 
0600         list_for_each_entry(q, &pdd->qpd.queues_list, list) {
0601             if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
0602                 q->properties.type == KFD_QUEUE_TYPE_SDMA ||
0603                 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
0604                 uint32_t mqd_size, ctl_stack_size;
0605 
0606                 *num_queues = *num_queues + 1;
0607 
0608                 ret = get_queue_data_sizes(pdd, q, &mqd_size, &ctl_stack_size);
0609                 if (ret)
0610                     return ret;
0611 
0612                 extra_data_sizes += mqd_size + ctl_stack_size;
0613             } else {
0614                 pr_err("Unsupported queue type (%d)\n", q->properties.type);
0615                 return -EOPNOTSUPP;
0616             }
0617         }
0618     }
0619     *priv_data_sizes = extra_data_sizes +
0620                 (*num_queues * sizeof(struct kfd_criu_queue_priv_data));
0621 
0622     return 0;
0623 }
0624 
0625 static int pqm_checkpoint_mqd(struct process_queue_manager *pqm,
0626                   unsigned int qid,
0627                   void *mqd,
0628                   void *ctl_stack)
0629 {
0630     struct process_queue_node *pqn;
0631 
0632     pqn = get_queue_by_qid(pqm, qid);
0633     if (!pqn) {
0634         pr_debug("amdkfd: No queue %d exists for operation\n", qid);
0635         return -EFAULT;
0636     }
0637 
0638     if (!pqn->q->device->dqm->ops.checkpoint_mqd) {
0639         pr_err("amdkfd: queue dumping not supported on this device\n");
0640         return -EOPNOTSUPP;
0641     }
0642 
0643     return pqn->q->device->dqm->ops.checkpoint_mqd(pqn->q->device->dqm,
0644                                pqn->q, mqd, ctl_stack);
0645 }
0646 
0647 static int criu_checkpoint_queue(struct kfd_process_device *pdd,
0648                struct queue *q,
0649                struct kfd_criu_queue_priv_data *q_data)
0650 {
0651     uint8_t *mqd, *ctl_stack;
0652     int ret;
0653 
0654     mqd = (void *)(q_data + 1);
0655     ctl_stack = mqd + q_data->mqd_size;
0656 
0657     q_data->gpu_id = pdd->user_gpu_id;
0658     q_data->type = q->properties.type;
0659     q_data->format = q->properties.format;
0660     q_data->q_id =  q->properties.queue_id;
0661     q_data->q_address = q->properties.queue_address;
0662     q_data->q_size = q->properties.queue_size;
0663     q_data->priority = q->properties.priority;
0664     q_data->q_percent = q->properties.queue_percent;
0665     q_data->read_ptr_addr = (uint64_t)q->properties.read_ptr;
0666     q_data->write_ptr_addr = (uint64_t)q->properties.write_ptr;
0667     q_data->doorbell_id = q->doorbell_id;
0668 
0669     q_data->sdma_id = q->sdma_id;
0670 
0671     q_data->eop_ring_buffer_address =
0672         q->properties.eop_ring_buffer_address;
0673 
0674     q_data->eop_ring_buffer_size = q->properties.eop_ring_buffer_size;
0675 
0676     q_data->ctx_save_restore_area_address =
0677         q->properties.ctx_save_restore_area_address;
0678 
0679     q_data->ctx_save_restore_area_size =
0680         q->properties.ctx_save_restore_area_size;
0681 
0682     q_data->gws = !!q->gws;
0683 
0684     ret = pqm_checkpoint_mqd(&pdd->process->pqm, q->properties.queue_id, mqd, ctl_stack);
0685     if (ret) {
0686         pr_err("Failed checkpoint queue_mqd (%d)\n", ret);
0687         return ret;
0688     }
0689 
0690     pr_debug("Dumping Queue: gpu_id:%x queue_id:%u\n", q_data->gpu_id, q_data->q_id);
0691     return ret;
0692 }
0693 
0694 static int criu_checkpoint_queues_device(struct kfd_process_device *pdd,
0695                    uint8_t __user *user_priv,
0696                    unsigned int *q_index,
0697                    uint64_t *queues_priv_data_offset)
0698 {
0699     unsigned int q_private_data_size = 0;
0700     uint8_t *q_private_data = NULL; /* Local buffer to store individual queue private data */
0701     struct queue *q;
0702     int ret = 0;
0703 
0704     list_for_each_entry(q, &pdd->qpd.queues_list, list) {
0705         struct kfd_criu_queue_priv_data *q_data;
0706         uint64_t q_data_size;
0707         uint32_t mqd_size;
0708         uint32_t ctl_stack_size;
0709 
0710         if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE &&
0711             q->properties.type != KFD_QUEUE_TYPE_SDMA &&
0712             q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI) {
0713 
0714             pr_err("Unsupported queue type (%d)\n", q->properties.type);
0715             ret = -EOPNOTSUPP;
0716             break;
0717         }
0718 
0719         ret = get_queue_data_sizes(pdd, q, &mqd_size, &ctl_stack_size);
0720         if (ret)
0721             break;
0722 
0723         q_data_size = sizeof(*q_data) + mqd_size + ctl_stack_size;
0724 
0725         /* Increase local buffer space if needed */
0726         if (q_private_data_size < q_data_size) {
0727             kfree(q_private_data);
0728 
0729             q_private_data = kzalloc(q_data_size, GFP_KERNEL);
0730             if (!q_private_data) {
0731                 ret = -ENOMEM;
0732                 break;
0733             }
0734             q_private_data_size = q_data_size;
0735         }
0736 
0737         q_data = (struct kfd_criu_queue_priv_data *)q_private_data;
0738 
0739         /* data stored in this order: priv_data, mqd, ctl_stack */
0740         q_data->mqd_size = mqd_size;
0741         q_data->ctl_stack_size = ctl_stack_size;
0742 
0743         ret = criu_checkpoint_queue(pdd, q, q_data);
0744         if (ret)
0745             break;
0746 
0747         q_data->object_type = KFD_CRIU_OBJECT_TYPE_QUEUE;
0748 
0749         ret = copy_to_user(user_priv + *queues_priv_data_offset,
0750                 q_data, q_data_size);
0751         if (ret) {
0752             ret = -EFAULT;
0753             break;
0754         }
0755         *queues_priv_data_offset += q_data_size;
0756         *q_index = *q_index + 1;
0757     }
0758 
0759     kfree(q_private_data);
0760 
0761     return ret;
0762 }
0763 
0764 int kfd_criu_checkpoint_queues(struct kfd_process *p,
0765              uint8_t __user *user_priv_data,
0766              uint64_t *priv_data_offset)
0767 {
0768     int ret = 0, pdd_index, q_index = 0;
0769 
0770     for (pdd_index = 0; pdd_index < p->n_pdds; pdd_index++) {
0771         struct kfd_process_device *pdd = p->pdds[pdd_index];
0772 
0773         /*
0774          * criu_checkpoint_queues_device will copy data to user and update q_index and
0775          * queues_priv_data_offset
0776          */
0777         ret = criu_checkpoint_queues_device(pdd, user_priv_data, &q_index,
0778                           priv_data_offset);
0779 
0780         if (ret)
0781             break;
0782     }
0783 
0784     return ret;
0785 }
0786 
0787 static void set_queue_properties_from_criu(struct queue_properties *qp,
0788                       struct kfd_criu_queue_priv_data *q_data)
0789 {
0790     qp->is_interop = false;
0791     qp->queue_percent = q_data->q_percent;
0792     qp->priority = q_data->priority;
0793     qp->queue_address = q_data->q_address;
0794     qp->queue_size = q_data->q_size;
0795     qp->read_ptr = (uint32_t *) q_data->read_ptr_addr;
0796     qp->write_ptr = (uint32_t *) q_data->write_ptr_addr;
0797     qp->eop_ring_buffer_address = q_data->eop_ring_buffer_address;
0798     qp->eop_ring_buffer_size = q_data->eop_ring_buffer_size;
0799     qp->ctx_save_restore_area_address = q_data->ctx_save_restore_area_address;
0800     qp->ctx_save_restore_area_size = q_data->ctx_save_restore_area_size;
0801     qp->ctl_stack_size = q_data->ctl_stack_size;
0802     qp->type = q_data->type;
0803     qp->format = q_data->format;
0804 }
0805 
0806 int kfd_criu_restore_queue(struct kfd_process *p,
0807                uint8_t __user *user_priv_ptr,
0808                uint64_t *priv_data_offset,
0809                uint64_t max_priv_data_size)
0810 {
0811     uint8_t *mqd, *ctl_stack, *q_extra_data = NULL;
0812     struct kfd_criu_queue_priv_data *q_data;
0813     struct kfd_process_device *pdd;
0814     uint64_t q_extra_data_size;
0815     struct queue_properties qp;
0816     unsigned int queue_id;
0817     int ret = 0;
0818 
0819     if (*priv_data_offset + sizeof(*q_data) > max_priv_data_size)
0820         return -EINVAL;
0821 
0822     q_data = kmalloc(sizeof(*q_data), GFP_KERNEL);
0823     if (!q_data)
0824         return -ENOMEM;
0825 
0826     ret = copy_from_user(q_data, user_priv_ptr + *priv_data_offset, sizeof(*q_data));
0827     if (ret) {
0828         ret = -EFAULT;
0829         goto exit;
0830     }
0831 
0832     *priv_data_offset += sizeof(*q_data);
0833     q_extra_data_size = (uint64_t)q_data->ctl_stack_size + q_data->mqd_size;
0834 
0835     if (*priv_data_offset + q_extra_data_size > max_priv_data_size) {
0836         ret = -EINVAL;
0837         goto exit;
0838     }
0839 
0840     q_extra_data = kmalloc(q_extra_data_size, GFP_KERNEL);
0841     if (!q_extra_data) {
0842         ret = -ENOMEM;
0843         goto exit;
0844     }
0845 
0846     ret = copy_from_user(q_extra_data, user_priv_ptr + *priv_data_offset, q_extra_data_size);
0847     if (ret) {
0848         ret = -EFAULT;
0849         goto exit;
0850     }
0851 
0852     *priv_data_offset += q_extra_data_size;
0853 
0854     pdd = kfd_process_device_data_by_id(p, q_data->gpu_id);
0855     if (!pdd) {
0856         pr_err("Failed to get pdd\n");
0857         ret = -EINVAL;
0858         goto exit;
0859     }
0860     /* data stored in this order: mqd, ctl_stack */
0861     mqd = q_extra_data;
0862     ctl_stack = mqd + q_data->mqd_size;
0863 
0864     memset(&qp, 0, sizeof(qp));
0865     set_queue_properties_from_criu(&qp, q_data);
0866 
0867     print_queue_properties(&qp);
0868 
0869     ret = pqm_create_queue(&p->pqm, pdd->dev, NULL, &qp, &queue_id, NULL, q_data, mqd, ctl_stack,
0870                 NULL);
0871     if (ret) {
0872         pr_err("Failed to create new queue err:%d\n", ret);
0873         goto exit;
0874     }
0875 
0876     if (q_data->gws)
0877         ret = pqm_set_gws(&p->pqm, q_data->q_id, pdd->dev->gws);
0878 
0879 exit:
0880     if (ret)
0881         pr_err("Failed to restore queue (%d)\n", ret);
0882     else
0883         pr_debug("Queue id %d was restored successfully\n", queue_id);
0884 
0885     kfree(q_data);
0886 
0887     return ret;
0888 }
0889 
0890 int pqm_get_queue_checkpoint_info(struct process_queue_manager *pqm,
0891                   unsigned int qid,
0892                   uint32_t *mqd_size,
0893                   uint32_t *ctl_stack_size)
0894 {
0895     struct process_queue_node *pqn;
0896 
0897     pqn = get_queue_by_qid(pqm, qid);
0898     if (!pqn) {
0899         pr_debug("amdkfd: No queue %d exists for operation\n", qid);
0900         return -EFAULT;
0901     }
0902 
0903     if (!pqn->q->device->dqm->ops.get_queue_checkpoint_info) {
0904         pr_err("amdkfd: queue dumping not supported on this device\n");
0905         return -EOPNOTSUPP;
0906     }
0907 
0908     pqn->q->device->dqm->ops.get_queue_checkpoint_info(pqn->q->device->dqm,
0909                                pqn->q, mqd_size,
0910                                ctl_stack_size);
0911     return 0;
0912 }
0913 
0914 #if defined(CONFIG_DEBUG_FS)
0915 
0916 int pqm_debugfs_mqds(struct seq_file *m, void *data)
0917 {
0918     struct process_queue_manager *pqm = data;
0919     struct process_queue_node *pqn;
0920     struct queue *q;
0921     enum KFD_MQD_TYPE mqd_type;
0922     struct mqd_manager *mqd_mgr;
0923     int r = 0;
0924 
0925     list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
0926         if (pqn->q) {
0927             q = pqn->q;
0928             switch (q->properties.type) {
0929             case KFD_QUEUE_TYPE_SDMA:
0930             case KFD_QUEUE_TYPE_SDMA_XGMI:
0931                 seq_printf(m, "  SDMA queue on device %x\n",
0932                        q->device->id);
0933                 mqd_type = KFD_MQD_TYPE_SDMA;
0934                 break;
0935             case KFD_QUEUE_TYPE_COMPUTE:
0936                 seq_printf(m, "  Compute queue on device %x\n",
0937                        q->device->id);
0938                 mqd_type = KFD_MQD_TYPE_CP;
0939                 break;
0940             default:
0941                 seq_printf(m,
0942                 "  Bad user queue type %d on device %x\n",
0943                        q->properties.type, q->device->id);
0944                 continue;
0945             }
0946             mqd_mgr = q->device->dqm->mqd_mgrs[mqd_type];
0947         } else if (pqn->kq) {
0948             q = pqn->kq->queue;
0949             mqd_mgr = pqn->kq->mqd_mgr;
0950             switch (q->properties.type) {
0951             case KFD_QUEUE_TYPE_DIQ:
0952                 seq_printf(m, "  DIQ on device %x\n",
0953                        pqn->kq->dev->id);
0954                 break;
0955             default:
0956                 seq_printf(m,
0957                 "  Bad kernel queue type %d on device %x\n",
0958                        q->properties.type,
0959                        pqn->kq->dev->id);
0960                 continue;
0961             }
0962         } else {
0963             seq_printf(m,
0964         "  Weird: Queue node with neither kernel nor user queue\n");
0965             continue;
0966         }
0967 
0968         r = mqd_mgr->debugfs_show_mqd(m, q->mqd);
0969         if (r != 0)
0970             break;
0971     }
0972 
0973     return r;
0974 }
0975 
0976 #endif
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!