amd/amdkfd/kfd_mqd_manager_cik.c

0001 // SPDX-License-Identifier: GPL-2.0 OR MIT
0002 /*
0003  * Copyright 2014-2022 Advanced Micro Devices, Inc.
0004  *
0005  * Permission is hereby granted, free of charge, to any person obtaining a
0006  * copy of this software and associated documentation files (the "Software"),
0007  * to deal in the Software without restriction, including without limitation
0008  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0009  * and/or sell copies of the Software, and to permit persons to whom the
0010  * Software is furnished to do so, subject to the following conditions:
0011  *
0012  * The above copyright notice and this permission notice shall be included in
0013  * all copies or substantial portions of the Software.
0014  *
0015  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0016  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0017  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0018  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
0019  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
0020  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
0021  * OTHER DEALINGS IN THE SOFTWARE.
0022  *
0023  */
0024
0025 #include <linux/printk.h>
0026 #include <linux/slab.h>
0027 #include <linux/mm_types.h>
0028
0029 #include "kfd_priv.h"
0030 #include "kfd_mqd_manager.h"
0031 #include "cik_regs.h"
0032 #include "cik_structs.h"
0033 #include "oss/oss_2_4_sh_mask.h"
0034
0035 static inline struct cik_mqd *get_mqd(void *mqd)
0036 {
0037     return (struct cik_mqd *)mqd;
0038 }
0039
0040 static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
0041 {
0042     return (struct cik_sdma_rlc_registers *)mqd;
0043 }
0044
0045 static void update_cu_mask(struct mqd_manager *mm, void *mqd,
0046             struct mqd_update_info *minfo)
0047 {
0048     struct cik_mqd *m;
0049     uint32_t se_mask[4] = {0}; /* 4 is the max # of SEs */
0050
0051     if (!minfo || (minfo->update_flag != UPDATE_FLAG_CU_MASK) ||
0052         !minfo->cu_mask.ptr)
0053         return;
0054
0055     mqd_symmetrically_map_cu_mask(mm,
0056         minfo->cu_mask.ptr, minfo->cu_mask.count, se_mask);
0057
0058     m = get_mqd(mqd);
0059     m->compute_static_thread_mgmt_se0 = se_mask[0];
0060     m->compute_static_thread_mgmt_se1 = se_mask[1];
0061     m->compute_static_thread_mgmt_se2 = se_mask[2];
0062     m->compute_static_thread_mgmt_se3 = se_mask[3];
0063
0064     pr_debug("Update cu mask to %#x %#x %#x %#x\n",
0065         m->compute_static_thread_mgmt_se0,
0066         m->compute_static_thread_mgmt_se1,
0067         m->compute_static_thread_mgmt_se2,
0068         m->compute_static_thread_mgmt_se3);
0069 }
0070
0071 static void set_priority(struct cik_mqd *m, struct queue_properties *q)
0072 {
0073     m->cp_hqd_pipe_priority = pipe_priority_map[q->priority];
0074     m->cp_hqd_queue_priority = q->priority;
0075 }
0076
0077 static struct kfd_mem_obj *allocate_mqd(struct kfd_dev *kfd,
0078                     struct queue_properties *q)
0079 {
0080     struct kfd_mem_obj *mqd_mem_obj;
0081
0082     if (kfd_gtt_sa_allocate(kfd, sizeof(struct cik_mqd),
0083             &mqd_mem_obj))
0084         return NULL;
0085
0086     return mqd_mem_obj;
0087 }
0088
0089 static void init_mqd(struct mqd_manager *mm, void **mqd,
0090         struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
0091         struct queue_properties *q)
0092 {
0093     uint64_t addr;
0094     struct cik_mqd *m;
0095
0096     m = (struct cik_mqd *) mqd_mem_obj->cpu_ptr;
0097     addr = mqd_mem_obj->gpu_addr;
0098
0099     memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256));
0100
0101     m->header = 0xC0310800;
0102     m->compute_pipelinestat_enable = 1;
0103     m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF;
0104     m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF;
0105     m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF;
0106     m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF;
0107
0108     /*
0109      * Make sure to use the last queue state saved on mqd when the cp
0110      * reassigns the queue, so when queue is switched on/off (e.g over
0111      * subscription or quantum timeout) the context will be consistent
0112      */
0113     m->cp_hqd_persistent_state =
0114                 DEFAULT_CP_HQD_PERSISTENT_STATE | PRELOAD_REQ;
0115
0116     m->cp_mqd_control             = MQD_CONTROL_PRIV_STATE_EN;
0117     m->cp_mqd_base_addr_lo        = lower_32_bits(addr);
0118     m->cp_mqd_base_addr_hi        = upper_32_bits(addr);
0119
0120     m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS |
0121                 QUANTUM_DURATION(10);
0122
0123     /*
0124      * Pipe Priority
0125      * Identifies the pipe relative priority when this queue is connected
0126      * to the pipeline. The pipe priority is against the GFX pipe and HP3D.
0127      * In KFD we are using a fixed pipe priority set to CS_MEDIUM.
0128      * 0 = CS_LOW (typically below GFX)
0129      * 1 = CS_MEDIUM (typically between HP3D and GFX
0130      * 2 = CS_HIGH (typically above HP3D)
0131      */
0132     set_priority(m, q);
0133
0134     if (q->format == KFD_QUEUE_FORMAT_AQL)
0135         m->cp_hqd_iq_rptr = AQL_ENABLE;
0136
0137     *mqd = m;
0138     if (gart_addr)
0139         *gart_addr = addr;
0140     mm->update_mqd(mm, m, q, NULL);
0141 }
0142
0143 static void init_mqd_sdma(struct mqd_manager *mm, void **mqd,
0144             struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
0145             struct queue_properties *q)
0146 {
0147     struct cik_sdma_rlc_registers *m;
0148
0149     m = (struct cik_sdma_rlc_registers *) mqd_mem_obj->cpu_ptr;
0150
0151     memset(m, 0, sizeof(struct cik_sdma_rlc_registers));
0152
0153     *mqd = m;
0154     if (gart_addr)
0155         *gart_addr = mqd_mem_obj->gpu_addr;
0156
0157     mm->update_mqd(mm, m, q, NULL);
0158 }
0159
0160 static int load_mqd(struct mqd_manager *mm, void *mqd, uint32_t pipe_id,
0161             uint32_t queue_id, struct queue_properties *p,
0162             struct mm_struct *mms)
0163 {
0164     /* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */
0165     uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0);
0166     uint32_t wptr_mask = (uint32_t)((p->queue_size / 4) - 1);
0167
0168     return mm->dev->kfd2kgd->hqd_load(mm->dev->adev, mqd, pipe_id, queue_id,
0169                       (uint32_t __user *)p->write_ptr,
0170                       wptr_shift, wptr_mask, mms);
0171 }
0172
0173 static void __update_mqd(struct mqd_manager *mm, void *mqd,
0174             struct queue_properties *q, struct mqd_update_info *minfo,
0175             unsigned int atc_bit)
0176 {
0177     struct cik_mqd *m;
0178
0179     m = get_mqd(mqd);
0180     m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE |
0181                 DEFAULT_MIN_AVAIL_SIZE;
0182     m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE;
0183     if (atc_bit) {
0184         m->cp_hqd_pq_control |= PQ_ATC_EN;
0185         m->cp_hqd_ib_control |= IB_ATC_EN;
0186     }
0187
0188     /*
0189      * Calculating queue size which is log base 2 of actual queue size -1
0190      * dwords and another -1 for ffs
0191      */
0192     m->cp_hqd_pq_control |= order_base_2(q->queue_size / 4) - 1;
0193     m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
0194     m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
0195     m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
0196     m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
0197     m->cp_hqd_pq_doorbell_control = DOORBELL_OFFSET(q->doorbell_off);
0198
0199     m->cp_hqd_vmid = q->vmid;
0200
0201     if (q->format == KFD_QUEUE_FORMAT_AQL)
0202         m->cp_hqd_pq_control |= NO_UPDATE_RPTR;
0203
0204     update_cu_mask(mm, mqd, minfo);
0205     set_priority(m, q);
0206
0207     q->is_active = QUEUE_IS_ACTIVE(*q);
0208 }
0209
0210 static void update_mqd(struct mqd_manager *mm, void *mqd,
0211             struct queue_properties *q,
0212             struct mqd_update_info *minfo)
0213 {
0214     __update_mqd(mm, mqd, q, minfo, 1);
0215 }
0216
0217 static uint32_t read_doorbell_id(void *mqd)
0218 {
0219     struct cik_mqd *m = (struct cik_mqd *)mqd;
0220
0221     return m->queue_doorbell_id0;
0222 }
0223
0224 static void update_mqd_hawaii(struct mqd_manager *mm, void *mqd,
0225             struct queue_properties *q,
0226             struct mqd_update_info *minfo)
0227 {
0228     __update_mqd(mm, mqd, q, minfo, 0);
0229 }
0230
0231 static void update_mqd_sdma(struct mqd_manager *mm, void *mqd,
0232             struct queue_properties *q,
0233             struct mqd_update_info *minfo)
0234 {
0235     struct cik_sdma_rlc_registers *m;
0236
0237     m = get_sdma_mqd(mqd);
0238     m->sdma_rlc_rb_cntl = order_base_2(q->queue_size / 4)
0239             << SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT |
0240             q->vmid << SDMA0_RLC0_RB_CNTL__RB_VMID__SHIFT |
0241             1 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
0242             6 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT;
0243
0244     m->sdma_rlc_rb_base = lower_32_bits(q->queue_address >> 8);
0245     m->sdma_rlc_rb_base_hi = upper_32_bits(q->queue_address >> 8);
0246     m->sdma_rlc_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
0247     m->sdma_rlc_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
0248     m->sdma_rlc_doorbell =
0249         q->doorbell_off << SDMA0_RLC0_DOORBELL__OFFSET__SHIFT;
0250
0251     m->sdma_rlc_virtual_addr = q->sdma_vm_addr;
0252
0253     m->sdma_engine_id = q->sdma_engine_id;
0254     m->sdma_queue_id = q->sdma_queue_id;
0255
0256     q->is_active = QUEUE_IS_ACTIVE(*q);
0257 }
0258
0259 static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst, void *ctl_stack_dst)
0260 {
0261     struct cik_mqd *m;
0262
0263     m = get_mqd(mqd);
0264
0265     memcpy(mqd_dst, m, sizeof(struct cik_mqd));
0266 }
0267
0268 static void restore_mqd(struct mqd_manager *mm, void **mqd,
0269             struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
0270             struct queue_properties *qp,
0271             const void *mqd_src,
0272             const void *ctl_stack_src, const u32 ctl_stack_size)
0273 {
0274     uint64_t addr;
0275     struct cik_mqd *m;
0276
0277     m = (struct cik_mqd *) mqd_mem_obj->cpu_ptr;
0278     addr = mqd_mem_obj->gpu_addr;
0279
0280     memcpy(m, mqd_src, sizeof(*m));
0281
0282     *mqd = m;
0283     if (gart_addr)
0284         *gart_addr = addr;
0285
0286     m->cp_hqd_pq_doorbell_control = DOORBELL_OFFSET(qp->doorbell_off);
0287
0288     pr_debug("cp_hqd_pq_doorbell_control 0x%x\n",
0289             m->cp_hqd_pq_doorbell_control);
0290
0291     qp->is_active = 0;
0292 }
0293
0294 static void checkpoint_mqd_sdma(struct mqd_manager *mm,
0295                 void *mqd,
0296                 void *mqd_dst,
0297                 void *ctl_stack_dst)
0298 {
0299     struct cik_sdma_rlc_registers *m;
0300
0301     m = get_sdma_mqd(mqd);
0302
0303     memcpy(mqd_dst, m, sizeof(struct cik_sdma_rlc_registers));
0304 }
0305
0306 static void restore_mqd_sdma(struct mqd_manager *mm, void **mqd,
0307                 struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
0308                 struct queue_properties *qp,
0309                 const void *mqd_src,
0310                 const void *ctl_stack_src, const u32 ctl_stack_size)
0311 {
0312     uint64_t addr;
0313     struct cik_sdma_rlc_registers *m;
0314
0315     m = (struct cik_sdma_rlc_registers *) mqd_mem_obj->cpu_ptr;
0316     addr = mqd_mem_obj->gpu_addr;
0317
0318     memcpy(m, mqd_src, sizeof(*m));
0319
0320     m->sdma_rlc_doorbell =
0321         qp->doorbell_off << SDMA0_RLC0_DOORBELL__OFFSET__SHIFT;
0322
0323     *mqd = m;
0324     if (gart_addr)
0325         *gart_addr = addr;
0326
0327     qp->is_active = 0;
0328 }
0329
0330 /*
0331  * HIQ MQD Implementation, concrete implementation for HIQ MQD implementation.
0332  * The HIQ queue in Kaveri is using the same MQD structure as all the user mode
0333  * queues but with different initial values.
0334  */
0335
0336 static void init_mqd_hiq(struct mqd_manager *mm, void **mqd,
0337         struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
0338         struct queue_properties *q)
0339 {
0340     init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q);
0341 }
0342
0343 static void update_mqd_hiq(struct mqd_manager *mm, void *mqd,
0344             struct queue_properties *q,
0345             struct mqd_update_info *minfo)
0346 {
0347     struct cik_mqd *m;
0348
0349     m = get_mqd(mqd);
0350     m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE |
0351                 DEFAULT_MIN_AVAIL_SIZE |
0352                 PRIV_STATE |
0353                 KMD_QUEUE;
0354
0355     /*
0356      * Calculating queue size which is log base 2 of actual queue
0357      * size -1 dwords
0358      */
0359     m->cp_hqd_pq_control |= order_base_2(q->queue_size / 4) - 1;
0360     m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
0361     m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
0362     m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
0363     m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
0364     m->cp_hqd_pq_doorbell_control = DOORBELL_OFFSET(q->doorbell_off);
0365
0366     m->cp_hqd_vmid = q->vmid;
0367
0368     q->is_active = QUEUE_IS_ACTIVE(*q);
0369
0370     set_priority(m, q);
0371 }
0372
0373 #if defined(CONFIG_DEBUG_FS)
0374
0375 static int debugfs_show_mqd(struct seq_file *m, void *data)
0376 {
0377     seq_hex_dump(m, "    ", DUMP_PREFIX_OFFSET, 32, 4,
0378              data, sizeof(struct cik_mqd), false);
0379     return 0;
0380 }
0381
0382 static int debugfs_show_mqd_sdma(struct seq_file *m, void *data)
0383 {
0384     seq_hex_dump(m, "    ", DUMP_PREFIX_OFFSET, 32, 4,
0385              data, sizeof(struct cik_sdma_rlc_registers), false);
0386     return 0;
0387 }
0388
0389 #endif
0390
0391
0392 struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
0393         struct kfd_dev *dev)
0394 {
0395     struct mqd_manager *mqd;
0396
0397     if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
0398         return NULL;
0399
0400     mqd = kzalloc(sizeof(*mqd), GFP_KERNEL);
0401     if (!mqd)
0402         return NULL;
0403
0404     mqd->dev = dev;
0405
0406     switch (type) {
0407     case KFD_MQD_TYPE_CP:
0408         mqd->allocate_mqd = allocate_mqd;
0409         mqd->init_mqd = init_mqd;
0410         mqd->free_mqd = kfd_free_mqd_cp;
0411         mqd->load_mqd = load_mqd;
0412         mqd->update_mqd = update_mqd;
0413         mqd->destroy_mqd = kfd_destroy_mqd_cp;
0414         mqd->is_occupied = kfd_is_occupied_cp;
0415         mqd->checkpoint_mqd = checkpoint_mqd;
0416         mqd->restore_mqd = restore_mqd;
0417         mqd->mqd_size = sizeof(struct cik_mqd);
0418 #if defined(CONFIG_DEBUG_FS)
0419         mqd->debugfs_show_mqd = debugfs_show_mqd;
0420 #endif
0421         break;
0422     case KFD_MQD_TYPE_HIQ:
0423         mqd->allocate_mqd = allocate_hiq_mqd;
0424         mqd->init_mqd = init_mqd_hiq;
0425         mqd->free_mqd = free_mqd_hiq_sdma;
0426         mqd->load_mqd = load_mqd;
0427         mqd->update_mqd = update_mqd_hiq;
0428         mqd->destroy_mqd = kfd_destroy_mqd_cp;
0429         mqd->is_occupied = kfd_is_occupied_cp;
0430         mqd->mqd_size = sizeof(struct cik_mqd);
0431 #if defined(CONFIG_DEBUG_FS)
0432         mqd->debugfs_show_mqd = debugfs_show_mqd;
0433 #endif
0434         mqd->read_doorbell_id = read_doorbell_id;
0435         break;
0436     case KFD_MQD_TYPE_DIQ:
0437         mqd->allocate_mqd = allocate_mqd;
0438         mqd->init_mqd = init_mqd_hiq;
0439         mqd->free_mqd = kfd_free_mqd_cp;
0440         mqd->load_mqd = load_mqd;
0441         mqd->update_mqd = update_mqd_hiq;
0442         mqd->destroy_mqd = kfd_destroy_mqd_cp;
0443         mqd->is_occupied = kfd_is_occupied_cp;
0444         mqd->mqd_size = sizeof(struct cik_mqd);
0445 #if defined(CONFIG_DEBUG_FS)
0446         mqd->debugfs_show_mqd = debugfs_show_mqd;
0447 #endif
0448         break;
0449     case KFD_MQD_TYPE_SDMA:
0450         mqd->allocate_mqd = allocate_sdma_mqd;
0451         mqd->init_mqd = init_mqd_sdma;
0452         mqd->free_mqd = free_mqd_hiq_sdma;
0453         mqd->load_mqd = kfd_load_mqd_sdma;
0454         mqd->update_mqd = update_mqd_sdma;
0455         mqd->destroy_mqd = kfd_destroy_mqd_sdma;
0456         mqd->is_occupied = kfd_is_occupied_sdma;
0457         mqd->checkpoint_mqd = checkpoint_mqd_sdma;
0458         mqd->restore_mqd = restore_mqd_sdma;
0459         mqd->mqd_size = sizeof(struct cik_sdma_rlc_registers);
0460 #if defined(CONFIG_DEBUG_FS)
0461         mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;
0462 #endif
0463         break;
0464     default:
0465         kfree(mqd);
0466         return NULL;
0467     }
0468
0469     return mqd;
0470 }
0471
0472 struct mqd_manager *mqd_manager_init_cik_hawaii(enum KFD_MQD_TYPE type,
0473             struct kfd_dev *dev)
0474 {
0475     struct mqd_manager *mqd;
0476
0477     mqd = mqd_manager_init_cik(type, dev);
0478     if (!mqd)
0479         return NULL;
0480     if (type == KFD_MQD_TYPE_CP)
0481         mqd->update_mqd = update_mqd_hawaii;
0482     return mqd;
0483 }