amd/amdgpu/mxgpu_nv.c

0001 /*
0002  * Copyright 2014 Advanced Micro Devices, Inc.
0003  *
0004  * Permission is hereby granted, free of charge, to any person obtaining a
0005  * copy of this software and associated documentation files (the "Software"),
0006  * to deal in the Software without restriction, including without limitation
0007  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0008  * and/or sell copies of the Software, and to permit persons to whom the
0009  * Software is furnished to do so, subject to the following conditions:
0010  *
0011  * The above copyright notice and this permission notice shall be included in
0012  * all copies or substantial portions of the Software.
0013  *
0014  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0015  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0016  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0017  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
0018  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
0019  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
0020  * OTHER DEALINGS IN THE SOFTWARE.
0021  *
0022  */
0023
0024 #include "amdgpu.h"
0025 #include "nbio/nbio_2_3_offset.h"
0026 #include "nbio/nbio_2_3_sh_mask.h"
0027 #include "gc/gc_10_1_0_offset.h"
0028 #include "gc/gc_10_1_0_sh_mask.h"
0029 #include "soc15.h"
0030 #include "navi10_ih.h"
0031 #include "soc15_common.h"
0032 #include "mxgpu_nv.h"
0033
0034 #include "amdgpu_reset.h"
0035
0036 static void xgpu_nv_mailbox_send_ack(struct amdgpu_device *adev)
0037 {
0038     WREG8(NV_MAIBOX_CONTROL_RCV_OFFSET_BYTE, 2);
0039 }
0040
0041 static void xgpu_nv_mailbox_set_valid(struct amdgpu_device *adev, bool val)
0042 {
0043     WREG8(NV_MAIBOX_CONTROL_TRN_OFFSET_BYTE, val ? 1 : 0);
0044 }
0045
0046 /*
0047  * this peek_msg could *only* be called in IRQ routine becuase in IRQ routine
0048  * RCV_MSG_VALID filed of BIF_BX_PF_MAILBOX_CONTROL must already be set to 1
0049  * by host.
0050  *
0051  * if called no in IRQ routine, this peek_msg cannot guaranteed to return the
0052  * correct value since it doesn't return the RCV_DW0 under the case that
0053  * RCV_MSG_VALID is set by host.
0054  */
0055 static enum idh_event xgpu_nv_mailbox_peek_msg(struct amdgpu_device *adev)
0056 {
0057     return RREG32_NO_KIQ(mmMAILBOX_MSGBUF_RCV_DW0);
0058 }
0059
0060
0061 static int xgpu_nv_mailbox_rcv_msg(struct amdgpu_device *adev,
0062                    enum idh_event event)
0063 {
0064     u32 reg;
0065
0066     reg = RREG32_NO_KIQ(mmMAILBOX_MSGBUF_RCV_DW0);
0067     if (reg != event)
0068         return -ENOENT;
0069
0070     xgpu_nv_mailbox_send_ack(adev);
0071
0072     return 0;
0073 }
0074
0075 static uint8_t xgpu_nv_peek_ack(struct amdgpu_device *adev)
0076 {
0077     return RREG8(NV_MAIBOX_CONTROL_TRN_OFFSET_BYTE) & 2;
0078 }
0079
0080 static int xgpu_nv_poll_ack(struct amdgpu_device *adev)
0081 {
0082     int timeout  = NV_MAILBOX_POLL_ACK_TIMEDOUT;
0083     u8 reg;
0084
0085     do {
0086         reg = RREG8(NV_MAIBOX_CONTROL_TRN_OFFSET_BYTE);
0087         if (reg & 2)
0088             return 0;
0089
0090         mdelay(5);
0091         timeout -= 5;
0092     } while (timeout > 1);
0093
0094     pr_err("Doesn't get TRN_MSG_ACK from pf in %d msec\n", NV_MAILBOX_POLL_ACK_TIMEDOUT);
0095
0096     return -ETIME;
0097 }
0098
0099 static int xgpu_nv_poll_msg(struct amdgpu_device *adev, enum idh_event event)
0100 {
0101     int r;
0102     uint64_t timeout, now;
0103
0104     now = (uint64_t)ktime_to_ms(ktime_get());
0105     timeout = now + NV_MAILBOX_POLL_MSG_TIMEDOUT;
0106
0107     do {
0108         r = xgpu_nv_mailbox_rcv_msg(adev, event);
0109         if (!r)
0110             return 0;
0111
0112         msleep(10);
0113         now = (uint64_t)ktime_to_ms(ktime_get());
0114     } while (timeout > now);
0115
0116
0117     return -ETIME;
0118 }
0119
0120 static void xgpu_nv_mailbox_trans_msg (struct amdgpu_device *adev,
0121           enum idh_request req, u32 data1, u32 data2, u32 data3)
0122 {
0123     int r;
0124     uint8_t trn;
0125
0126     /* IMPORTANT:
0127      * clear TRN_MSG_VALID valid to clear host's RCV_MSG_ACK
0128      * and with host's RCV_MSG_ACK cleared hw automatically clear host's RCV_MSG_ACK
0129      * which lead to VF's TRN_MSG_ACK cleared, otherwise below xgpu_nv_poll_ack()
0130      * will return immediatly
0131      */
0132     do {
0133         xgpu_nv_mailbox_set_valid(adev, false);
0134         trn = xgpu_nv_peek_ack(adev);
0135         if (trn) {
0136             pr_err("trn=%x ACK should not assert! wait again !\n", trn);
0137             msleep(1);
0138         }
0139     } while (trn);
0140
0141     WREG32_NO_KIQ(mmMAILBOX_MSGBUF_TRN_DW0, req);
0142     WREG32_NO_KIQ(mmMAILBOX_MSGBUF_TRN_DW1, data1);
0143     WREG32_NO_KIQ(mmMAILBOX_MSGBUF_TRN_DW2, data2);
0144     WREG32_NO_KIQ(mmMAILBOX_MSGBUF_TRN_DW3, data3);
0145     xgpu_nv_mailbox_set_valid(adev, true);
0146
0147     /* start to poll ack */
0148     r = xgpu_nv_poll_ack(adev);
0149     if (r)
0150         pr_err("Doesn't get ack from pf, continue\n");
0151
0152     xgpu_nv_mailbox_set_valid(adev, false);
0153 }
0154
0155 static int xgpu_nv_send_access_requests(struct amdgpu_device *adev,
0156                     enum idh_request req)
0157 {
0158     int r, retry = 1;
0159     enum idh_event event = -1;
0160
0161 send_request:
0162     xgpu_nv_mailbox_trans_msg(adev, req, 0, 0, 0);
0163
0164     switch (req) {
0165     case IDH_REQ_GPU_INIT_ACCESS:
0166     case IDH_REQ_GPU_FINI_ACCESS:
0167     case IDH_REQ_GPU_RESET_ACCESS:
0168         event = IDH_READY_TO_ACCESS_GPU;
0169         break;
0170     case IDH_REQ_GPU_INIT_DATA:
0171         event = IDH_REQ_GPU_INIT_DATA_READY;
0172         break;
0173     default:
0174         break;
0175     }
0176
0177     if (event != -1) {
0178         r = xgpu_nv_poll_msg(adev, event);
0179         if (r) {
0180             if (retry++ < 2)
0181                 goto send_request;
0182
0183             if (req != IDH_REQ_GPU_INIT_DATA) {
0184                 pr_err("Doesn't get msg:%d from pf, error=%d\n", event, r);
0185                 return r;
0186             }
0187             else /* host doesn't support REQ_GPU_INIT_DATA handshake */
0188                 adev->virt.req_init_data_ver = 0;
0189         } else {
0190             if (req == IDH_REQ_GPU_INIT_DATA)
0191             {
0192                 adev->virt.req_init_data_ver =
0193                     RREG32_NO_KIQ(mmMAILBOX_MSGBUF_RCV_DW1);
0194
0195                 /* assume V1 in case host doesn't set version number */
0196                 if (adev->virt.req_init_data_ver < 1)
0197                     adev->virt.req_init_data_ver = 1;
0198             }
0199         }
0200
0201         /* Retrieve checksum from mailbox2 */
0202         if (req == IDH_REQ_GPU_INIT_ACCESS || req == IDH_REQ_GPU_RESET_ACCESS) {
0203             adev->virt.fw_reserve.checksum_key =
0204                 RREG32_NO_KIQ(mmMAILBOX_MSGBUF_RCV_DW2);
0205         }
0206     }
0207
0208     return 0;
0209 }
0210
0211 static int xgpu_nv_request_reset(struct amdgpu_device *adev)
0212 {
0213     int ret, i = 0;
0214
0215     while (i < NV_MAILBOX_POLL_MSG_REP_MAX) {
0216         ret = xgpu_nv_send_access_requests(adev, IDH_REQ_GPU_RESET_ACCESS);
0217         if (!ret)
0218             break;
0219         i++;
0220     }
0221
0222     return ret;
0223 }
0224
0225 static int xgpu_nv_request_full_gpu_access(struct amdgpu_device *adev,
0226                        bool init)
0227 {
0228     enum idh_request req;
0229
0230     req = init ? IDH_REQ_GPU_INIT_ACCESS : IDH_REQ_GPU_FINI_ACCESS;
0231     return xgpu_nv_send_access_requests(adev, req);
0232 }
0233
0234 static int xgpu_nv_release_full_gpu_access(struct amdgpu_device *adev,
0235                        bool init)
0236 {
0237     enum idh_request req;
0238     int r = 0;
0239
0240     req = init ? IDH_REL_GPU_INIT_ACCESS : IDH_REL_GPU_FINI_ACCESS;
0241     r = xgpu_nv_send_access_requests(adev, req);
0242
0243     return r;
0244 }
0245
0246 static int xgpu_nv_request_init_data(struct amdgpu_device *adev)
0247 {
0248     return xgpu_nv_send_access_requests(adev, IDH_REQ_GPU_INIT_DATA);
0249 }
0250
0251 static int xgpu_nv_mailbox_ack_irq(struct amdgpu_device *adev,
0252                     struct amdgpu_irq_src *source,
0253                     struct amdgpu_iv_entry *entry)
0254 {
0255     DRM_DEBUG("get ack intr and do nothing.\n");
0256     return 0;
0257 }
0258
0259 static int xgpu_nv_set_mailbox_ack_irq(struct amdgpu_device *adev,
0260                     struct amdgpu_irq_src *source,
0261                     unsigned type,
0262                     enum amdgpu_interrupt_state state)
0263 {
0264     u32 tmp = RREG32_NO_KIQ(mmMAILBOX_INT_CNTL);
0265
0266     if (state == AMDGPU_IRQ_STATE_ENABLE)
0267         tmp |= 2;
0268     else
0269         tmp &= ~2;
0270
0271     WREG32_NO_KIQ(mmMAILBOX_INT_CNTL, tmp);
0272
0273     return 0;
0274 }
0275
0276 static void xgpu_nv_mailbox_flr_work(struct work_struct *work)
0277 {
0278     struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work);
0279     struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt);
0280     int timeout = NV_MAILBOX_POLL_FLR_TIMEDOUT;
0281
0282     /* block amdgpu_gpu_recover till msg FLR COMPLETE received,
0283      * otherwise the mailbox msg will be ruined/reseted by
0284      * the VF FLR.
0285      */
0286     if (atomic_cmpxchg(&adev->reset_domain->in_gpu_reset, 0, 1) != 0)
0287         return;
0288
0289     down_write(&adev->reset_domain->sem);
0290
0291     amdgpu_virt_fini_data_exchange(adev);
0292
0293     xgpu_nv_mailbox_trans_msg(adev, IDH_READY_TO_RESET, 0, 0, 0);
0294
0295     do {
0296         if (xgpu_nv_mailbox_peek_msg(adev) == IDH_FLR_NOTIFICATION_CMPL)
0297             goto flr_done;
0298
0299         msleep(10);
0300         timeout -= 10;
0301     } while (timeout > 1);
0302
0303 flr_done:
0304     atomic_set(&adev->reset_domain->in_gpu_reset, 0);
0305     up_write(&adev->reset_domain->sem);
0306
0307     /* Trigger recovery for world switch failure if no TDR */
0308     if (amdgpu_device_should_recover_gpu(adev)
0309         && (!amdgpu_device_has_job_running(adev) ||
0310         adev->sdma_timeout == MAX_SCHEDULE_TIMEOUT ||
0311         adev->gfx_timeout == MAX_SCHEDULE_TIMEOUT ||
0312         adev->compute_timeout == MAX_SCHEDULE_TIMEOUT ||
0313         adev->video_timeout == MAX_SCHEDULE_TIMEOUT)) {
0314         struct amdgpu_reset_context reset_context;
0315         memset(&reset_context, 0, sizeof(reset_context));
0316
0317         reset_context.method = AMD_RESET_METHOD_NONE;
0318         reset_context.reset_req_dev = adev;
0319         clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
0320
0321         amdgpu_device_gpu_recover(adev, NULL, &reset_context);
0322     }
0323 }
0324
0325 static int xgpu_nv_set_mailbox_rcv_irq(struct amdgpu_device *adev,
0326                        struct amdgpu_irq_src *src,
0327                        unsigned type,
0328                        enum amdgpu_interrupt_state state)
0329 {
0330     u32 tmp = RREG32_NO_KIQ(mmMAILBOX_INT_CNTL);
0331
0332     if (state == AMDGPU_IRQ_STATE_ENABLE)
0333         tmp |= 1;
0334     else
0335         tmp &= ~1;
0336
0337     WREG32_NO_KIQ(mmMAILBOX_INT_CNTL, tmp);
0338
0339     return 0;
0340 }
0341
0342 static int xgpu_nv_mailbox_rcv_irq(struct amdgpu_device *adev,
0343                    struct amdgpu_irq_src *source,
0344                    struct amdgpu_iv_entry *entry)
0345 {
0346     enum idh_event event = xgpu_nv_mailbox_peek_msg(adev);
0347
0348     switch (event) {
0349     case IDH_FLR_NOTIFICATION:
0350         if (amdgpu_sriov_runtime(adev) && !amdgpu_in_reset(adev))
0351             WARN_ONCE(!amdgpu_reset_domain_schedule(adev->reset_domain,
0352                    &adev->virt.flr_work),
0353                   "Failed to queue work! at %s",
0354                   __func__);
0355         break;
0356         /* READY_TO_ACCESS_GPU is fetched by kernel polling, IRQ can ignore
0357          * it byfar since that polling thread will handle it,
0358          * other msg like flr complete is not handled here.
0359          */
0360     case IDH_CLR_MSG_BUF:
0361     case IDH_FLR_NOTIFICATION_CMPL:
0362     case IDH_READY_TO_ACCESS_GPU:
0363     default:
0364         break;
0365     }
0366
0367     return 0;
0368 }
0369
0370 static const struct amdgpu_irq_src_funcs xgpu_nv_mailbox_ack_irq_funcs = {
0371     .set = xgpu_nv_set_mailbox_ack_irq,
0372     .process = xgpu_nv_mailbox_ack_irq,
0373 };
0374
0375 static const struct amdgpu_irq_src_funcs xgpu_nv_mailbox_rcv_irq_funcs = {
0376     .set = xgpu_nv_set_mailbox_rcv_irq,
0377     .process = xgpu_nv_mailbox_rcv_irq,
0378 };
0379
0380 void xgpu_nv_mailbox_set_irq_funcs(struct amdgpu_device *adev)
0381 {
0382     adev->virt.ack_irq.num_types = 1;
0383     adev->virt.ack_irq.funcs = &xgpu_nv_mailbox_ack_irq_funcs;
0384     adev->virt.rcv_irq.num_types = 1;
0385     adev->virt.rcv_irq.funcs = &xgpu_nv_mailbox_rcv_irq_funcs;
0386 }
0387
0388 int xgpu_nv_mailbox_add_irq_id(struct amdgpu_device *adev)
0389 {
0390     int r;
0391
0392     r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_BIF, 135, &adev->virt.rcv_irq);
0393     if (r)
0394         return r;
0395
0396     r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_BIF, 138, &adev->virt.ack_irq);
0397     if (r) {
0398         amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
0399         return r;
0400     }
0401
0402     return 0;
0403 }
0404
0405 int xgpu_nv_mailbox_get_irq(struct amdgpu_device *adev)
0406 {
0407     int r;
0408
0409     r = amdgpu_irq_get(adev, &adev->virt.rcv_irq, 0);
0410     if (r)
0411         return r;
0412     r = amdgpu_irq_get(adev, &adev->virt.ack_irq, 0);
0413     if (r) {
0414         amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
0415         return r;
0416     }
0417
0418     INIT_WORK(&adev->virt.flr_work, xgpu_nv_mailbox_flr_work);
0419
0420     return 0;
0421 }
0422
0423 void xgpu_nv_mailbox_put_irq(struct amdgpu_device *adev)
0424 {
0425     amdgpu_irq_put(adev, &adev->virt.ack_irq, 0);
0426     amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
0427 }
0428
0429 const struct amdgpu_virt_ops xgpu_nv_virt_ops = {
0430     .req_full_gpu   = xgpu_nv_request_full_gpu_access,
0431     .rel_full_gpu   = xgpu_nv_release_full_gpu_access,
0432     .req_init_data  = xgpu_nv_request_init_data,
0433     .reset_gpu = xgpu_nv_request_reset,
0434     .wait_reset = NULL,
0435     .trans_msg = xgpu_nv_mailbox_trans_msg,
0436 };