amd/amdgpu/amdgpu_doorbell.h

0001 /*
0002  * Copyright 2018 Advanced Micro Devices, Inc.
0003  *
0004  * Permission is hereby granted, free of charge, to any person obtaining a
0005  * copy of this software and associated documentation files (the "Software"),
0006  * to deal in the Software without restriction, including without limitation
0007  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0008  * and/or sell copies of the Software, and to permit persons to whom the
0009  * Software is furnished to do so, subject to the following conditions:
0010  *
0011  * The above copyright notice and this permission notice shall be included in
0012  * all copies or substantial portions of the Software.
0013  *
0014  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0015  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0016  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0017  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
0018  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
0019  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
0020  * OTHER DEALINGS IN THE SOFTWARE.
0021  *
0022  */
0023
0024 /*
0025  * GPU doorbell structures, functions & helpers
0026  */
0027 struct amdgpu_doorbell {
0028     /* doorbell mmio */
0029     resource_size_t     base;
0030     resource_size_t     size;
0031     u32 __iomem     *ptr;
0032     u32         num_doorbells;  /* Number of doorbells actually reserved for amdgpu. */
0033 };
0034
0035 /* Reserved doorbells for amdgpu (including multimedia).
0036  * KFD can use all the rest in the 2M doorbell bar.
0037  * For asic before vega10, doorbell is 32-bit, so the
0038  * index/offset is in dword. For vega10 and after, doorbell
0039  * can be 64-bit, so the index defined is in qword.
0040  */
0041 struct amdgpu_doorbell_index {
0042     uint32_t kiq;
0043     uint32_t mec_ring0;
0044     uint32_t mec_ring1;
0045     uint32_t mec_ring2;
0046     uint32_t mec_ring3;
0047     uint32_t mec_ring4;
0048     uint32_t mec_ring5;
0049     uint32_t mec_ring6;
0050     uint32_t mec_ring7;
0051     uint32_t userqueue_start;
0052     uint32_t userqueue_end;
0053     uint32_t gfx_ring0;
0054     uint32_t gfx_ring1;
0055     uint32_t gfx_userqueue_start;
0056     uint32_t gfx_userqueue_end;
0057     uint32_t sdma_engine[8];
0058     uint32_t mes_ring0;
0059     uint32_t mes_ring1;
0060     uint32_t ih;
0061     union {
0062         struct {
0063             uint32_t vcn_ring0_1;
0064             uint32_t vcn_ring2_3;
0065             uint32_t vcn_ring4_5;
0066             uint32_t vcn_ring6_7;
0067         } vcn;
0068         struct {
0069             uint32_t uvd_ring0_1;
0070             uint32_t uvd_ring2_3;
0071             uint32_t uvd_ring4_5;
0072             uint32_t uvd_ring6_7;
0073             uint32_t vce_ring0_1;
0074             uint32_t vce_ring2_3;
0075             uint32_t vce_ring4_5;
0076             uint32_t vce_ring6_7;
0077         } uvd_vce;
0078     };
0079     uint32_t first_non_cp;
0080     uint32_t last_non_cp;
0081     uint32_t max_assignment;
0082     /* Per engine SDMA doorbell size in dword */
0083     uint32_t sdma_doorbell_range;
0084 };
0085
0086 typedef enum _AMDGPU_DOORBELL_ASSIGNMENT
0087 {
0088     AMDGPU_DOORBELL_KIQ                     = 0x000,
0089     AMDGPU_DOORBELL_HIQ                     = 0x001,
0090     AMDGPU_DOORBELL_DIQ                     = 0x002,
0091     AMDGPU_DOORBELL_MEC_RING0               = 0x010,
0092     AMDGPU_DOORBELL_MEC_RING1               = 0x011,
0093     AMDGPU_DOORBELL_MEC_RING2               = 0x012,
0094     AMDGPU_DOORBELL_MEC_RING3               = 0x013,
0095     AMDGPU_DOORBELL_MEC_RING4               = 0x014,
0096     AMDGPU_DOORBELL_MEC_RING5               = 0x015,
0097     AMDGPU_DOORBELL_MEC_RING6               = 0x016,
0098     AMDGPU_DOORBELL_MEC_RING7               = 0x017,
0099     AMDGPU_DOORBELL_GFX_RING0               = 0x020,
0100     AMDGPU_DOORBELL_sDMA_ENGINE0            = 0x1E0,
0101     AMDGPU_DOORBELL_sDMA_ENGINE1            = 0x1E1,
0102     AMDGPU_DOORBELL_IH                      = 0x1E8,
0103     AMDGPU_DOORBELL_MAX_ASSIGNMENT          = 0x3FF,
0104     AMDGPU_DOORBELL_INVALID                 = 0xFFFF
0105 } AMDGPU_DOORBELL_ASSIGNMENT;
0106
0107 typedef enum _AMDGPU_VEGA20_DOORBELL_ASSIGNMENT
0108 {
0109     /* Compute + GFX: 0~255 */
0110     AMDGPU_VEGA20_DOORBELL_KIQ                     = 0x000,
0111     AMDGPU_VEGA20_DOORBELL_HIQ                     = 0x001,
0112     AMDGPU_VEGA20_DOORBELL_DIQ                     = 0x002,
0113     AMDGPU_VEGA20_DOORBELL_MEC_RING0               = 0x003,
0114     AMDGPU_VEGA20_DOORBELL_MEC_RING1               = 0x004,
0115     AMDGPU_VEGA20_DOORBELL_MEC_RING2               = 0x005,
0116     AMDGPU_VEGA20_DOORBELL_MEC_RING3               = 0x006,
0117     AMDGPU_VEGA20_DOORBELL_MEC_RING4               = 0x007,
0118     AMDGPU_VEGA20_DOORBELL_MEC_RING5               = 0x008,
0119     AMDGPU_VEGA20_DOORBELL_MEC_RING6               = 0x009,
0120     AMDGPU_VEGA20_DOORBELL_MEC_RING7               = 0x00A,
0121     AMDGPU_VEGA20_DOORBELL_USERQUEUE_START         = 0x00B,
0122     AMDGPU_VEGA20_DOORBELL_USERQUEUE_END           = 0x08A,
0123     AMDGPU_VEGA20_DOORBELL_GFX_RING0               = 0x08B,
0124     /* SDMA:256~335*/
0125     AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE0            = 0x100,
0126     AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE1            = 0x10A,
0127     AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE2            = 0x114,
0128     AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE3            = 0x11E,
0129     AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE4            = 0x128,
0130     AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE5            = 0x132,
0131     AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE6            = 0x13C,
0132     AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE7            = 0x146,
0133     /* IH: 376~391 */
0134     AMDGPU_VEGA20_DOORBELL_IH                      = 0x178,
0135     /* MMSCH: 392~407
0136      * overlap the doorbell assignment with VCN as they are  mutually exclusive
0137      * VCN engine's doorbell is 32 bit and two VCN ring share one QWORD
0138      */
0139     AMDGPU_VEGA20_DOORBELL64_VCN0_1                  = 0x188, /* VNC0 */
0140     AMDGPU_VEGA20_DOORBELL64_VCN2_3                  = 0x189,
0141     AMDGPU_VEGA20_DOORBELL64_VCN4_5                  = 0x18A,
0142     AMDGPU_VEGA20_DOORBELL64_VCN6_7                  = 0x18B,
0143
0144     AMDGPU_VEGA20_DOORBELL64_VCN8_9                  = 0x18C, /* VNC1 */
0145     AMDGPU_VEGA20_DOORBELL64_VCNa_b                  = 0x18D,
0146     AMDGPU_VEGA20_DOORBELL64_VCNc_d                  = 0x18E,
0147     AMDGPU_VEGA20_DOORBELL64_VCNe_f                  = 0x18F,
0148
0149     AMDGPU_VEGA20_DOORBELL64_UVD_RING0_1             = 0x188,
0150     AMDGPU_VEGA20_DOORBELL64_UVD_RING2_3             = 0x189,
0151     AMDGPU_VEGA20_DOORBELL64_UVD_RING4_5             = 0x18A,
0152     AMDGPU_VEGA20_DOORBELL64_UVD_RING6_7             = 0x18B,
0153
0154     AMDGPU_VEGA20_DOORBELL64_VCE_RING0_1             = 0x18C,
0155     AMDGPU_VEGA20_DOORBELL64_VCE_RING2_3             = 0x18D,
0156     AMDGPU_VEGA20_DOORBELL64_VCE_RING4_5             = 0x18E,
0157     AMDGPU_VEGA20_DOORBELL64_VCE_RING6_7             = 0x18F,
0158
0159     AMDGPU_VEGA20_DOORBELL64_FIRST_NON_CP            = AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE0,
0160     AMDGPU_VEGA20_DOORBELL64_LAST_NON_CP             = AMDGPU_VEGA20_DOORBELL64_VCE_RING6_7,
0161
0162     AMDGPU_VEGA20_DOORBELL_MAX_ASSIGNMENT            = 0x18F,
0163     AMDGPU_VEGA20_DOORBELL_INVALID                   = 0xFFFF
0164 } AMDGPU_VEGA20_DOORBELL_ASSIGNMENT;
0165
0166 typedef enum _AMDGPU_NAVI10_DOORBELL_ASSIGNMENT
0167 {
0168     /* Compute + GFX: 0~255 */
0169     AMDGPU_NAVI10_DOORBELL_KIQ          = 0x000,
0170     AMDGPU_NAVI10_DOORBELL_HIQ          = 0x001,
0171     AMDGPU_NAVI10_DOORBELL_DIQ          = 0x002,
0172     AMDGPU_NAVI10_DOORBELL_MEC_RING0        = 0x003,
0173     AMDGPU_NAVI10_DOORBELL_MEC_RING1        = 0x004,
0174     AMDGPU_NAVI10_DOORBELL_MEC_RING2        = 0x005,
0175     AMDGPU_NAVI10_DOORBELL_MEC_RING3        = 0x006,
0176     AMDGPU_NAVI10_DOORBELL_MEC_RING4        = 0x007,
0177     AMDGPU_NAVI10_DOORBELL_MEC_RING5        = 0x008,
0178     AMDGPU_NAVI10_DOORBELL_MEC_RING6        = 0x009,
0179     AMDGPU_NAVI10_DOORBELL_MEC_RING7        = 0x00A,
0180     AMDGPU_NAVI10_DOORBELL_MES_RING0            = 0x00B,
0181     AMDGPU_NAVI10_DOORBELL_MES_RING1        = 0x00C,
0182     AMDGPU_NAVI10_DOORBELL_USERQUEUE_START      = 0x00D,
0183     AMDGPU_NAVI10_DOORBELL_USERQUEUE_END        = 0x08A,
0184     AMDGPU_NAVI10_DOORBELL_GFX_RING0        = 0x08B,
0185     AMDGPU_NAVI10_DOORBELL_GFX_RING1        = 0x08C,
0186     AMDGPU_NAVI10_DOORBELL_GFX_USERQUEUE_START  = 0x08D,
0187     AMDGPU_NAVI10_DOORBELL_GFX_USERQUEUE_END    = 0x0FF,
0188
0189     /* SDMA:256~335*/
0190     AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE0     = 0x100,
0191     AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE1     = 0x10A,
0192     AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE2     = 0x114,
0193     AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE3     = 0x11E,
0194     /* IH: 376~391 */
0195     AMDGPU_NAVI10_DOORBELL_IH           = 0x178,
0196     /* MMSCH: 392~407
0197      * overlap the doorbell assignment with VCN as they are  mutually exclusive
0198      * VCE engine's doorbell is 32 bit and two VCE ring share one QWORD
0199      */
0200     AMDGPU_NAVI10_DOORBELL64_VCN0_1         = 0x188, /* lower 32 bits for VNC0 and upper 32 bits for VNC1 */
0201     AMDGPU_NAVI10_DOORBELL64_VCN2_3         = 0x189,
0202     AMDGPU_NAVI10_DOORBELL64_VCN4_5         = 0x18A,
0203     AMDGPU_NAVI10_DOORBELL64_VCN6_7         = 0x18B,
0204
0205     AMDGPU_NAVI10_DOORBELL64_VCN8_9         = 0x18C,
0206     AMDGPU_NAVI10_DOORBELL64_VCNa_b         = 0x18D,
0207     AMDGPU_NAVI10_DOORBELL64_VCNc_d         = 0x18E,
0208     AMDGPU_NAVI10_DOORBELL64_VCNe_f         = 0x18F,
0209
0210     AMDGPU_NAVI10_DOORBELL64_FIRST_NON_CP       = AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE0,
0211     AMDGPU_NAVI10_DOORBELL64_LAST_NON_CP        = AMDGPU_NAVI10_DOORBELL64_VCNe_f,
0212
0213     AMDGPU_NAVI10_DOORBELL_MAX_ASSIGNMENT       = 0x18F,
0214     AMDGPU_NAVI10_DOORBELL_INVALID          = 0xFFFF
0215 } AMDGPU_NAVI10_DOORBELL_ASSIGNMENT;
0216
0217 /*
0218  * 64bit doorbell, offset are in QWORD, occupy 2KB doorbell space
0219  */
0220 typedef enum _AMDGPU_DOORBELL64_ASSIGNMENT
0221 {
0222     /*
0223      * All compute related doorbells: kiq, hiq, diq, traditional compute queue, user queue, should locate in
0224      * a continues range so that programming CP_MEC_DOORBELL_RANGE_LOWER/UPPER can cover this range.
0225      *  Compute related doorbells are allocated from 0x00 to 0x8a
0226      */
0227
0228
0229     /* kernel scheduling */
0230     AMDGPU_DOORBELL64_KIQ                     = 0x00,
0231
0232     /* HSA interface queue and debug queue */
0233     AMDGPU_DOORBELL64_HIQ                     = 0x01,
0234     AMDGPU_DOORBELL64_DIQ                     = 0x02,
0235
0236     /* Compute engines */
0237     AMDGPU_DOORBELL64_MEC_RING0               = 0x03,
0238     AMDGPU_DOORBELL64_MEC_RING1               = 0x04,
0239     AMDGPU_DOORBELL64_MEC_RING2               = 0x05,
0240     AMDGPU_DOORBELL64_MEC_RING3               = 0x06,
0241     AMDGPU_DOORBELL64_MEC_RING4               = 0x07,
0242     AMDGPU_DOORBELL64_MEC_RING5               = 0x08,
0243     AMDGPU_DOORBELL64_MEC_RING6               = 0x09,
0244     AMDGPU_DOORBELL64_MEC_RING7               = 0x0a,
0245
0246     /* User queue doorbell range (128 doorbells) */
0247     AMDGPU_DOORBELL64_USERQUEUE_START         = 0x0b,
0248     AMDGPU_DOORBELL64_USERQUEUE_END           = 0x8a,
0249
0250     /* Graphics engine */
0251     AMDGPU_DOORBELL64_GFX_RING0               = 0x8b,
0252
0253     /*
0254      * Other graphics doorbells can be allocated here: from 0x8c to 0xdf
0255      * Graphics voltage island aperture 1
0256      * default non-graphics QWORD index is 0xe0 - 0xFF inclusive
0257      */
0258
0259     /* For vega10 sriov, the sdma doorbell must be fixed as follow
0260      * to keep the same setting with host driver, or it will
0261      * happen conflicts
0262      */
0263     AMDGPU_DOORBELL64_sDMA_ENGINE0            = 0xF0,
0264     AMDGPU_DOORBELL64_sDMA_HI_PRI_ENGINE0     = 0xF1,
0265     AMDGPU_DOORBELL64_sDMA_ENGINE1            = 0xF2,
0266     AMDGPU_DOORBELL64_sDMA_HI_PRI_ENGINE1     = 0xF3,
0267
0268     /* Interrupt handler */
0269     AMDGPU_DOORBELL64_IH                      = 0xF4,  /* For legacy interrupt ring buffer */
0270     AMDGPU_DOORBELL64_IH_RING1                = 0xF5,  /* For page migration request log */
0271     AMDGPU_DOORBELL64_IH_RING2                = 0xF6,  /* For page migration translation/invalidation log */
0272
0273     /* VCN engine use 32 bits doorbell  */
0274     AMDGPU_DOORBELL64_VCN0_1                  = 0xF8, /* lower 32 bits for VNC0 and upper 32 bits for VNC1 */
0275     AMDGPU_DOORBELL64_VCN2_3                  = 0xF9,
0276     AMDGPU_DOORBELL64_VCN4_5                  = 0xFA,
0277     AMDGPU_DOORBELL64_VCN6_7                  = 0xFB,
0278
0279     /* overlap the doorbell assignment with VCN as they are  mutually exclusive
0280      * VCE engine's doorbell is 32 bit and two VCE ring share one QWORD
0281      */
0282     AMDGPU_DOORBELL64_UVD_RING0_1             = 0xF8,
0283     AMDGPU_DOORBELL64_UVD_RING2_3             = 0xF9,
0284     AMDGPU_DOORBELL64_UVD_RING4_5             = 0xFA,
0285     AMDGPU_DOORBELL64_UVD_RING6_7             = 0xFB,
0286
0287     AMDGPU_DOORBELL64_VCE_RING0_1             = 0xFC,
0288     AMDGPU_DOORBELL64_VCE_RING2_3             = 0xFD,
0289     AMDGPU_DOORBELL64_VCE_RING4_5             = 0xFE,
0290     AMDGPU_DOORBELL64_VCE_RING6_7             = 0xFF,
0291
0292     AMDGPU_DOORBELL64_FIRST_NON_CP            = AMDGPU_DOORBELL64_sDMA_ENGINE0,
0293     AMDGPU_DOORBELL64_LAST_NON_CP             = AMDGPU_DOORBELL64_VCE_RING6_7,
0294
0295     AMDGPU_DOORBELL64_MAX_ASSIGNMENT          = 0xFF,
0296     AMDGPU_DOORBELL64_INVALID                 = 0xFFFF
0297 } AMDGPU_DOORBELL64_ASSIGNMENT;
0298
0299 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index);
0300 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v);
0301 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index);
0302 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v);
0303
0304 #define RDOORBELL32(index) amdgpu_mm_rdoorbell(adev, (index))
0305 #define WDOORBELL32(index, v) amdgpu_mm_wdoorbell(adev, (index), (v))
0306 #define RDOORBELL64(index) amdgpu_mm_rdoorbell64(adev, (index))
0307 #define WDOORBELL64(index, v) amdgpu_mm_wdoorbell64(adev, (index), (v))
0308