Back to home page

OSCL-LXR
 
 

     

0001 /*
0002  * Copyright 2008 Advanced Micro Devices, Inc.
0003  * Copyright 2008 Red Hat Inc.
0004  * Copyright 2009 Jerome Glisse.
0005  *
0006  * Permission is hereby granted, free of charge, to any person obtaining a
0007  * copy of this software and associated documentation files (the "Software"),
0008  * to deal in the Software without restriction, including without limitation
0009  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0010  * and/or sell copies of the Software, and to permit persons to whom the
0011  * Software is furnished to do so, subject to the following conditions:
0012  *
0013  * The above copyright notice and this permission notice shall be included in
0014  * all copies or substantial portions of the Software.
0015  *
0016  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0017  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0018  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0019  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
0020  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
0021  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
0022  * OTHER DEALINGS IN THE SOFTWARE.
0023  *
0024  * Authors: Dave Airlie
0025  *          Alex Deucher
0026  *          Jerome Glisse
0027  */
0028 #ifndef __RADEON_H__
0029 #define __RADEON_H__
0030 
0031 /* TODO: Here are things that needs to be done :
0032  *  - surface allocator & initializer : (bit like scratch reg) should
0033  *    initialize HDP_ stuff on RS600, R600, R700 hw, well anythings
0034  *    related to surface
0035  *  - WB : write back stuff (do it bit like scratch reg things)
0036  *  - Vblank : look at Jesse's rework and what we should do
0037  *  - r600/r700: gart & cp
0038  *  - cs : clean cs ioctl use bitmap & things like that.
0039  *  - power management stuff
0040  *  - Barrier in gart code
0041  *  - Unmappabled vram ?
0042  *  - TESTING, TESTING, TESTING
0043  */
0044 
0045 /* Initialization path:
0046  *  We expect that acceleration initialization might fail for various
0047  *  reasons even thought we work hard to make it works on most
0048  *  configurations. In order to still have a working userspace in such
0049  *  situation the init path must succeed up to the memory controller
0050  *  initialization point. Failure before this point are considered as
0051  *  fatal error. Here is the init callchain :
0052  *      radeon_device_init  perform common structure, mutex initialization
0053  *      asic_init           setup the GPU memory layout and perform all
0054  *                          one time initialization (failure in this
0055  *                          function are considered fatal)
0056  *      asic_startup        setup the GPU acceleration, in order to
0057  *                          follow guideline the first thing this
0058  *                          function should do is setting the GPU
0059  *                          memory controller (only MC setup failure
0060  *                          are considered as fatal)
0061  */
0062 
0063 #include <linux/agp_backend.h>
0064 #include <linux/atomic.h>
0065 #include <linux/wait.h>
0066 #include <linux/list.h>
0067 #include <linux/kref.h>
0068 #include <linux/interval_tree.h>
0069 #include <linux/hashtable.h>
0070 #include <linux/dma-fence.h>
0071 
0072 #ifdef CONFIG_MMU_NOTIFIER
0073 #include <linux/mmu_notifier.h>
0074 #endif
0075 
0076 #include <drm/ttm/ttm_bo_api.h>
0077 #include <drm/ttm/ttm_bo_driver.h>
0078 #include <drm/ttm/ttm_placement.h>
0079 #include <drm/ttm/ttm_execbuf_util.h>
0080 
0081 #include <drm/drm_gem.h>
0082 
0083 #include "radeon_family.h"
0084 #include "radeon_mode.h"
0085 #include "radeon_reg.h"
0086 
0087 /*
0088  * Modules parameters.
0089  */
0090 extern int radeon_no_wb;
0091 extern int radeon_modeset;
0092 extern int radeon_dynclks;
0093 extern int radeon_r4xx_atom;
0094 extern int radeon_agpmode;
0095 extern int radeon_vram_limit;
0096 extern int radeon_gart_size;
0097 extern int radeon_benchmarking;
0098 extern int radeon_testing;
0099 extern int radeon_connector_table;
0100 extern int radeon_tv;
0101 extern int radeon_audio;
0102 extern int radeon_disp_priority;
0103 extern int radeon_hw_i2c;
0104 extern int radeon_pcie_gen2;
0105 extern int radeon_msi;
0106 extern int radeon_lockup_timeout;
0107 extern int radeon_fastfb;
0108 extern int radeon_dpm;
0109 extern int radeon_aspm;
0110 extern int radeon_runtime_pm;
0111 extern int radeon_hard_reset;
0112 extern int radeon_vm_size;
0113 extern int radeon_vm_block_size;
0114 extern int radeon_deep_color;
0115 extern int radeon_use_pflipirq;
0116 extern int radeon_bapm;
0117 extern int radeon_backlight;
0118 extern int radeon_auxch;
0119 extern int radeon_mst;
0120 extern int radeon_uvd;
0121 extern int radeon_vce;
0122 extern int radeon_si_support;
0123 extern int radeon_cik_support;
0124 
0125 /*
0126  * Copy from radeon_drv.h so we don't have to include both and have conflicting
0127  * symbol;
0128  */
0129 #define RADEON_MAX_USEC_TIMEOUT         100000  /* 100 ms */
0130 #define RADEON_FENCE_JIFFIES_TIMEOUT        (HZ / 2)
0131 #define RADEON_USEC_IB_TEST_TIMEOUT     1000000 /* 1s */
0132 /* RADEON_IB_POOL_SIZE must be a power of 2 */
0133 #define RADEON_IB_POOL_SIZE         16
0134 #define RADEON_DEBUGFS_MAX_COMPONENTS       32
0135 #define RADEONFB_CONN_LIMIT         4
0136 #define RADEON_BIOS_NUM_SCRATCH         8
0137 
0138 /* internal ring indices */
0139 /* r1xx+ has gfx CP ring */
0140 #define RADEON_RING_TYPE_GFX_INDEX      0
0141 
0142 /* cayman has 2 compute CP rings */
0143 #define CAYMAN_RING_TYPE_CP1_INDEX      1
0144 #define CAYMAN_RING_TYPE_CP2_INDEX      2
0145 
0146 /* R600+ has an async dma ring */
0147 #define R600_RING_TYPE_DMA_INDEX        3
0148 /* cayman add a second async dma ring */
0149 #define CAYMAN_RING_TYPE_DMA1_INDEX     4
0150 
0151 /* R600+ */
0152 #define R600_RING_TYPE_UVD_INDEX        5
0153 
0154 /* TN+ */
0155 #define TN_RING_TYPE_VCE1_INDEX         6
0156 #define TN_RING_TYPE_VCE2_INDEX         7
0157 
0158 /* max number of rings */
0159 #define RADEON_NUM_RINGS            8
0160 
0161 /* number of hw syncs before falling back on blocking */
0162 #define RADEON_NUM_SYNCS            4
0163 
0164 /* hardcode those limit for now */
0165 #define RADEON_VA_IB_OFFSET         (1 << 20)
0166 #define RADEON_VA_RESERVED_SIZE         (8 << 20)
0167 #define RADEON_IB_VM_MAX_SIZE           (64 << 10)
0168 
0169 /* hard reset data */
0170 #define RADEON_ASIC_RESET_DATA                  0x39d5e86b
0171 
0172 /* reset flags */
0173 #define RADEON_RESET_GFX            (1 << 0)
0174 #define RADEON_RESET_COMPUTE            (1 << 1)
0175 #define RADEON_RESET_DMA            (1 << 2)
0176 #define RADEON_RESET_CP             (1 << 3)
0177 #define RADEON_RESET_GRBM           (1 << 4)
0178 #define RADEON_RESET_DMA1           (1 << 5)
0179 #define RADEON_RESET_RLC            (1 << 6)
0180 #define RADEON_RESET_SEM            (1 << 7)
0181 #define RADEON_RESET_IH             (1 << 8)
0182 #define RADEON_RESET_VMC            (1 << 9)
0183 #define RADEON_RESET_MC             (1 << 10)
0184 #define RADEON_RESET_DISPLAY            (1 << 11)
0185 
0186 /* CG block flags */
0187 #define RADEON_CG_BLOCK_GFX         (1 << 0)
0188 #define RADEON_CG_BLOCK_MC          (1 << 1)
0189 #define RADEON_CG_BLOCK_SDMA            (1 << 2)
0190 #define RADEON_CG_BLOCK_UVD         (1 << 3)
0191 #define RADEON_CG_BLOCK_VCE         (1 << 4)
0192 #define RADEON_CG_BLOCK_HDP         (1 << 5)
0193 #define RADEON_CG_BLOCK_BIF         (1 << 6)
0194 
0195 /* CG flags */
0196 #define RADEON_CG_SUPPORT_GFX_MGCG      (1 << 0)
0197 #define RADEON_CG_SUPPORT_GFX_MGLS      (1 << 1)
0198 #define RADEON_CG_SUPPORT_GFX_CGCG      (1 << 2)
0199 #define RADEON_CG_SUPPORT_GFX_CGLS      (1 << 3)
0200 #define RADEON_CG_SUPPORT_GFX_CGTS      (1 << 4)
0201 #define RADEON_CG_SUPPORT_GFX_CGTS_LS       (1 << 5)
0202 #define RADEON_CG_SUPPORT_GFX_CP_LS     (1 << 6)
0203 #define RADEON_CG_SUPPORT_GFX_RLC_LS        (1 << 7)
0204 #define RADEON_CG_SUPPORT_MC_LS         (1 << 8)
0205 #define RADEON_CG_SUPPORT_MC_MGCG       (1 << 9)
0206 #define RADEON_CG_SUPPORT_SDMA_LS       (1 << 10)
0207 #define RADEON_CG_SUPPORT_SDMA_MGCG     (1 << 11)
0208 #define RADEON_CG_SUPPORT_BIF_LS        (1 << 12)
0209 #define RADEON_CG_SUPPORT_UVD_MGCG      (1 << 13)
0210 #define RADEON_CG_SUPPORT_VCE_MGCG      (1 << 14)
0211 #define RADEON_CG_SUPPORT_HDP_LS        (1 << 15)
0212 #define RADEON_CG_SUPPORT_HDP_MGCG      (1 << 16)
0213 
0214 /* PG flags */
0215 #define RADEON_PG_SUPPORT_GFX_PG        (1 << 0)
0216 #define RADEON_PG_SUPPORT_GFX_SMG       (1 << 1)
0217 #define RADEON_PG_SUPPORT_GFX_DMG       (1 << 2)
0218 #define RADEON_PG_SUPPORT_UVD           (1 << 3)
0219 #define RADEON_PG_SUPPORT_VCE           (1 << 4)
0220 #define RADEON_PG_SUPPORT_CP            (1 << 5)
0221 #define RADEON_PG_SUPPORT_GDS           (1 << 6)
0222 #define RADEON_PG_SUPPORT_RLC_SMU_HS        (1 << 7)
0223 #define RADEON_PG_SUPPORT_SDMA          (1 << 8)
0224 #define RADEON_PG_SUPPORT_ACP           (1 << 9)
0225 #define RADEON_PG_SUPPORT_SAMU          (1 << 10)
0226 
0227 /* max cursor sizes (in pixels) */
0228 #define CURSOR_WIDTH 64
0229 #define CURSOR_HEIGHT 64
0230 
0231 #define CIK_CURSOR_WIDTH 128
0232 #define CIK_CURSOR_HEIGHT 128
0233 
0234 /*
0235  * Errata workarounds.
0236  */
0237 enum radeon_pll_errata {
0238     CHIP_ERRATA_R300_CG             = 0x00000001,
0239     CHIP_ERRATA_PLL_DUMMYREADS      = 0x00000002,
0240     CHIP_ERRATA_PLL_DELAY           = 0x00000004
0241 };
0242 
0243 
0244 struct radeon_device;
0245 
0246 
0247 /*
0248  * BIOS.
0249  */
0250 bool radeon_get_bios(struct radeon_device *rdev);
0251 
0252 /*
0253  * Dummy page
0254  */
0255 struct radeon_dummy_page {
0256     uint64_t    entry;
0257     struct page *page;
0258     dma_addr_t  addr;
0259 };
0260 int radeon_dummy_page_init(struct radeon_device *rdev);
0261 void radeon_dummy_page_fini(struct radeon_device *rdev);
0262 
0263 
0264 /*
0265  * Clocks
0266  */
0267 struct radeon_clock {
0268     struct radeon_pll p1pll;
0269     struct radeon_pll p2pll;
0270     struct radeon_pll dcpll;
0271     struct radeon_pll spll;
0272     struct radeon_pll mpll;
0273     /* 10 Khz units */
0274     uint32_t default_mclk;
0275     uint32_t default_sclk;
0276     uint32_t default_dispclk;
0277     uint32_t current_dispclk;
0278     uint32_t dp_extclk;
0279     uint32_t max_pixel_clock;
0280     uint32_t vco_freq;
0281 };
0282 
0283 /*
0284  * Power management
0285  */
0286 int radeon_pm_init(struct radeon_device *rdev);
0287 int radeon_pm_late_init(struct radeon_device *rdev);
0288 void radeon_pm_fini(struct radeon_device *rdev);
0289 void radeon_pm_compute_clocks(struct radeon_device *rdev);
0290 void radeon_pm_suspend(struct radeon_device *rdev);
0291 void radeon_pm_resume(struct radeon_device *rdev);
0292 void radeon_combios_get_power_modes(struct radeon_device *rdev);
0293 void radeon_atombios_get_power_modes(struct radeon_device *rdev);
0294 int radeon_atom_get_clock_dividers(struct radeon_device *rdev,
0295                    u8 clock_type,
0296                    u32 clock,
0297                    bool strobe_mode,
0298                    struct atom_clock_dividers *dividers);
0299 int radeon_atom_get_memory_pll_dividers(struct radeon_device *rdev,
0300                     u32 clock,
0301                     bool strobe_mode,
0302                     struct atom_mpll_param *mpll_param);
0303 void radeon_atom_set_voltage(struct radeon_device *rdev, u16 voltage_level, u8 voltage_type);
0304 int radeon_atom_get_voltage_gpio_settings(struct radeon_device *rdev,
0305                       u16 voltage_level, u8 voltage_type,
0306                       u32 *gpio_value, u32 *gpio_mask);
0307 void radeon_atom_set_engine_dram_timings(struct radeon_device *rdev,
0308                      u32 eng_clock, u32 mem_clock);
0309 int radeon_atom_get_voltage_step(struct radeon_device *rdev,
0310                  u8 voltage_type, u16 *voltage_step);
0311 int radeon_atom_get_max_vddc(struct radeon_device *rdev, u8 voltage_type,
0312                  u16 voltage_id, u16 *voltage);
0313 int radeon_atom_get_leakage_vddc_based_on_leakage_idx(struct radeon_device *rdev,
0314                               u16 *voltage,
0315                               u16 leakage_idx);
0316 int radeon_atom_get_leakage_id_from_vbios(struct radeon_device *rdev,
0317                       u16 *leakage_id);
0318 int radeon_atom_get_leakage_vddc_based_on_leakage_params(struct radeon_device *rdev,
0319                              u16 *vddc, u16 *vddci,
0320                              u16 virtual_voltage_id,
0321                              u16 vbios_voltage_id);
0322 int radeon_atom_get_voltage_evv(struct radeon_device *rdev,
0323                 u16 virtual_voltage_id,
0324                 u16 *voltage);
0325 int radeon_atom_round_to_true_voltage(struct radeon_device *rdev,
0326                       u8 voltage_type,
0327                       u16 nominal_voltage,
0328                       u16 *true_voltage);
0329 int radeon_atom_get_min_voltage(struct radeon_device *rdev,
0330                 u8 voltage_type, u16 *min_voltage);
0331 int radeon_atom_get_max_voltage(struct radeon_device *rdev,
0332                 u8 voltage_type, u16 *max_voltage);
0333 int radeon_atom_get_voltage_table(struct radeon_device *rdev,
0334                   u8 voltage_type, u8 voltage_mode,
0335                   struct atom_voltage_table *voltage_table);
0336 bool radeon_atom_is_voltage_gpio(struct radeon_device *rdev,
0337                  u8 voltage_type, u8 voltage_mode);
0338 int radeon_atom_get_svi2_info(struct radeon_device *rdev,
0339                   u8 voltage_type,
0340                   u8 *svd_gpio_id, u8 *svc_gpio_id);
0341 void radeon_atom_update_memory_dll(struct radeon_device *rdev,
0342                    u32 mem_clock);
0343 void radeon_atom_set_ac_timing(struct radeon_device *rdev,
0344                    u32 mem_clock);
0345 int radeon_atom_init_mc_reg_table(struct radeon_device *rdev,
0346                   u8 module_index,
0347                   struct atom_mc_reg_table *reg_table);
0348 int radeon_atom_get_memory_info(struct radeon_device *rdev,
0349                 u8 module_index, struct atom_memory_info *mem_info);
0350 int radeon_atom_get_mclk_range_table(struct radeon_device *rdev,
0351                      bool gddr5, u8 module_index,
0352                      struct atom_memory_clock_range_table *mclk_range_table);
0353 int radeon_atom_get_max_vddc(struct radeon_device *rdev, u8 voltage_type,
0354                  u16 voltage_id, u16 *voltage);
0355 void rs690_pm_info(struct radeon_device *rdev);
0356 extern void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
0357                     unsigned *bankh, unsigned *mtaspect,
0358                     unsigned *tile_split);
0359 
0360 /*
0361  * Fences.
0362  */
0363 struct radeon_fence_driver {
0364     struct radeon_device        *rdev;
0365     uint32_t            scratch_reg;
0366     uint64_t            gpu_addr;
0367     volatile uint32_t       *cpu_addr;
0368     /* sync_seq is protected by ring emission lock */
0369     uint64_t            sync_seq[RADEON_NUM_RINGS];
0370     atomic64_t          last_seq;
0371     bool                initialized, delayed_irq;
0372     struct delayed_work     lockup_work;
0373 };
0374 
0375 struct radeon_fence {
0376     struct dma_fence        base;
0377 
0378     struct radeon_device    *rdev;
0379     uint64_t        seq;
0380     /* RB, DMA, etc. */
0381     unsigned        ring;
0382     bool            is_vm_update;
0383 
0384     wait_queue_entry_t      fence_wake;
0385 };
0386 
0387 int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring);
0388 void radeon_fence_driver_init(struct radeon_device *rdev);
0389 void radeon_fence_driver_fini(struct radeon_device *rdev);
0390 void radeon_fence_driver_force_completion(struct radeon_device *rdev, int ring);
0391 int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence **fence, int ring);
0392 void radeon_fence_process(struct radeon_device *rdev, int ring);
0393 bool radeon_fence_signaled(struct radeon_fence *fence);
0394 long radeon_fence_wait_timeout(struct radeon_fence *fence, bool interruptible, long timeout);
0395 int radeon_fence_wait(struct radeon_fence *fence, bool interruptible);
0396 int radeon_fence_wait_next(struct radeon_device *rdev, int ring);
0397 int radeon_fence_wait_empty(struct radeon_device *rdev, int ring);
0398 int radeon_fence_wait_any(struct radeon_device *rdev,
0399               struct radeon_fence **fences,
0400               bool intr);
0401 struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence);
0402 void radeon_fence_unref(struct radeon_fence **fence);
0403 unsigned radeon_fence_count_emitted(struct radeon_device *rdev, int ring);
0404 bool radeon_fence_need_sync(struct radeon_fence *fence, int ring);
0405 void radeon_fence_note_sync(struct radeon_fence *fence, int ring);
0406 static inline struct radeon_fence *radeon_fence_later(struct radeon_fence *a,
0407                               struct radeon_fence *b)
0408 {
0409     if (!a) {
0410         return b;
0411     }
0412 
0413     if (!b) {
0414         return a;
0415     }
0416 
0417     BUG_ON(a->ring != b->ring);
0418 
0419     if (a->seq > b->seq) {
0420         return a;
0421     } else {
0422         return b;
0423     }
0424 }
0425 
0426 static inline bool radeon_fence_is_earlier(struct radeon_fence *a,
0427                        struct radeon_fence *b)
0428 {
0429     if (!a) {
0430         return false;
0431     }
0432 
0433     if (!b) {
0434         return true;
0435     }
0436 
0437     BUG_ON(a->ring != b->ring);
0438 
0439     return a->seq < b->seq;
0440 }
0441 
0442 /*
0443  * Tiling registers
0444  */
0445 struct radeon_surface_reg {
0446     struct radeon_bo *bo;
0447 };
0448 
0449 #define RADEON_GEM_MAX_SURFACES 8
0450 
0451 /*
0452  * TTM.
0453  */
0454 struct radeon_mman {
0455     struct ttm_device       bdev;
0456     bool                initialized;
0457 };
0458 
0459 struct radeon_bo_list {
0460     struct radeon_bo        *robj;
0461     struct ttm_validate_buffer  tv;
0462     uint64_t            gpu_offset;
0463     unsigned            preferred_domains;
0464     unsigned            allowed_domains;
0465     uint32_t            tiling_flags;
0466 };
0467 
0468 /* bo virtual address in a specific vm */
0469 struct radeon_bo_va {
0470     /* protected by bo being reserved */
0471     struct list_head        bo_list;
0472     uint32_t            flags;
0473     struct radeon_fence     *last_pt_update;
0474     unsigned            ref_count;
0475 
0476     /* protected by vm mutex */
0477     struct interval_tree_node   it;
0478     struct list_head        vm_status;
0479 
0480     /* constant after initialization */
0481     struct radeon_vm        *vm;
0482     struct radeon_bo        *bo;
0483 };
0484 
0485 struct radeon_bo {
0486     /* Protected by gem.mutex */
0487     struct list_head        list;
0488     /* Protected by tbo.reserved */
0489     u32             initial_domain;
0490     struct ttm_place        placements[4];
0491     struct ttm_placement        placement;
0492     struct ttm_buffer_object    tbo;
0493     struct ttm_bo_kmap_obj      kmap;
0494     u32             flags;
0495     void                *kptr;
0496     u32             tiling_flags;
0497     u32             pitch;
0498     int             surface_reg;
0499     unsigned            prime_shared_count;
0500     /* list of all virtual address to which this bo
0501      * is associated to
0502      */
0503     struct list_head        va;
0504     /* Constant after initialization */
0505     struct radeon_device        *rdev;
0506 
0507     pid_t               pid;
0508 
0509 #ifdef CONFIG_MMU_NOTIFIER
0510     struct mmu_interval_notifier    notifier;
0511 #endif
0512 };
0513 #define gem_to_radeon_bo(gobj) container_of((gobj), struct radeon_bo, tbo.base)
0514 
0515 /* sub-allocation manager, it has to be protected by another lock.
0516  * By conception this is an helper for other part of the driver
0517  * like the indirect buffer or semaphore, which both have their
0518  * locking.
0519  *
0520  * Principe is simple, we keep a list of sub allocation in offset
0521  * order (first entry has offset == 0, last entry has the highest
0522  * offset).
0523  *
0524  * When allocating new object we first check if there is room at
0525  * the end total_size - (last_object_offset + last_object_size) >=
0526  * alloc_size. If so we allocate new object there.
0527  *
0528  * When there is not enough room at the end, we start waiting for
0529  * each sub object until we reach object_offset+object_size >=
0530  * alloc_size, this object then become the sub object we return.
0531  *
0532  * Alignment can't be bigger than page size.
0533  *
0534  * Hole are not considered for allocation to keep things simple.
0535  * Assumption is that there won't be hole (all object on same
0536  * alignment).
0537  */
0538 struct radeon_sa_manager {
0539     wait_queue_head_t   wq;
0540     struct radeon_bo    *bo;
0541     struct list_head    *hole;
0542     struct list_head    flist[RADEON_NUM_RINGS];
0543     struct list_head    olist;
0544     unsigned        size;
0545     uint64_t        gpu_addr;
0546     void            *cpu_ptr;
0547     uint32_t        domain;
0548     uint32_t        align;
0549 };
0550 
0551 struct radeon_sa_bo;
0552 
0553 /* sub-allocation buffer */
0554 struct radeon_sa_bo {
0555     struct list_head        olist;
0556     struct list_head        flist;
0557     struct radeon_sa_manager    *manager;
0558     unsigned            soffset;
0559     unsigned            eoffset;
0560     struct radeon_fence     *fence;
0561 };
0562 
0563 /*
0564  * GEM objects.
0565  */
0566 struct radeon_gem {
0567     struct mutex        mutex;
0568     struct list_head    objects;
0569 };
0570 
0571 extern const struct drm_gem_object_funcs radeon_gem_object_funcs;
0572 
0573 int radeon_gem_init(struct radeon_device *rdev);
0574 void radeon_gem_fini(struct radeon_device *rdev);
0575 int radeon_gem_object_create(struct radeon_device *rdev, unsigned long size,
0576                 int alignment, int initial_domain,
0577                 u32 flags, bool kernel,
0578                 struct drm_gem_object **obj);
0579 
0580 int radeon_mode_dumb_create(struct drm_file *file_priv,
0581                 struct drm_device *dev,
0582                 struct drm_mode_create_dumb *args);
0583 int radeon_mode_dumb_mmap(struct drm_file *filp,
0584               struct drm_device *dev,
0585               uint32_t handle, uint64_t *offset_p);
0586 
0587 /*
0588  * Semaphores.
0589  */
0590 struct radeon_semaphore {
0591     struct radeon_sa_bo *sa_bo;
0592     signed          waiters;
0593     uint64_t        gpu_addr;
0594 };
0595 
0596 int radeon_semaphore_create(struct radeon_device *rdev,
0597                 struct radeon_semaphore **semaphore);
0598 bool radeon_semaphore_emit_signal(struct radeon_device *rdev, int ring,
0599                   struct radeon_semaphore *semaphore);
0600 bool radeon_semaphore_emit_wait(struct radeon_device *rdev, int ring,
0601                 struct radeon_semaphore *semaphore);
0602 void radeon_semaphore_free(struct radeon_device *rdev,
0603                struct radeon_semaphore **semaphore,
0604                struct radeon_fence *fence);
0605 
0606 /*
0607  * Synchronization
0608  */
0609 struct radeon_sync {
0610     struct radeon_semaphore *semaphores[RADEON_NUM_SYNCS];
0611     struct radeon_fence *sync_to[RADEON_NUM_RINGS];
0612     struct radeon_fence *last_vm_update;
0613 };
0614 
0615 void radeon_sync_create(struct radeon_sync *sync);
0616 void radeon_sync_fence(struct radeon_sync *sync,
0617                struct radeon_fence *fence);
0618 int radeon_sync_resv(struct radeon_device *rdev,
0619              struct radeon_sync *sync,
0620              struct dma_resv *resv,
0621              bool shared);
0622 int radeon_sync_rings(struct radeon_device *rdev,
0623               struct radeon_sync *sync,
0624               int waiting_ring);
0625 void radeon_sync_free(struct radeon_device *rdev, struct radeon_sync *sync,
0626               struct radeon_fence *fence);
0627 
0628 /*
0629  * GART structures, functions & helpers
0630  */
0631 struct radeon_mc;
0632 
0633 #define RADEON_GPU_PAGE_SIZE 4096
0634 #define RADEON_GPU_PAGE_MASK (RADEON_GPU_PAGE_SIZE - 1)
0635 #define RADEON_GPU_PAGE_SHIFT 12
0636 #define RADEON_GPU_PAGE_ALIGN(a) (((a) + RADEON_GPU_PAGE_MASK) & ~RADEON_GPU_PAGE_MASK)
0637 
0638 #define RADEON_GART_PAGE_DUMMY  0
0639 #define RADEON_GART_PAGE_VALID  (1 << 0)
0640 #define RADEON_GART_PAGE_READ   (1 << 1)
0641 #define RADEON_GART_PAGE_WRITE  (1 << 2)
0642 #define RADEON_GART_PAGE_SNOOP  (1 << 3)
0643 
0644 struct radeon_gart {
0645     dma_addr_t          table_addr;
0646     struct radeon_bo        *robj;
0647     void                *ptr;
0648     unsigned            num_gpu_pages;
0649     unsigned            num_cpu_pages;
0650     unsigned            table_size;
0651     struct page         **pages;
0652     uint64_t            *pages_entry;
0653     bool                ready;
0654 };
0655 
0656 int radeon_gart_table_ram_alloc(struct radeon_device *rdev);
0657 void radeon_gart_table_ram_free(struct radeon_device *rdev);
0658 int radeon_gart_table_vram_alloc(struct radeon_device *rdev);
0659 void radeon_gart_table_vram_free(struct radeon_device *rdev);
0660 int radeon_gart_table_vram_pin(struct radeon_device *rdev);
0661 void radeon_gart_table_vram_unpin(struct radeon_device *rdev);
0662 int radeon_gart_init(struct radeon_device *rdev);
0663 void radeon_gart_fini(struct radeon_device *rdev);
0664 void radeon_gart_unbind(struct radeon_device *rdev, unsigned offset,
0665             int pages);
0666 int radeon_gart_bind(struct radeon_device *rdev, unsigned offset,
0667              int pages, struct page **pagelist,
0668              dma_addr_t *dma_addr, uint32_t flags);
0669 
0670 
0671 /*
0672  * GPU MC structures, functions & helpers
0673  */
0674 struct radeon_mc {
0675     resource_size_t     aper_size;
0676     resource_size_t     aper_base;
0677     resource_size_t     agp_base;
0678     /* for some chips with <= 32MB we need to lie
0679      * about vram size near mc fb location */
0680     u64         mc_vram_size;
0681     u64         visible_vram_size;
0682     u64         gtt_size;
0683     u64         gtt_start;
0684     u64         gtt_end;
0685     u64         vram_start;
0686     u64         vram_end;
0687     unsigned        vram_width;
0688     u64         real_vram_size;
0689     int         vram_mtrr;
0690     bool            vram_is_ddr;
0691     bool            igp_sideport_enabled;
0692     u64                     gtt_base_align;
0693     u64                     mc_mask;
0694 };
0695 
0696 bool radeon_combios_sideport_present(struct radeon_device *rdev);
0697 bool radeon_atombios_sideport_present(struct radeon_device *rdev);
0698 
0699 /*
0700  * GPU scratch registers structures, functions & helpers
0701  */
0702 struct radeon_scratch {
0703     unsigned        num_reg;
0704     uint32_t                reg_base;
0705     bool            free[32];
0706     uint32_t        reg[32];
0707 };
0708 
0709 int radeon_scratch_get(struct radeon_device *rdev, uint32_t *reg);
0710 void radeon_scratch_free(struct radeon_device *rdev, uint32_t reg);
0711 
0712 /*
0713  * GPU doorbell structures, functions & helpers
0714  */
0715 #define RADEON_MAX_DOORBELLS 1024   /* Reserve at most 1024 doorbell slots for radeon-owned rings. */
0716 
0717 struct radeon_doorbell {
0718     /* doorbell mmio */
0719     resource_size_t     base;
0720     resource_size_t     size;
0721     u32 __iomem     *ptr;
0722     u32         num_doorbells;  /* Number of doorbells actually reserved for radeon. */
0723     DECLARE_BITMAP(used, RADEON_MAX_DOORBELLS);
0724 };
0725 
0726 int radeon_doorbell_get(struct radeon_device *rdev, u32 *page);
0727 void radeon_doorbell_free(struct radeon_device *rdev, u32 doorbell);
0728 
0729 /*
0730  * IRQS.
0731  */
0732 
0733 struct radeon_flip_work {
0734     struct work_struct      flip_work;
0735     struct work_struct      unpin_work;
0736     struct radeon_device        *rdev;
0737     int             crtc_id;
0738     u32             target_vblank;
0739     uint64_t            base;
0740     struct drm_pending_vblank_event *event;
0741     struct radeon_bo        *old_rbo;
0742     struct dma_fence        *fence;
0743     bool                async;
0744 };
0745 
0746 struct r500_irq_stat_regs {
0747     u32 disp_int;
0748     u32 hdmi0_status;
0749 };
0750 
0751 struct r600_irq_stat_regs {
0752     u32 disp_int;
0753     u32 disp_int_cont;
0754     u32 disp_int_cont2;
0755     u32 d1grph_int;
0756     u32 d2grph_int;
0757     u32 hdmi0_status;
0758     u32 hdmi1_status;
0759 };
0760 
0761 struct evergreen_irq_stat_regs {
0762     u32 disp_int[6];
0763     u32 grph_int[6];
0764     u32 afmt_status[6];
0765 };
0766 
0767 struct cik_irq_stat_regs {
0768     u32 disp_int;
0769     u32 disp_int_cont;
0770     u32 disp_int_cont2;
0771     u32 disp_int_cont3;
0772     u32 disp_int_cont4;
0773     u32 disp_int_cont5;
0774     u32 disp_int_cont6;
0775     u32 d1grph_int;
0776     u32 d2grph_int;
0777     u32 d3grph_int;
0778     u32 d4grph_int;
0779     u32 d5grph_int;
0780     u32 d6grph_int;
0781 };
0782 
0783 union radeon_irq_stat_regs {
0784     struct r500_irq_stat_regs r500;
0785     struct r600_irq_stat_regs r600;
0786     struct evergreen_irq_stat_regs evergreen;
0787     struct cik_irq_stat_regs cik;
0788 };
0789 
0790 struct radeon_irq {
0791     bool                installed;
0792     spinlock_t          lock;
0793     atomic_t            ring_int[RADEON_NUM_RINGS];
0794     bool                crtc_vblank_int[RADEON_MAX_CRTCS];
0795     atomic_t            pflip[RADEON_MAX_CRTCS];
0796     wait_queue_head_t       vblank_queue;
0797     bool                hpd[RADEON_MAX_HPD_PINS];
0798     bool                afmt[RADEON_MAX_AFMT_BLOCKS];
0799     union radeon_irq_stat_regs  stat_regs;
0800     bool                dpm_thermal;
0801 };
0802 
0803 int radeon_irq_kms_init(struct radeon_device *rdev);
0804 void radeon_irq_kms_fini(struct radeon_device *rdev);
0805 void radeon_irq_kms_sw_irq_get(struct radeon_device *rdev, int ring);
0806 bool radeon_irq_kms_sw_irq_get_delayed(struct radeon_device *rdev, int ring);
0807 void radeon_irq_kms_sw_irq_put(struct radeon_device *rdev, int ring);
0808 void radeon_irq_kms_pflip_irq_get(struct radeon_device *rdev, int crtc);
0809 void radeon_irq_kms_pflip_irq_put(struct radeon_device *rdev, int crtc);
0810 void radeon_irq_kms_enable_afmt(struct radeon_device *rdev, int block);
0811 void radeon_irq_kms_disable_afmt(struct radeon_device *rdev, int block);
0812 void radeon_irq_kms_enable_hpd(struct radeon_device *rdev, unsigned hpd_mask);
0813 void radeon_irq_kms_disable_hpd(struct radeon_device *rdev, unsigned hpd_mask);
0814 
0815 /*
0816  * CP & rings.
0817  */
0818 
0819 struct radeon_ib {
0820     struct radeon_sa_bo     *sa_bo;
0821     uint32_t            length_dw;
0822     uint64_t            gpu_addr;
0823     uint32_t            *ptr;
0824     int             ring;
0825     struct radeon_fence     *fence;
0826     struct radeon_vm        *vm;
0827     bool                is_const_ib;
0828     struct radeon_sync      sync;
0829 };
0830 
0831 struct radeon_ring {
0832     struct radeon_device    *rdev;
0833     struct radeon_bo    *ring_obj;
0834     volatile uint32_t   *ring;
0835     unsigned        rptr_offs;
0836     unsigned        rptr_save_reg;
0837     u64         next_rptr_gpu_addr;
0838     volatile u32        *next_rptr_cpu_addr;
0839     unsigned        wptr;
0840     unsigned        wptr_old;
0841     unsigned        ring_size;
0842     unsigned        ring_free_dw;
0843     int         count_dw;
0844     atomic_t        last_rptr;
0845     atomic64_t      last_activity;
0846     uint64_t        gpu_addr;
0847     uint32_t        align_mask;
0848     uint32_t        ptr_mask;
0849     bool            ready;
0850     u32         nop;
0851     u32         idx;
0852     u64         last_semaphore_signal_addr;
0853     u64         last_semaphore_wait_addr;
0854     /* for CIK queues */
0855     u32 me;
0856     u32 pipe;
0857     u32 queue;
0858     struct radeon_bo    *mqd_obj;
0859     u32 doorbell_index;
0860     unsigned        wptr_offs;
0861 };
0862 
0863 struct radeon_mec {
0864     struct radeon_bo    *hpd_eop_obj;
0865     u64         hpd_eop_gpu_addr;
0866     u32 num_pipe;
0867     u32 num_mec;
0868     u32 num_queue;
0869 };
0870 
0871 /*
0872  * VM
0873  */
0874 
0875 /* maximum number of VMIDs */
0876 #define RADEON_NUM_VM   16
0877 
0878 /* number of entries in page table */
0879 #define RADEON_VM_PTE_COUNT (1 << radeon_vm_block_size)
0880 
0881 /* PTBs (Page Table Blocks) need to be aligned to 32K */
0882 #define RADEON_VM_PTB_ALIGN_SIZE   32768
0883 #define RADEON_VM_PTB_ALIGN_MASK (RADEON_VM_PTB_ALIGN_SIZE - 1)
0884 #define RADEON_VM_PTB_ALIGN(a) (((a) + RADEON_VM_PTB_ALIGN_MASK) & ~RADEON_VM_PTB_ALIGN_MASK)
0885 
0886 #define R600_PTE_VALID      (1 << 0)
0887 #define R600_PTE_SYSTEM     (1 << 1)
0888 #define R600_PTE_SNOOPED    (1 << 2)
0889 #define R600_PTE_READABLE   (1 << 5)
0890 #define R600_PTE_WRITEABLE  (1 << 6)
0891 
0892 /* PTE (Page Table Entry) fragment field for different page sizes */
0893 #define R600_PTE_FRAG_4KB   (0 << 7)
0894 #define R600_PTE_FRAG_64KB  (4 << 7)
0895 #define R600_PTE_FRAG_256KB (6 << 7)
0896 
0897 /* flags needed to be set so we can copy directly from the GART table */
0898 #define R600_PTE_GART_MASK  ( R600_PTE_READABLE | R600_PTE_WRITEABLE | \
0899                   R600_PTE_SYSTEM | R600_PTE_VALID )
0900 
0901 struct radeon_vm_pt {
0902     struct radeon_bo        *bo;
0903     uint64_t            addr;
0904 };
0905 
0906 struct radeon_vm_id {
0907     unsigned        id;
0908     uint64_t        pd_gpu_addr;
0909     /* last flushed PD/PT update */
0910     struct radeon_fence *flushed_updates;
0911     /* last use of vmid */
0912     struct radeon_fence *last_id_use;
0913 };
0914 
0915 struct radeon_vm {
0916     struct mutex        mutex;
0917 
0918     struct rb_root_cached   va;
0919 
0920     /* protecting invalidated and freed */
0921     spinlock_t      status_lock;
0922 
0923     /* BOs moved, but not yet updated in the PT */
0924     struct list_head    invalidated;
0925 
0926     /* BOs freed, but not yet updated in the PT */
0927     struct list_head    freed;
0928 
0929     /* BOs cleared in the PT */
0930     struct list_head    cleared;
0931 
0932     /* contains the page directory */
0933     struct radeon_bo    *page_directory;
0934     unsigned        max_pde_used;
0935 
0936     /* array of page tables, one for each page directory entry */
0937     struct radeon_vm_pt *page_tables;
0938 
0939     struct radeon_bo_va *ib_bo_va;
0940 
0941     /* for id and flush management per ring */
0942     struct radeon_vm_id ids[RADEON_NUM_RINGS];
0943 };
0944 
0945 struct radeon_vm_manager {
0946     struct radeon_fence     *active[RADEON_NUM_VM];
0947     uint32_t            max_pfn;
0948     /* number of VMIDs */
0949     unsigned            nvm;
0950     /* vram base address for page table entry  */
0951     u64             vram_base_offset;
0952     /* is vm enabled? */
0953     bool                enabled;
0954     /* for hw to save the PD addr on suspend/resume */
0955     uint32_t            saved_table_addr[RADEON_NUM_VM];
0956 };
0957 
0958 /*
0959  * file private structure
0960  */
0961 struct radeon_fpriv {
0962     struct radeon_vm        vm;
0963 };
0964 
0965 /*
0966  * R6xx+ IH ring
0967  */
0968 struct r600_ih {
0969     struct radeon_bo    *ring_obj;
0970     volatile uint32_t   *ring;
0971     unsigned        rptr;
0972     unsigned        ring_size;
0973     uint64_t        gpu_addr;
0974     uint32_t        ptr_mask;
0975     atomic_t        lock;
0976     bool                    enabled;
0977 };
0978 
0979 /*
0980  * RLC stuff
0981  */
0982 #include "clearstate_defs.h"
0983 
0984 struct radeon_rlc {
0985     /* for power gating */
0986     struct radeon_bo    *save_restore_obj;
0987     uint64_t        save_restore_gpu_addr;
0988     volatile uint32_t   *sr_ptr;
0989     const u32               *reg_list;
0990     u32                     reg_list_size;
0991     /* for clear state */
0992     struct radeon_bo    *clear_state_obj;
0993     uint64_t        clear_state_gpu_addr;
0994     volatile uint32_t   *cs_ptr;
0995     const struct cs_section_def   *cs_data;
0996     u32                     clear_state_size;
0997     /* for cp tables */
0998     struct radeon_bo    *cp_table_obj;
0999     uint64_t        cp_table_gpu_addr;
1000     volatile uint32_t   *cp_table_ptr;
1001     u32                     cp_table_size;
1002 };
1003 
1004 int radeon_ib_get(struct radeon_device *rdev, int ring,
1005           struct radeon_ib *ib, struct radeon_vm *vm,
1006           unsigned size);
1007 void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib *ib);
1008 int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib,
1009                struct radeon_ib *const_ib, bool hdp_flush);
1010 int radeon_ib_pool_init(struct radeon_device *rdev);
1011 void radeon_ib_pool_fini(struct radeon_device *rdev);
1012 int radeon_ib_ring_tests(struct radeon_device *rdev);
1013 /* Ring access between begin & end cannot sleep */
1014 bool radeon_ring_supports_scratch_reg(struct radeon_device *rdev,
1015                       struct radeon_ring *ring);
1016 void radeon_ring_free_size(struct radeon_device *rdev, struct radeon_ring *cp);
1017 int radeon_ring_alloc(struct radeon_device *rdev, struct radeon_ring *cp, unsigned ndw);
1018 int radeon_ring_lock(struct radeon_device *rdev, struct radeon_ring *cp, unsigned ndw);
1019 void radeon_ring_commit(struct radeon_device *rdev, struct radeon_ring *cp,
1020             bool hdp_flush);
1021 void radeon_ring_unlock_commit(struct radeon_device *rdev, struct radeon_ring *cp,
1022                    bool hdp_flush);
1023 void radeon_ring_undo(struct radeon_ring *ring);
1024 void radeon_ring_unlock_undo(struct radeon_device *rdev, struct radeon_ring *cp);
1025 int radeon_ring_test(struct radeon_device *rdev, struct radeon_ring *cp);
1026 void radeon_ring_lockup_update(struct radeon_device *rdev,
1027                    struct radeon_ring *ring);
1028 bool radeon_ring_test_lockup(struct radeon_device *rdev, struct radeon_ring *ring);
1029 unsigned radeon_ring_backup(struct radeon_device *rdev, struct radeon_ring *ring,
1030                 uint32_t **data);
1031 int radeon_ring_restore(struct radeon_device *rdev, struct radeon_ring *ring,
1032             unsigned size, uint32_t *data);
1033 int radeon_ring_init(struct radeon_device *rdev, struct radeon_ring *cp, unsigned ring_size,
1034              unsigned rptr_offs, u32 nop);
1035 void radeon_ring_fini(struct radeon_device *rdev, struct radeon_ring *cp);
1036 
1037 
1038 /* r600 async dma */
1039 void r600_dma_stop(struct radeon_device *rdev);
1040 int r600_dma_resume(struct radeon_device *rdev);
1041 void r600_dma_fini(struct radeon_device *rdev);
1042 
1043 void cayman_dma_stop(struct radeon_device *rdev);
1044 int cayman_dma_resume(struct radeon_device *rdev);
1045 void cayman_dma_fini(struct radeon_device *rdev);
1046 
1047 /*
1048  * CS.
1049  */
1050 struct radeon_cs_chunk {
1051     uint32_t        length_dw;
1052     uint32_t        *kdata;
1053     void __user     *user_ptr;
1054 };
1055 
1056 struct radeon_cs_parser {
1057     struct device       *dev;
1058     struct radeon_device    *rdev;
1059     struct drm_file     *filp;
1060     /* chunks */
1061     unsigned        nchunks;
1062     struct radeon_cs_chunk  *chunks;
1063     uint64_t        *chunks_array;
1064     /* IB */
1065     unsigned        idx;
1066     /* relocations */
1067     unsigned        nrelocs;
1068     struct radeon_bo_list   *relocs;
1069     struct radeon_bo_list   *vm_bos;
1070     struct list_head    validated;
1071     unsigned        dma_reloc_idx;
1072     /* indices of various chunks */
1073     struct radeon_cs_chunk  *chunk_ib;
1074     struct radeon_cs_chunk  *chunk_relocs;
1075     struct radeon_cs_chunk  *chunk_flags;
1076     struct radeon_cs_chunk  *chunk_const_ib;
1077     struct radeon_ib    ib;
1078     struct radeon_ib    const_ib;
1079     void            *track;
1080     unsigned        family;
1081     int         parser_error;
1082     u32         cs_flags;
1083     u32         ring;
1084     s32         priority;
1085     struct ww_acquire_ctx   ticket;
1086 };
1087 
1088 static inline u32 radeon_get_ib_value(struct radeon_cs_parser *p, int idx)
1089 {
1090     struct radeon_cs_chunk *ibc = p->chunk_ib;
1091 
1092     if (ibc->kdata)
1093         return ibc->kdata[idx];
1094     return p->ib.ptr[idx];
1095 }
1096 
1097 
1098 struct radeon_cs_packet {
1099     unsigned    idx;
1100     unsigned    type;
1101     unsigned    reg;
1102     unsigned    opcode;
1103     int     count;
1104     unsigned    one_reg_wr;
1105 };
1106 
1107 typedef int (*radeon_packet0_check_t)(struct radeon_cs_parser *p,
1108                       struct radeon_cs_packet *pkt,
1109                       unsigned idx, unsigned reg);
1110 
1111 /*
1112  * AGP
1113  */
1114 
1115 struct radeon_agp_mode {
1116     unsigned long mode; /**< AGP mode */
1117 };
1118 
1119 struct radeon_agp_info {
1120     int agp_version_major;
1121     int agp_version_minor;
1122     unsigned long mode;
1123     unsigned long aperture_base;    /* physical address */
1124     unsigned long aperture_size;    /* bytes */
1125     unsigned long memory_allowed;   /* bytes */
1126     unsigned long memory_used;
1127 
1128     /* PCI information */
1129     unsigned short id_vendor;
1130     unsigned short id_device;
1131 };
1132 
1133 struct radeon_agp_head {
1134     struct agp_kern_info agp_info;
1135     struct list_head memory;
1136     unsigned long mode;
1137     struct agp_bridge_data *bridge;
1138     int enabled;
1139     int acquired;
1140     unsigned long base;
1141     int agp_mtrr;
1142     int cant_use_aperture;
1143     unsigned long page_mask;
1144 };
1145 
1146 #if IS_ENABLED(CONFIG_AGP)
1147 struct radeon_agp_head *radeon_agp_head_init(struct drm_device *dev);
1148 #else
1149 static inline struct radeon_agp_head *radeon_agp_head_init(struct drm_device *dev)
1150 {
1151     return NULL;
1152 }
1153 #endif
1154 int radeon_agp_init(struct radeon_device *rdev);
1155 void radeon_agp_resume(struct radeon_device *rdev);
1156 void radeon_agp_suspend(struct radeon_device *rdev);
1157 void radeon_agp_fini(struct radeon_device *rdev);
1158 
1159 
1160 /*
1161  * Writeback
1162  */
1163 struct radeon_wb {
1164     struct radeon_bo    *wb_obj;
1165     volatile uint32_t   *wb;
1166     uint64_t        gpu_addr;
1167     bool                    enabled;
1168     bool                    use_event;
1169 };
1170 
1171 #define RADEON_WB_SCRATCH_OFFSET 0
1172 #define RADEON_WB_RING0_NEXT_RPTR 256
1173 #define RADEON_WB_CP_RPTR_OFFSET 1024
1174 #define RADEON_WB_CP1_RPTR_OFFSET 1280
1175 #define RADEON_WB_CP2_RPTR_OFFSET 1536
1176 #define R600_WB_DMA_RPTR_OFFSET   1792
1177 #define R600_WB_IH_WPTR_OFFSET   2048
1178 #define CAYMAN_WB_DMA1_RPTR_OFFSET   2304
1179 #define R600_WB_EVENT_OFFSET     3072
1180 #define CIK_WB_CP1_WPTR_OFFSET     3328
1181 #define CIK_WB_CP2_WPTR_OFFSET     3584
1182 #define R600_WB_DMA_RING_TEST_OFFSET 3588
1183 #define CAYMAN_WB_DMA1_RING_TEST_OFFSET 3592
1184 
1185 /**
1186  * struct radeon_pm - power management datas
1187  * @max_bandwidth:      maximum bandwidth the gpu has (MByte/s)
1188  * @igp_sideport_mclk:  sideport memory clock Mhz (rs690,rs740,rs780,rs880)
1189  * @igp_system_mclk:    system clock Mhz (rs690,rs740,rs780,rs880)
1190  * @igp_ht_link_clk:    ht link clock Mhz (rs690,rs740,rs780,rs880)
1191  * @igp_ht_link_width:  ht link width in bits (rs690,rs740,rs780,rs880)
1192  * @k8_bandwidth:       k8 bandwidth the gpu has (MByte/s) (IGP)
1193  * @sideport_bandwidth: sideport bandwidth the gpu has (MByte/s) (IGP)
1194  * @ht_bandwidth:       ht bandwidth the gpu has (MByte/s) (IGP)
1195  * @core_bandwidth:     core GPU bandwidth the gpu has (MByte/s) (IGP)
1196  * @sclk:           GPU clock Mhz (core bandwidth depends of this clock)
1197  * @needed_bandwidth:   current bandwidth needs
1198  *
1199  * It keeps track of various data needed to take powermanagement decision.
1200  * Bandwidth need is used to determine minimun clock of the GPU and memory.
1201  * Equation between gpu/memory clock and available bandwidth is hw dependent
1202  * (type of memory, bus size, efficiency, ...)
1203  */
1204 
1205 enum radeon_pm_method {
1206     PM_METHOD_PROFILE,
1207     PM_METHOD_DYNPM,
1208     PM_METHOD_DPM,
1209 };
1210 
1211 enum radeon_dynpm_state {
1212     DYNPM_STATE_DISABLED,
1213     DYNPM_STATE_MINIMUM,
1214     DYNPM_STATE_PAUSED,
1215     DYNPM_STATE_ACTIVE,
1216     DYNPM_STATE_SUSPENDED,
1217 };
1218 enum radeon_dynpm_action {
1219     DYNPM_ACTION_NONE,
1220     DYNPM_ACTION_MINIMUM,
1221     DYNPM_ACTION_DOWNCLOCK,
1222     DYNPM_ACTION_UPCLOCK,
1223     DYNPM_ACTION_DEFAULT
1224 };
1225 
1226 enum radeon_voltage_type {
1227     VOLTAGE_NONE = 0,
1228     VOLTAGE_GPIO,
1229     VOLTAGE_VDDC,
1230     VOLTAGE_SW
1231 };
1232 
1233 enum radeon_pm_state_type {
1234     /* not used for dpm */
1235     POWER_STATE_TYPE_DEFAULT,
1236     POWER_STATE_TYPE_POWERSAVE,
1237     /* user selectable states */
1238     POWER_STATE_TYPE_BATTERY,
1239     POWER_STATE_TYPE_BALANCED,
1240     POWER_STATE_TYPE_PERFORMANCE,
1241     /* internal states */
1242     POWER_STATE_TYPE_INTERNAL_UVD,
1243     POWER_STATE_TYPE_INTERNAL_UVD_SD,
1244     POWER_STATE_TYPE_INTERNAL_UVD_HD,
1245     POWER_STATE_TYPE_INTERNAL_UVD_HD2,
1246     POWER_STATE_TYPE_INTERNAL_UVD_MVC,
1247     POWER_STATE_TYPE_INTERNAL_BOOT,
1248     POWER_STATE_TYPE_INTERNAL_THERMAL,
1249     POWER_STATE_TYPE_INTERNAL_ACPI,
1250     POWER_STATE_TYPE_INTERNAL_ULV,
1251     POWER_STATE_TYPE_INTERNAL_3DPERF,
1252 };
1253 
1254 enum radeon_pm_profile_type {
1255     PM_PROFILE_DEFAULT,
1256     PM_PROFILE_AUTO,
1257     PM_PROFILE_LOW,
1258     PM_PROFILE_MID,
1259     PM_PROFILE_HIGH,
1260 };
1261 
1262 #define PM_PROFILE_DEFAULT_IDX 0
1263 #define PM_PROFILE_LOW_SH_IDX  1
1264 #define PM_PROFILE_MID_SH_IDX  2
1265 #define PM_PROFILE_HIGH_SH_IDX 3
1266 #define PM_PROFILE_LOW_MH_IDX  4
1267 #define PM_PROFILE_MID_MH_IDX  5
1268 #define PM_PROFILE_HIGH_MH_IDX 6
1269 #define PM_PROFILE_MAX         7
1270 
1271 struct radeon_pm_profile {
1272     int dpms_off_ps_idx;
1273     int dpms_on_ps_idx;
1274     int dpms_off_cm_idx;
1275     int dpms_on_cm_idx;
1276 };
1277 
1278 enum radeon_int_thermal_type {
1279     THERMAL_TYPE_NONE,
1280     THERMAL_TYPE_EXTERNAL,
1281     THERMAL_TYPE_EXTERNAL_GPIO,
1282     THERMAL_TYPE_RV6XX,
1283     THERMAL_TYPE_RV770,
1284     THERMAL_TYPE_ADT7473_WITH_INTERNAL,
1285     THERMAL_TYPE_EVERGREEN,
1286     THERMAL_TYPE_SUMO,
1287     THERMAL_TYPE_NI,
1288     THERMAL_TYPE_SI,
1289     THERMAL_TYPE_EMC2103_WITH_INTERNAL,
1290     THERMAL_TYPE_CI,
1291     THERMAL_TYPE_KV,
1292 };
1293 
1294 struct radeon_voltage {
1295     enum radeon_voltage_type type;
1296     /* gpio voltage */
1297     struct radeon_gpio_rec gpio;
1298     u32 delay; /* delay in usec from voltage drop to sclk change */
1299     bool active_high; /* voltage drop is active when bit is high */
1300     /* VDDC voltage */
1301     u8 vddc_id; /* index into vddc voltage table */
1302     u8 vddci_id; /* index into vddci voltage table */
1303     bool vddci_enabled;
1304     /* r6xx+ sw */
1305     u16 voltage;
1306     /* evergreen+ vddci */
1307     u16 vddci;
1308 };
1309 
1310 /* clock mode flags */
1311 #define RADEON_PM_MODE_NO_DISPLAY          (1 << 0)
1312 
1313 struct radeon_pm_clock_info {
1314     /* memory clock */
1315     u32 mclk;
1316     /* engine clock */
1317     u32 sclk;
1318     /* voltage info */
1319     struct radeon_voltage voltage;
1320     /* standardized clock flags */
1321     u32 flags;
1322 };
1323 
1324 /* state flags */
1325 #define RADEON_PM_STATE_SINGLE_DISPLAY_ONLY (1 << 0)
1326 
1327 struct radeon_power_state {
1328     enum radeon_pm_state_type type;
1329     struct radeon_pm_clock_info *clock_info;
1330     /* number of valid clock modes in this power state */
1331     int num_clock_modes;
1332     struct radeon_pm_clock_info *default_clock_mode;
1333     /* standardized state flags */
1334     u32 flags;
1335     u32 misc; /* vbios specific flags */
1336     u32 misc2; /* vbios specific flags */
1337     int pcie_lanes; /* pcie lanes */
1338 };
1339 
1340 /*
1341  * Some modes are overclocked by very low value, accept them
1342  */
1343 #define RADEON_MODE_OVERCLOCK_MARGIN 500 /* 5 MHz */
1344 
1345 enum radeon_dpm_auto_throttle_src {
1346     RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL,
1347     RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL
1348 };
1349 
1350 enum radeon_dpm_event_src {
1351     RADEON_DPM_EVENT_SRC_ANALOG = 0,
1352     RADEON_DPM_EVENT_SRC_EXTERNAL = 1,
1353     RADEON_DPM_EVENT_SRC_DIGITAL = 2,
1354     RADEON_DPM_EVENT_SRC_ANALOG_OR_EXTERNAL = 3,
1355     RADEON_DPM_EVENT_SRC_DIGIAL_OR_EXTERNAL = 4
1356 };
1357 
1358 #define RADEON_MAX_VCE_LEVELS 6
1359 
1360 enum radeon_vce_level {
1361     RADEON_VCE_LEVEL_AC_ALL = 0,     /* AC, All cases */
1362     RADEON_VCE_LEVEL_DC_EE = 1,      /* DC, entropy encoding */
1363     RADEON_VCE_LEVEL_DC_LL_LOW = 2,  /* DC, low latency queue, res <= 720 */
1364     RADEON_VCE_LEVEL_DC_LL_HIGH = 3, /* DC, low latency queue, 1080 >= res > 720 */
1365     RADEON_VCE_LEVEL_DC_GP_LOW = 4,  /* DC, general purpose queue, res <= 720 */
1366     RADEON_VCE_LEVEL_DC_GP_HIGH = 5, /* DC, general purpose queue, 1080 >= res > 720 */
1367 };
1368 
1369 struct radeon_ps {
1370     u32 caps; /* vbios flags */
1371     u32 class; /* vbios flags */
1372     u32 class2; /* vbios flags */
1373     /* UVD clocks */
1374     u32 vclk;
1375     u32 dclk;
1376     /* VCE clocks */
1377     u32 evclk;
1378     u32 ecclk;
1379     bool vce_active;
1380     enum radeon_vce_level vce_level;
1381     /* asic priv */
1382     void *ps_priv;
1383 };
1384 
1385 struct radeon_dpm_thermal {
1386     /* thermal interrupt work */
1387     struct work_struct work;
1388     /* low temperature threshold */
1389     int                min_temp;
1390     /* high temperature threshold */
1391     int                max_temp;
1392     /* was interrupt low to high or high to low */
1393     bool               high_to_low;
1394 };
1395 
1396 enum radeon_clk_action
1397 {
1398     RADEON_SCLK_UP = 1,
1399     RADEON_SCLK_DOWN
1400 };
1401 
1402 struct radeon_blacklist_clocks
1403 {
1404     u32 sclk;
1405     u32 mclk;
1406     enum radeon_clk_action action;
1407 };
1408 
1409 struct radeon_clock_and_voltage_limits {
1410     u32 sclk;
1411     u32 mclk;
1412     u16 vddc;
1413     u16 vddci;
1414 };
1415 
1416 struct radeon_clock_array {
1417     u32 count;
1418     u32 *values;
1419 };
1420 
1421 struct radeon_clock_voltage_dependency_entry {
1422     u32 clk;
1423     u16 v;
1424 };
1425 
1426 struct radeon_clock_voltage_dependency_table {
1427     u32 count;
1428     struct radeon_clock_voltage_dependency_entry *entries;
1429 };
1430 
1431 union radeon_cac_leakage_entry {
1432     struct {
1433         u16 vddc;
1434         u32 leakage;
1435     };
1436     struct {
1437         u16 vddc1;
1438         u16 vddc2;
1439         u16 vddc3;
1440     };
1441 };
1442 
1443 struct radeon_cac_leakage_table {
1444     u32 count;
1445     union radeon_cac_leakage_entry *entries;
1446 };
1447 
1448 struct radeon_phase_shedding_limits_entry {
1449     u16 voltage;
1450     u32 sclk;
1451     u32 mclk;
1452 };
1453 
1454 struct radeon_phase_shedding_limits_table {
1455     u32 count;
1456     struct radeon_phase_shedding_limits_entry *entries;
1457 };
1458 
1459 struct radeon_uvd_clock_voltage_dependency_entry {
1460     u32 vclk;
1461     u32 dclk;
1462     u16 v;
1463 };
1464 
1465 struct radeon_uvd_clock_voltage_dependency_table {
1466     u8 count;
1467     struct radeon_uvd_clock_voltage_dependency_entry *entries;
1468 };
1469 
1470 struct radeon_vce_clock_voltage_dependency_entry {
1471     u32 ecclk;
1472     u32 evclk;
1473     u16 v;
1474 };
1475 
1476 struct radeon_vce_clock_voltage_dependency_table {
1477     u8 count;
1478     struct radeon_vce_clock_voltage_dependency_entry *entries;
1479 };
1480 
1481 struct radeon_ppm_table {
1482     u8 ppm_design;
1483     u16 cpu_core_number;
1484     u32 platform_tdp;
1485     u32 small_ac_platform_tdp;
1486     u32 platform_tdc;
1487     u32 small_ac_platform_tdc;
1488     u32 apu_tdp;
1489     u32 dgpu_tdp;
1490     u32 dgpu_ulv_power;
1491     u32 tj_max;
1492 };
1493 
1494 struct radeon_cac_tdp_table {
1495     u16 tdp;
1496     u16 configurable_tdp;
1497     u16 tdc;
1498     u16 battery_power_limit;
1499     u16 small_power_limit;
1500     u16 low_cac_leakage;
1501     u16 high_cac_leakage;
1502     u16 maximum_power_delivery_limit;
1503 };
1504 
1505 struct radeon_dpm_dynamic_state {
1506     struct radeon_clock_voltage_dependency_table vddc_dependency_on_sclk;
1507     struct radeon_clock_voltage_dependency_table vddci_dependency_on_mclk;
1508     struct radeon_clock_voltage_dependency_table vddc_dependency_on_mclk;
1509     struct radeon_clock_voltage_dependency_table mvdd_dependency_on_mclk;
1510     struct radeon_clock_voltage_dependency_table vddc_dependency_on_dispclk;
1511     struct radeon_uvd_clock_voltage_dependency_table uvd_clock_voltage_dependency_table;
1512     struct radeon_vce_clock_voltage_dependency_table vce_clock_voltage_dependency_table;
1513     struct radeon_clock_voltage_dependency_table samu_clock_voltage_dependency_table;
1514     struct radeon_clock_voltage_dependency_table acp_clock_voltage_dependency_table;
1515     struct radeon_clock_array valid_sclk_values;
1516     struct radeon_clock_array valid_mclk_values;
1517     struct radeon_clock_and_voltage_limits max_clock_voltage_on_dc;
1518     struct radeon_clock_and_voltage_limits max_clock_voltage_on_ac;
1519     u32 mclk_sclk_ratio;
1520     u32 sclk_mclk_delta;
1521     u16 vddc_vddci_delta;
1522     u16 min_vddc_for_pcie_gen2;
1523     struct radeon_cac_leakage_table cac_leakage_table;
1524     struct radeon_phase_shedding_limits_table phase_shedding_limits_table;
1525     struct radeon_ppm_table *ppm_table;
1526     struct radeon_cac_tdp_table *cac_tdp_table;
1527 };
1528 
1529 struct radeon_dpm_fan {
1530     u16 t_min;
1531     u16 t_med;
1532     u16 t_high;
1533     u16 pwm_min;
1534     u16 pwm_med;
1535     u16 pwm_high;
1536     u8 t_hyst;
1537     u32 cycle_delay;
1538     u16 t_max;
1539     u8 control_mode;
1540     u16 default_max_fan_pwm;
1541     u16 default_fan_output_sensitivity;
1542     u16 fan_output_sensitivity;
1543     bool ucode_fan_control;
1544 };
1545 
1546 enum radeon_pcie_gen {
1547     RADEON_PCIE_GEN1 = 0,
1548     RADEON_PCIE_GEN2 = 1,
1549     RADEON_PCIE_GEN3 = 2,
1550     RADEON_PCIE_GEN_INVALID = 0xffff
1551 };
1552 
1553 enum radeon_dpm_forced_level {
1554     RADEON_DPM_FORCED_LEVEL_AUTO = 0,
1555     RADEON_DPM_FORCED_LEVEL_LOW = 1,
1556     RADEON_DPM_FORCED_LEVEL_HIGH = 2,
1557 };
1558 
1559 struct radeon_vce_state {
1560     /* vce clocks */
1561     u32 evclk;
1562     u32 ecclk;
1563     /* gpu clocks */
1564     u32 sclk;
1565     u32 mclk;
1566     u8 clk_idx;
1567     u8 pstate;
1568 };
1569 
1570 struct radeon_dpm {
1571     struct radeon_ps        *ps;
1572     /* number of valid power states */
1573     int                     num_ps;
1574     /* current power state that is active */
1575     struct radeon_ps        *current_ps;
1576     /* requested power state */
1577     struct radeon_ps        *requested_ps;
1578     /* boot up power state */
1579     struct radeon_ps        *boot_ps;
1580     /* default uvd power state */
1581     struct radeon_ps        *uvd_ps;
1582     /* vce requirements */
1583     struct radeon_vce_state vce_states[RADEON_MAX_VCE_LEVELS];
1584     enum radeon_vce_level vce_level;
1585     enum radeon_pm_state_type state;
1586     enum radeon_pm_state_type user_state;
1587     u32                     platform_caps;
1588     u32                     voltage_response_time;
1589     u32                     backbias_response_time;
1590     void                    *priv;
1591     u32         new_active_crtcs;
1592     int         new_active_crtc_count;
1593     int         high_pixelclock_count;
1594     u32         current_active_crtcs;
1595     int         current_active_crtc_count;
1596     bool single_display;
1597     struct radeon_dpm_dynamic_state dyn_state;
1598     struct radeon_dpm_fan fan;
1599     u32 tdp_limit;
1600     u32 near_tdp_limit;
1601     u32 near_tdp_limit_adjusted;
1602     u32 sq_ramping_threshold;
1603     u32 cac_leakage;
1604     u16 tdp_od_limit;
1605     u32 tdp_adjustment;
1606     u16 load_line_slope;
1607     bool power_control;
1608     bool ac_power;
1609     /* special states active */
1610     bool                    thermal_active;
1611     bool                    uvd_active;
1612     bool                    vce_active;
1613     /* thermal handling */
1614     struct radeon_dpm_thermal thermal;
1615     /* forced levels */
1616     enum radeon_dpm_forced_level forced_level;
1617     /* track UVD streams */
1618     unsigned sd;
1619     unsigned hd;
1620 };
1621 
1622 void radeon_dpm_enable_uvd(struct radeon_device *rdev, bool enable);
1623 void radeon_dpm_enable_vce(struct radeon_device *rdev, bool enable);
1624 
1625 struct radeon_pm {
1626     struct mutex        mutex;
1627     /* write locked while reprogramming mclk */
1628     struct rw_semaphore mclk_lock;
1629     u32         active_crtcs;
1630     int         active_crtc_count;
1631     int         req_vblank;
1632     bool            vblank_sync;
1633     fixed20_12      max_bandwidth;
1634     fixed20_12      igp_sideport_mclk;
1635     fixed20_12      igp_system_mclk;
1636     fixed20_12      igp_ht_link_clk;
1637     fixed20_12      igp_ht_link_width;
1638     fixed20_12      k8_bandwidth;
1639     fixed20_12      sideport_bandwidth;
1640     fixed20_12      ht_bandwidth;
1641     fixed20_12      core_bandwidth;
1642     fixed20_12      sclk;
1643     fixed20_12      mclk;
1644     fixed20_12      needed_bandwidth;
1645     struct radeon_power_state *power_state;
1646     /* number of valid power states */
1647     int                     num_power_states;
1648     int                     current_power_state_index;
1649     int                     current_clock_mode_index;
1650     int                     requested_power_state_index;
1651     int                     requested_clock_mode_index;
1652     int                     default_power_state_index;
1653     u32                     current_sclk;
1654     u32                     current_mclk;
1655     u16                     current_vddc;
1656     u16                     current_vddci;
1657     u32                     default_sclk;
1658     u32                     default_mclk;
1659     u16                     default_vddc;
1660     u16                     default_vddci;
1661     struct radeon_i2c_chan *i2c_bus;
1662     /* selected pm method */
1663     enum radeon_pm_method     pm_method;
1664     /* dynpm power management */
1665     struct delayed_work dynpm_idle_work;
1666     enum radeon_dynpm_state dynpm_state;
1667     enum radeon_dynpm_action    dynpm_planned_action;
1668     unsigned long       dynpm_action_timeout;
1669     bool                    dynpm_can_upclock;
1670     bool                    dynpm_can_downclock;
1671     /* profile-based power management */
1672     enum radeon_pm_profile_type profile;
1673     int                     profile_index;
1674     struct radeon_pm_profile profiles[PM_PROFILE_MAX];
1675     /* internal thermal controller on rv6xx+ */
1676     enum radeon_int_thermal_type int_thermal_type;
1677     struct device           *int_hwmon_dev;
1678     /* fan control parameters */
1679     bool                    no_fan;
1680     u8                      fan_pulses_per_revolution;
1681     u8                      fan_min_rpm;
1682     u8                      fan_max_rpm;
1683     /* dpm */
1684     bool                    dpm_enabled;
1685     bool                    sysfs_initialized;
1686     struct radeon_dpm       dpm;
1687 };
1688 
1689 #define RADEON_PCIE_SPEED_25 1
1690 #define RADEON_PCIE_SPEED_50 2
1691 #define RADEON_PCIE_SPEED_80 4
1692 
1693 int radeon_pm_get_type_index(struct radeon_device *rdev,
1694                  enum radeon_pm_state_type ps_type,
1695                  int instance);
1696 /*
1697  * UVD
1698  */
1699 #define RADEON_DEFAULT_UVD_HANDLES  10
1700 #define RADEON_MAX_UVD_HANDLES      30
1701 #define RADEON_UVD_STACK_SIZE       (200*1024)
1702 #define RADEON_UVD_HEAP_SIZE        (256*1024)
1703 #define RADEON_UVD_SESSION_SIZE     (50*1024)
1704 
1705 struct radeon_uvd {
1706     bool            fw_header_present;
1707     struct radeon_bo    *vcpu_bo;
1708     void            *cpu_addr;
1709     uint64_t        gpu_addr;
1710     unsigned        max_handles;
1711     atomic_t        handles[RADEON_MAX_UVD_HANDLES];
1712     struct drm_file     *filp[RADEON_MAX_UVD_HANDLES];
1713     unsigned        img_size[RADEON_MAX_UVD_HANDLES];
1714     struct delayed_work idle_work;
1715 };
1716 
1717 int radeon_uvd_init(struct radeon_device *rdev);
1718 void radeon_uvd_fini(struct radeon_device *rdev);
1719 int radeon_uvd_suspend(struct radeon_device *rdev);
1720 int radeon_uvd_resume(struct radeon_device *rdev);
1721 int radeon_uvd_get_create_msg(struct radeon_device *rdev, int ring,
1722                   uint32_t handle, struct radeon_fence **fence);
1723 int radeon_uvd_get_destroy_msg(struct radeon_device *rdev, int ring,
1724                    uint32_t handle, struct radeon_fence **fence);
1725 void radeon_uvd_force_into_uvd_segment(struct radeon_bo *rbo,
1726                        uint32_t allowed_domains);
1727 void radeon_uvd_free_handles(struct radeon_device *rdev,
1728                  struct drm_file *filp);
1729 int radeon_uvd_cs_parse(struct radeon_cs_parser *parser);
1730 void radeon_uvd_note_usage(struct radeon_device *rdev);
1731 int radeon_uvd_calc_upll_dividers(struct radeon_device *rdev,
1732                   unsigned vclk, unsigned dclk,
1733                   unsigned vco_min, unsigned vco_max,
1734                   unsigned fb_factor, unsigned fb_mask,
1735                   unsigned pd_min, unsigned pd_max,
1736                   unsigned pd_even,
1737                   unsigned *optimal_fb_div,
1738                   unsigned *optimal_vclk_div,
1739                   unsigned *optimal_dclk_div);
1740 int radeon_uvd_send_upll_ctlreq(struct radeon_device *rdev,
1741                                 unsigned cg_upll_func_cntl);
1742 
1743 /*
1744  * VCE
1745  */
1746 #define RADEON_MAX_VCE_HANDLES  16
1747 
1748 struct radeon_vce {
1749     struct radeon_bo    *vcpu_bo;
1750     uint64_t        gpu_addr;
1751     unsigned        fw_version;
1752     unsigned        fb_version;
1753     atomic_t        handles[RADEON_MAX_VCE_HANDLES];
1754     struct drm_file     *filp[RADEON_MAX_VCE_HANDLES];
1755     unsigned        img_size[RADEON_MAX_VCE_HANDLES];
1756     struct delayed_work idle_work;
1757     uint32_t        keyselect;
1758 };
1759 
1760 int radeon_vce_init(struct radeon_device *rdev);
1761 void radeon_vce_fini(struct radeon_device *rdev);
1762 int radeon_vce_suspend(struct radeon_device *rdev);
1763 int radeon_vce_resume(struct radeon_device *rdev);
1764 int radeon_vce_get_create_msg(struct radeon_device *rdev, int ring,
1765                   uint32_t handle, struct radeon_fence **fence);
1766 int radeon_vce_get_destroy_msg(struct radeon_device *rdev, int ring,
1767                    uint32_t handle, struct radeon_fence **fence);
1768 void radeon_vce_free_handles(struct radeon_device *rdev, struct drm_file *filp);
1769 void radeon_vce_note_usage(struct radeon_device *rdev);
1770 int radeon_vce_cs_reloc(struct radeon_cs_parser *p, int lo, int hi, unsigned size);
1771 int radeon_vce_cs_parse(struct radeon_cs_parser *p);
1772 bool radeon_vce_semaphore_emit(struct radeon_device *rdev,
1773                    struct radeon_ring *ring,
1774                    struct radeon_semaphore *semaphore,
1775                    bool emit_wait);
1776 void radeon_vce_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib);
1777 void radeon_vce_fence_emit(struct radeon_device *rdev,
1778                struct radeon_fence *fence);
1779 int radeon_vce_ring_test(struct radeon_device *rdev, struct radeon_ring *ring);
1780 int radeon_vce_ib_test(struct radeon_device *rdev, struct radeon_ring *ring);
1781 
1782 struct r600_audio_pin {
1783     int         channels;
1784     int         rate;
1785     int         bits_per_sample;
1786     u8          status_bits;
1787     u8          category_code;
1788     u32         offset;
1789     bool            connected;
1790     u32         id;
1791 };
1792 
1793 struct r600_audio {
1794     bool enabled;
1795     struct r600_audio_pin pin[RADEON_MAX_AFMT_BLOCKS];
1796     int num_pins;
1797     struct radeon_audio_funcs *hdmi_funcs;
1798     struct radeon_audio_funcs *dp_funcs;
1799     struct radeon_audio_basic_funcs *funcs;
1800 };
1801 
1802 /*
1803  * Benchmarking
1804  */
1805 void radeon_benchmark(struct radeon_device *rdev, int test_number);
1806 
1807 
1808 /*
1809  * Testing
1810  */
1811 void radeon_test_moves(struct radeon_device *rdev);
1812 void radeon_test_ring_sync(struct radeon_device *rdev,
1813                struct radeon_ring *cpA,
1814                struct radeon_ring *cpB);
1815 void radeon_test_syncing(struct radeon_device *rdev);
1816 
1817 /*
1818  * MMU Notifier
1819  */
1820 #if defined(CONFIG_MMU_NOTIFIER)
1821 int radeon_mn_register(struct radeon_bo *bo, unsigned long addr);
1822 void radeon_mn_unregister(struct radeon_bo *bo);
1823 #else
1824 static inline int radeon_mn_register(struct radeon_bo *bo, unsigned long addr)
1825 {
1826     return -ENODEV;
1827 }
1828 static inline void radeon_mn_unregister(struct radeon_bo *bo) {}
1829 #endif
1830 
1831 /*
1832  * Debugfs
1833  */
1834 void radeon_debugfs_fence_init(struct radeon_device *rdev);
1835 void radeon_gem_debugfs_init(struct radeon_device *rdev);
1836 
1837 /*
1838  * ASIC ring specific functions.
1839  */
1840 struct radeon_asic_ring {
1841     /* ring read/write ptr handling */
1842     u32 (*get_rptr)(struct radeon_device *rdev, struct radeon_ring *ring);
1843     u32 (*get_wptr)(struct radeon_device *rdev, struct radeon_ring *ring);
1844     void (*set_wptr)(struct radeon_device *rdev, struct radeon_ring *ring);
1845 
1846     /* validating and patching of IBs */
1847     int (*ib_parse)(struct radeon_device *rdev, struct radeon_ib *ib);
1848     int (*cs_parse)(struct radeon_cs_parser *p);
1849 
1850     /* command emmit functions */
1851     void (*ib_execute)(struct radeon_device *rdev, struct radeon_ib *ib);
1852     void (*emit_fence)(struct radeon_device *rdev, struct radeon_fence *fence);
1853     void (*hdp_flush)(struct radeon_device *rdev, struct radeon_ring *ring);
1854     bool (*emit_semaphore)(struct radeon_device *rdev, struct radeon_ring *cp,
1855                    struct radeon_semaphore *semaphore, bool emit_wait);
1856     void (*vm_flush)(struct radeon_device *rdev, struct radeon_ring *ring,
1857              unsigned vm_id, uint64_t pd_addr);
1858 
1859     /* testing functions */
1860     int (*ring_test)(struct radeon_device *rdev, struct radeon_ring *cp);
1861     int (*ib_test)(struct radeon_device *rdev, struct radeon_ring *cp);
1862     bool (*is_lockup)(struct radeon_device *rdev, struct radeon_ring *cp);
1863 
1864     /* deprecated */
1865     void (*ring_start)(struct radeon_device *rdev, struct radeon_ring *cp);
1866 };
1867 
1868 /*
1869  * ASIC specific functions.
1870  */
1871 struct radeon_asic {
1872     int (*init)(struct radeon_device *rdev);
1873     void (*fini)(struct radeon_device *rdev);
1874     int (*resume)(struct radeon_device *rdev);
1875     int (*suspend)(struct radeon_device *rdev);
1876     void (*vga_set_state)(struct radeon_device *rdev, bool state);
1877     int (*asic_reset)(struct radeon_device *rdev, bool hard);
1878     /* Flush the HDP cache via MMIO */
1879     void (*mmio_hdp_flush)(struct radeon_device *rdev);
1880     /* check if 3D engine is idle */
1881     bool (*gui_idle)(struct radeon_device *rdev);
1882     /* wait for mc_idle */
1883     int (*mc_wait_for_idle)(struct radeon_device *rdev);
1884     /* get the reference clock */
1885     u32 (*get_xclk)(struct radeon_device *rdev);
1886     /* get the gpu clock counter */
1887     uint64_t (*get_gpu_clock_counter)(struct radeon_device *rdev);
1888     /* get register for info ioctl */
1889     int (*get_allowed_info_register)(struct radeon_device *rdev, u32 reg, u32 *val);
1890     /* gart */
1891     struct {
1892         void (*tlb_flush)(struct radeon_device *rdev);
1893         uint64_t (*get_page_entry)(uint64_t addr, uint32_t flags);
1894         void (*set_page)(struct radeon_device *rdev, unsigned i,
1895                  uint64_t entry);
1896     } gart;
1897     struct {
1898         int (*init)(struct radeon_device *rdev);
1899         void (*fini)(struct radeon_device *rdev);
1900         void (*copy_pages)(struct radeon_device *rdev,
1901                    struct radeon_ib *ib,
1902                    uint64_t pe, uint64_t src,
1903                    unsigned count);
1904         void (*write_pages)(struct radeon_device *rdev,
1905                     struct radeon_ib *ib,
1906                     uint64_t pe,
1907                     uint64_t addr, unsigned count,
1908                     uint32_t incr, uint32_t flags);
1909         void (*set_pages)(struct radeon_device *rdev,
1910                   struct radeon_ib *ib,
1911                   uint64_t pe,
1912                   uint64_t addr, unsigned count,
1913                   uint32_t incr, uint32_t flags);
1914         void (*pad_ib)(struct radeon_ib *ib);
1915     } vm;
1916     /* ring specific callbacks */
1917     const struct radeon_asic_ring *ring[RADEON_NUM_RINGS];
1918     /* irqs */
1919     struct {
1920         int (*set)(struct radeon_device *rdev);
1921         int (*process)(struct radeon_device *rdev);
1922     } irq;
1923     /* displays */
1924     struct {
1925         /* display watermarks */
1926         void (*bandwidth_update)(struct radeon_device *rdev);
1927         /* get frame count */
1928         u32 (*get_vblank_counter)(struct radeon_device *rdev, int crtc);
1929         /* wait for vblank */
1930         void (*wait_for_vblank)(struct radeon_device *rdev, int crtc);
1931         /* set backlight level */
1932         void (*set_backlight_level)(struct radeon_encoder *radeon_encoder, u8 level);
1933         /* get backlight level */
1934         u8 (*get_backlight_level)(struct radeon_encoder *radeon_encoder);
1935         /* audio callbacks */
1936         void (*hdmi_enable)(struct drm_encoder *encoder, bool enable);
1937         void (*hdmi_setmode)(struct drm_encoder *encoder, struct drm_display_mode *mode);
1938     } display;
1939     /* copy functions for bo handling */
1940     struct {
1941         struct radeon_fence *(*blit)(struct radeon_device *rdev,
1942                          uint64_t src_offset,
1943                          uint64_t dst_offset,
1944                          unsigned num_gpu_pages,
1945                          struct dma_resv *resv);
1946         u32 blit_ring_index;
1947         struct radeon_fence *(*dma)(struct radeon_device *rdev,
1948                         uint64_t src_offset,
1949                         uint64_t dst_offset,
1950                         unsigned num_gpu_pages,
1951                         struct dma_resv *resv);
1952         u32 dma_ring_index;
1953         /* method used for bo copy */
1954         struct radeon_fence *(*copy)(struct radeon_device *rdev,
1955                          uint64_t src_offset,
1956                          uint64_t dst_offset,
1957                          unsigned num_gpu_pages,
1958                          struct dma_resv *resv);
1959         /* ring used for bo copies */
1960         u32 copy_ring_index;
1961     } copy;
1962     /* surfaces */
1963     struct {
1964         int (*set_reg)(struct radeon_device *rdev, int reg,
1965                        uint32_t tiling_flags, uint32_t pitch,
1966                        uint32_t offset, uint32_t obj_size);
1967         void (*clear_reg)(struct radeon_device *rdev, int reg);
1968     } surface;
1969     /* hotplug detect */
1970     struct {
1971         void (*init)(struct radeon_device *rdev);
1972         void (*fini)(struct radeon_device *rdev);
1973         bool (*sense)(struct radeon_device *rdev, enum radeon_hpd_id hpd);
1974         void (*set_polarity)(struct radeon_device *rdev, enum radeon_hpd_id hpd);
1975     } hpd;
1976     /* static power management */
1977     struct {
1978         void (*misc)(struct radeon_device *rdev);
1979         void (*prepare)(struct radeon_device *rdev);
1980         void (*finish)(struct radeon_device *rdev);
1981         void (*init_profile)(struct radeon_device *rdev);
1982         void (*get_dynpm_state)(struct radeon_device *rdev);
1983         uint32_t (*get_engine_clock)(struct radeon_device *rdev);
1984         void (*set_engine_clock)(struct radeon_device *rdev, uint32_t eng_clock);
1985         uint32_t (*get_memory_clock)(struct radeon_device *rdev);
1986         void (*set_memory_clock)(struct radeon_device *rdev, uint32_t mem_clock);
1987         int (*get_pcie_lanes)(struct radeon_device *rdev);
1988         void (*set_pcie_lanes)(struct radeon_device *rdev, int lanes);
1989         void (*set_clock_gating)(struct radeon_device *rdev, int enable);
1990         int (*set_uvd_clocks)(struct radeon_device *rdev, u32 vclk, u32 dclk);
1991         int (*set_vce_clocks)(struct radeon_device *rdev, u32 evclk, u32 ecclk);
1992         int (*get_temperature)(struct radeon_device *rdev);
1993     } pm;
1994     /* dynamic power management */
1995     struct {
1996         int (*init)(struct radeon_device *rdev);
1997         void (*setup_asic)(struct radeon_device *rdev);
1998         int (*enable)(struct radeon_device *rdev);
1999         int (*late_enable)(struct radeon_device *rdev);
2000         void (*disable)(struct radeon_device *rdev);
2001         int (*pre_set_power_state)(struct radeon_device *rdev);
2002         int (*set_power_state)(struct radeon_device *rdev);
2003         void (*post_set_power_state)(struct radeon_device *rdev);
2004         void (*display_configuration_changed)(struct radeon_device *rdev);
2005         void (*fini)(struct radeon_device *rdev);
2006         u32 (*get_sclk)(struct radeon_device *rdev, bool low);
2007         u32 (*get_mclk)(struct radeon_device *rdev, bool low);
2008         void (*print_power_state)(struct radeon_device *rdev, struct radeon_ps *ps);
2009         void (*debugfs_print_current_performance_level)(struct radeon_device *rdev, struct seq_file *m);
2010         int (*force_performance_level)(struct radeon_device *rdev, enum radeon_dpm_forced_level level);
2011         bool (*vblank_too_short)(struct radeon_device *rdev);
2012         void (*powergate_uvd)(struct radeon_device *rdev, bool gate);
2013         void (*enable_bapm)(struct radeon_device *rdev, bool enable);
2014         void (*fan_ctrl_set_mode)(struct radeon_device *rdev, u32 mode);
2015         u32 (*fan_ctrl_get_mode)(struct radeon_device *rdev);
2016         int (*set_fan_speed_percent)(struct radeon_device *rdev, u32 speed);
2017         int (*get_fan_speed_percent)(struct radeon_device *rdev, u32 *speed);
2018         u32 (*get_current_sclk)(struct radeon_device *rdev);
2019         u32 (*get_current_mclk)(struct radeon_device *rdev);
2020         u16 (*get_current_vddc)(struct radeon_device *rdev);
2021     } dpm;
2022     /* pageflipping */
2023     struct {
2024         void (*page_flip)(struct radeon_device *rdev, int crtc, u64 crtc_base, bool async);
2025         bool (*page_flip_pending)(struct radeon_device *rdev, int crtc);
2026     } pflip;
2027 };
2028 
2029 /*
2030  * Asic structures
2031  */
2032 struct r100_asic {
2033     const unsigned      *reg_safe_bm;
2034     unsigned        reg_safe_bm_size;
2035     u32         hdp_cntl;
2036 };
2037 
2038 struct r300_asic {
2039     const unsigned      *reg_safe_bm;
2040     unsigned        reg_safe_bm_size;
2041     u32         resync_scratch;
2042     u32         hdp_cntl;
2043 };
2044 
2045 struct r600_asic {
2046     unsigned        max_pipes;
2047     unsigned        max_tile_pipes;
2048     unsigned        max_simds;
2049     unsigned        max_backends;
2050     unsigned        max_gprs;
2051     unsigned        max_threads;
2052     unsigned        max_stack_entries;
2053     unsigned        max_hw_contexts;
2054     unsigned        max_gs_threads;
2055     unsigned        sx_max_export_size;
2056     unsigned        sx_max_export_pos_size;
2057     unsigned        sx_max_export_smx_size;
2058     unsigned        sq_num_cf_insts;
2059     unsigned        tiling_nbanks;
2060     unsigned        tiling_npipes;
2061     unsigned        tiling_group_size;
2062     unsigned        tile_config;
2063     unsigned        backend_map;
2064     unsigned        active_simds;
2065 };
2066 
2067 struct rv770_asic {
2068     unsigned        max_pipes;
2069     unsigned        max_tile_pipes;
2070     unsigned        max_simds;
2071     unsigned        max_backends;
2072     unsigned        max_gprs;
2073     unsigned        max_threads;
2074     unsigned        max_stack_entries;
2075     unsigned        max_hw_contexts;
2076     unsigned        max_gs_threads;
2077     unsigned        sx_max_export_size;
2078     unsigned        sx_max_export_pos_size;
2079     unsigned        sx_max_export_smx_size;
2080     unsigned        sq_num_cf_insts;
2081     unsigned        sx_num_of_sets;
2082     unsigned        sc_prim_fifo_size;
2083     unsigned        sc_hiz_tile_fifo_size;
2084     unsigned        sc_earlyz_tile_fifo_fize;
2085     unsigned        tiling_nbanks;
2086     unsigned        tiling_npipes;
2087     unsigned        tiling_group_size;
2088     unsigned        tile_config;
2089     unsigned        backend_map;
2090     unsigned        active_simds;
2091 };
2092 
2093 struct evergreen_asic {
2094     unsigned num_ses;
2095     unsigned max_pipes;
2096     unsigned max_tile_pipes;
2097     unsigned max_simds;
2098     unsigned max_backends;
2099     unsigned max_gprs;
2100     unsigned max_threads;
2101     unsigned max_stack_entries;
2102     unsigned max_hw_contexts;
2103     unsigned max_gs_threads;
2104     unsigned sx_max_export_size;
2105     unsigned sx_max_export_pos_size;
2106     unsigned sx_max_export_smx_size;
2107     unsigned sq_num_cf_insts;
2108     unsigned sx_num_of_sets;
2109     unsigned sc_prim_fifo_size;
2110     unsigned sc_hiz_tile_fifo_size;
2111     unsigned sc_earlyz_tile_fifo_size;
2112     unsigned tiling_nbanks;
2113     unsigned tiling_npipes;
2114     unsigned tiling_group_size;
2115     unsigned tile_config;
2116     unsigned backend_map;
2117     unsigned active_simds;
2118 };
2119 
2120 struct cayman_asic {
2121     unsigned max_shader_engines;
2122     unsigned max_pipes_per_simd;
2123     unsigned max_tile_pipes;
2124     unsigned max_simds_per_se;
2125     unsigned max_backends_per_se;
2126     unsigned max_texture_channel_caches;
2127     unsigned max_gprs;
2128     unsigned max_threads;
2129     unsigned max_gs_threads;
2130     unsigned max_stack_entries;
2131     unsigned sx_num_of_sets;
2132     unsigned sx_max_export_size;
2133     unsigned sx_max_export_pos_size;
2134     unsigned sx_max_export_smx_size;
2135     unsigned max_hw_contexts;
2136     unsigned sq_num_cf_insts;
2137     unsigned sc_prim_fifo_size;
2138     unsigned sc_hiz_tile_fifo_size;
2139     unsigned sc_earlyz_tile_fifo_size;
2140 
2141     unsigned num_shader_engines;
2142     unsigned num_shader_pipes_per_simd;
2143     unsigned num_tile_pipes;
2144     unsigned num_simds_per_se;
2145     unsigned num_backends_per_se;
2146     unsigned backend_disable_mask_per_asic;
2147     unsigned backend_map;
2148     unsigned num_texture_channel_caches;
2149     unsigned mem_max_burst_length_bytes;
2150     unsigned mem_row_size_in_kb;
2151     unsigned shader_engine_tile_size;
2152     unsigned num_gpus;
2153     unsigned multi_gpu_tile_size;
2154 
2155     unsigned tile_config;
2156     unsigned active_simds;
2157 };
2158 
2159 struct si_asic {
2160     unsigned max_shader_engines;
2161     unsigned max_tile_pipes;
2162     unsigned max_cu_per_sh;
2163     unsigned max_sh_per_se;
2164     unsigned max_backends_per_se;
2165     unsigned max_texture_channel_caches;
2166     unsigned max_gprs;
2167     unsigned max_gs_threads;
2168     unsigned max_hw_contexts;
2169     unsigned sc_prim_fifo_size_frontend;
2170     unsigned sc_prim_fifo_size_backend;
2171     unsigned sc_hiz_tile_fifo_size;
2172     unsigned sc_earlyz_tile_fifo_size;
2173 
2174     unsigned num_tile_pipes;
2175     unsigned backend_enable_mask;
2176     unsigned backend_disable_mask_per_asic;
2177     unsigned backend_map;
2178     unsigned num_texture_channel_caches;
2179     unsigned mem_max_burst_length_bytes;
2180     unsigned mem_row_size_in_kb;
2181     unsigned shader_engine_tile_size;
2182     unsigned num_gpus;
2183     unsigned multi_gpu_tile_size;
2184 
2185     unsigned tile_config;
2186     uint32_t tile_mode_array[32];
2187     uint32_t active_cus;
2188 };
2189 
2190 struct cik_asic {
2191     unsigned max_shader_engines;
2192     unsigned max_tile_pipes;
2193     unsigned max_cu_per_sh;
2194     unsigned max_sh_per_se;
2195     unsigned max_backends_per_se;
2196     unsigned max_texture_channel_caches;
2197     unsigned max_gprs;
2198     unsigned max_gs_threads;
2199     unsigned max_hw_contexts;
2200     unsigned sc_prim_fifo_size_frontend;
2201     unsigned sc_prim_fifo_size_backend;
2202     unsigned sc_hiz_tile_fifo_size;
2203     unsigned sc_earlyz_tile_fifo_size;
2204 
2205     unsigned num_tile_pipes;
2206     unsigned backend_enable_mask;
2207     unsigned backend_disable_mask_per_asic;
2208     unsigned backend_map;
2209     unsigned num_texture_channel_caches;
2210     unsigned mem_max_burst_length_bytes;
2211     unsigned mem_row_size_in_kb;
2212     unsigned shader_engine_tile_size;
2213     unsigned num_gpus;
2214     unsigned multi_gpu_tile_size;
2215 
2216     unsigned tile_config;
2217     uint32_t tile_mode_array[32];
2218     uint32_t macrotile_mode_array[16];
2219     uint32_t active_cus;
2220 };
2221 
2222 union radeon_asic_config {
2223     struct r300_asic    r300;
2224     struct r100_asic    r100;
2225     struct r600_asic    r600;
2226     struct rv770_asic   rv770;
2227     struct evergreen_asic   evergreen;
2228     struct cayman_asic  cayman;
2229     struct si_asic      si;
2230     struct cik_asic     cik;
2231 };
2232 
2233 /*
2234  * asic initizalization from radeon_asic.c
2235  */
2236 void radeon_agp_disable(struct radeon_device *rdev);
2237 int radeon_asic_init(struct radeon_device *rdev);
2238 
2239 
2240 /*
2241  * IOCTL.
2242  */
2243 int radeon_gem_info_ioctl(struct drm_device *dev, void *data,
2244               struct drm_file *filp);
2245 int radeon_gem_create_ioctl(struct drm_device *dev, void *data,
2246                 struct drm_file *filp);
2247 int radeon_gem_userptr_ioctl(struct drm_device *dev, void *data,
2248                  struct drm_file *filp);
2249 int radeon_gem_pin_ioctl(struct drm_device *dev, void *data,
2250              struct drm_file *file_priv);
2251 int radeon_gem_unpin_ioctl(struct drm_device *dev, void *data,
2252                struct drm_file *file_priv);
2253 int radeon_gem_pwrite_ioctl(struct drm_device *dev, void *data,
2254                 struct drm_file *file_priv);
2255 int radeon_gem_pread_ioctl(struct drm_device *dev, void *data,
2256                struct drm_file *file_priv);
2257 int radeon_gem_set_domain_ioctl(struct drm_device *dev, void *data,
2258                 struct drm_file *filp);
2259 int radeon_gem_mmap_ioctl(struct drm_device *dev, void *data,
2260               struct drm_file *filp);
2261 int radeon_gem_busy_ioctl(struct drm_device *dev, void *data,
2262               struct drm_file *filp);
2263 int radeon_gem_wait_idle_ioctl(struct drm_device *dev, void *data,
2264                   struct drm_file *filp);
2265 int radeon_gem_va_ioctl(struct drm_device *dev, void *data,
2266               struct drm_file *filp);
2267 int radeon_gem_op_ioctl(struct drm_device *dev, void *data,
2268             struct drm_file *filp);
2269 int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
2270 int radeon_gem_set_tiling_ioctl(struct drm_device *dev, void *data,
2271                 struct drm_file *filp);
2272 int radeon_gem_get_tiling_ioctl(struct drm_device *dev, void *data,
2273                 struct drm_file *filp);
2274 int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
2275 
2276 /* VRAM scratch page for HDP bug, default vram page */
2277 struct r600_vram_scratch {
2278     struct radeon_bo        *robj;
2279     volatile uint32_t       *ptr;
2280     u64             gpu_addr;
2281 };
2282 
2283 /*
2284  * ACPI
2285  */
2286 struct radeon_atif_notification_cfg {
2287     bool enabled;
2288     int command_code;
2289 };
2290 
2291 struct radeon_atif_notifications {
2292     bool display_switch;
2293     bool expansion_mode_change;
2294     bool thermal_state;
2295     bool forced_power_state;
2296     bool system_power_state;
2297     bool display_conf_change;
2298     bool px_gfx_switch;
2299     bool brightness_change;
2300     bool dgpu_display_event;
2301 };
2302 
2303 struct radeon_atif_functions {
2304     bool system_params;
2305     bool sbios_requests;
2306     bool select_active_disp;
2307     bool lid_state;
2308     bool get_tv_standard;
2309     bool set_tv_standard;
2310     bool get_panel_expansion_mode;
2311     bool set_panel_expansion_mode;
2312     bool temperature_change;
2313     bool graphics_device_types;
2314 };
2315 
2316 struct radeon_atif {
2317     struct radeon_atif_notifications notifications;
2318     struct radeon_atif_functions functions;
2319     struct radeon_atif_notification_cfg notification_cfg;
2320     struct radeon_encoder *encoder_for_bl;
2321 };
2322 
2323 struct radeon_atcs_functions {
2324     bool get_ext_state;
2325     bool pcie_perf_req;
2326     bool pcie_dev_rdy;
2327     bool pcie_bus_width;
2328 };
2329 
2330 struct radeon_atcs {
2331     struct radeon_atcs_functions functions;
2332 };
2333 
2334 /*
2335  * Core structure, functions and helpers.
2336  */
2337 typedef uint32_t (*radeon_rreg_t)(struct radeon_device*, uint32_t);
2338 typedef void (*radeon_wreg_t)(struct radeon_device*, uint32_t, uint32_t);
2339 
2340 struct radeon_device {
2341     struct device           *dev;
2342     struct drm_device       *ddev;
2343     struct pci_dev          *pdev;
2344 #ifdef __alpha__
2345     struct pci_controller       *hose;
2346 #endif
2347     struct radeon_agp_head      *agp;
2348     struct rw_semaphore     exclusive_lock;
2349     /* ASIC */
2350     union radeon_asic_config    config;
2351     enum radeon_family      family;
2352     unsigned long           flags;
2353     int             usec_timeout;
2354     enum radeon_pll_errata      pll_errata;
2355     int             num_gb_pipes;
2356     int             num_z_pipes;
2357     int             disp_priority;
2358     /* BIOS */
2359     uint8_t             *bios;
2360     bool                is_atom_bios;
2361     uint16_t            bios_header_start;
2362     struct radeon_bo        *stolen_vga_memory;
2363     /* Register mmio */
2364     resource_size_t         rmmio_base;
2365     resource_size_t         rmmio_size;
2366     /* protects concurrent MM_INDEX/DATA based register access */
2367     spinlock_t mmio_idx_lock;
2368     /* protects concurrent SMC based register access */
2369     spinlock_t smc_idx_lock;
2370     /* protects concurrent PLL register access */
2371     spinlock_t pll_idx_lock;
2372     /* protects concurrent MC register access */
2373     spinlock_t mc_idx_lock;
2374     /* protects concurrent PCIE register access */
2375     spinlock_t pcie_idx_lock;
2376     /* protects concurrent PCIE_PORT register access */
2377     spinlock_t pciep_idx_lock;
2378     /* protects concurrent PIF register access */
2379     spinlock_t pif_idx_lock;
2380     /* protects concurrent CG register access */
2381     spinlock_t cg_idx_lock;
2382     /* protects concurrent UVD register access */
2383     spinlock_t uvd_idx_lock;
2384     /* protects concurrent RCU register access */
2385     spinlock_t rcu_idx_lock;
2386     /* protects concurrent DIDT register access */
2387     spinlock_t didt_idx_lock;
2388     /* protects concurrent ENDPOINT (audio) register access */
2389     spinlock_t end_idx_lock;
2390     void __iomem            *rmmio;
2391     radeon_rreg_t           mc_rreg;
2392     radeon_wreg_t           mc_wreg;
2393     radeon_rreg_t           pll_rreg;
2394     radeon_wreg_t           pll_wreg;
2395     uint32_t                        pcie_reg_mask;
2396     radeon_rreg_t           pciep_rreg;
2397     radeon_wreg_t           pciep_wreg;
2398     /* io port */
2399     void __iomem                    *rio_mem;
2400     resource_size_t         rio_mem_size;
2401     struct radeon_clock             clock;
2402     struct radeon_mc        mc;
2403     struct radeon_gart      gart;
2404     struct radeon_mode_info     mode_info;
2405     struct radeon_scratch       scratch;
2406     struct radeon_doorbell      doorbell;
2407     struct radeon_mman      mman;
2408     struct radeon_fence_driver  fence_drv[RADEON_NUM_RINGS];
2409     wait_queue_head_t       fence_queue;
2410     u64             fence_context;
2411     struct mutex            ring_lock;
2412     struct radeon_ring      ring[RADEON_NUM_RINGS];
2413     bool                ib_pool_ready;
2414     struct radeon_sa_manager    ring_tmp_bo;
2415     struct radeon_irq       irq;
2416     struct radeon_asic      *asic;
2417     struct radeon_gem       gem;
2418     struct radeon_pm        pm;
2419     struct radeon_uvd       uvd;
2420     struct radeon_vce       vce;
2421     uint32_t            bios_scratch[RADEON_BIOS_NUM_SCRATCH];
2422     struct radeon_wb        wb;
2423     struct radeon_dummy_page    dummy_page;
2424     bool                shutdown;
2425     bool                need_swiotlb;
2426     bool                accel_working;
2427     bool                fastfb_working; /* IGP feature*/
2428     bool                needs_reset, in_reset;
2429     struct radeon_surface_reg surface_regs[RADEON_GEM_MAX_SURFACES];
2430     const struct firmware *me_fw;   /* all family ME firmware */
2431     const struct firmware *pfp_fw;  /* r6/700 PFP firmware */
2432     const struct firmware *rlc_fw;  /* r6/700 RLC firmware */
2433     const struct firmware *mc_fw;   /* NI MC firmware */
2434     const struct firmware *ce_fw;   /* SI CE firmware */
2435     const struct firmware *mec_fw;  /* CIK MEC firmware */
2436     const struct firmware *mec2_fw; /* KV MEC2 firmware */
2437     const struct firmware *sdma_fw; /* CIK SDMA firmware */
2438     const struct firmware *smc_fw;  /* SMC firmware */
2439     const struct firmware *uvd_fw;  /* UVD firmware */
2440     const struct firmware *vce_fw;  /* VCE firmware */
2441     bool new_fw;
2442     struct r600_vram_scratch vram_scratch;
2443     int msi_enabled; /* msi enabled */
2444     struct r600_ih ih; /* r6/700 interrupt ring */
2445     struct radeon_rlc rlc;
2446     struct radeon_mec mec;
2447     struct delayed_work hotplug_work;
2448     struct work_struct dp_work;
2449     struct work_struct audio_work;
2450     int num_crtc; /* number of crtcs */
2451     struct mutex dc_hw_i2c_mutex; /* display controller hw i2c mutex */
2452     bool has_uvd;
2453     bool has_vce;
2454     struct r600_audio audio; /* audio stuff */
2455     struct notifier_block acpi_nb;
2456     /* only one userspace can use Hyperz features or CMASK at a time */
2457     struct drm_file *hyperz_filp;
2458     struct drm_file *cmask_filp;
2459     /* i2c buses */
2460     struct radeon_i2c_chan *i2c_bus[RADEON_MAX_I2C_BUS];
2461     /* virtual memory */
2462     struct radeon_vm_manager    vm_manager;
2463     struct mutex            gpu_clock_mutex;
2464     /* memory stats */
2465     atomic64_t          num_bytes_moved;
2466     atomic_t            gpu_reset_counter;
2467     /* ACPI interface */
2468     struct radeon_atif      atif;
2469     struct radeon_atcs      atcs;
2470     /* srbm instance registers */
2471     struct mutex            srbm_mutex;
2472     /* clock, powergating flags */
2473     u32 cg_flags;
2474     u32 pg_flags;
2475 
2476     struct dev_pm_domain vga_pm_domain;
2477     bool have_disp_power_ref;
2478     u32 px_quirk_flags;
2479 
2480     /* tracking pinned memory */
2481     u64 vram_pin_size;
2482     u64 gart_pin_size;
2483 };
2484 
2485 bool radeon_is_px(struct drm_device *dev);
2486 int radeon_device_init(struct radeon_device *rdev,
2487                struct drm_device *ddev,
2488                struct pci_dev *pdev,
2489                uint32_t flags);
2490 void radeon_device_fini(struct radeon_device *rdev);
2491 int radeon_gpu_wait_for_idle(struct radeon_device *rdev);
2492 
2493 #define RADEON_MIN_MMIO_SIZE 0x10000
2494 
2495 uint32_t r100_mm_rreg_slow(struct radeon_device *rdev, uint32_t reg);
2496 void r100_mm_wreg_slow(struct radeon_device *rdev, uint32_t reg, uint32_t v);
2497 static inline uint32_t r100_mm_rreg(struct radeon_device *rdev, uint32_t reg,
2498                     bool always_indirect)
2499 {
2500     /* The mmio size is 64kb at minimum. Allows the if to be optimized out. */
2501     if ((reg < rdev->rmmio_size || reg < RADEON_MIN_MMIO_SIZE) && !always_indirect)
2502         return readl(((void __iomem *)rdev->rmmio) + reg);
2503     else
2504         return r100_mm_rreg_slow(rdev, reg);
2505 }
2506 static inline void r100_mm_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v,
2507                 bool always_indirect)
2508 {
2509     if ((reg < rdev->rmmio_size || reg < RADEON_MIN_MMIO_SIZE) && !always_indirect)
2510         writel(v, ((void __iomem *)rdev->rmmio) + reg);
2511     else
2512         r100_mm_wreg_slow(rdev, reg, v);
2513 }
2514 
2515 u32 r100_io_rreg(struct radeon_device *rdev, u32 reg);
2516 void r100_io_wreg(struct radeon_device *rdev, u32 reg, u32 v);
2517 
2518 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index);
2519 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v);
2520 
2521 /*
2522  * Cast helper
2523  */
2524 extern const struct dma_fence_ops radeon_fence_ops;
2525 
2526 static inline struct radeon_fence *to_radeon_fence(struct dma_fence *f)
2527 {
2528     struct radeon_fence *__f = container_of(f, struct radeon_fence, base);
2529 
2530     if (__f->base.ops == &radeon_fence_ops)
2531         return __f;
2532 
2533     return NULL;
2534 }
2535 
2536 /*
2537  * Registers read & write functions.
2538  */
2539 #define RREG8(reg) readb((rdev->rmmio) + (reg))
2540 #define WREG8(reg, v) writeb(v, (rdev->rmmio) + (reg))
2541 #define RREG16(reg) readw((rdev->rmmio) + (reg))
2542 #define WREG16(reg, v) writew(v, (rdev->rmmio) + (reg))
2543 #define RREG32(reg) r100_mm_rreg(rdev, (reg), false)
2544 #define RREG32_IDX(reg) r100_mm_rreg(rdev, (reg), true)
2545 #define DREG32(reg) pr_info("REGISTER: " #reg " : 0x%08X\n",    \
2546                 r100_mm_rreg(rdev, (reg), false))
2547 #define WREG32(reg, v) r100_mm_wreg(rdev, (reg), (v), false)
2548 #define WREG32_IDX(reg, v) r100_mm_wreg(rdev, (reg), (v), true)
2549 #define REG_SET(FIELD, v) (((v) << FIELD##_SHIFT) & FIELD##_MASK)
2550 #define REG_GET(FIELD, v) (((v) << FIELD##_SHIFT) & FIELD##_MASK)
2551 #define RREG32_PLL(reg) rdev->pll_rreg(rdev, (reg))
2552 #define WREG32_PLL(reg, v) rdev->pll_wreg(rdev, (reg), (v))
2553 #define RREG32_MC(reg) rdev->mc_rreg(rdev, (reg))
2554 #define WREG32_MC(reg, v) rdev->mc_wreg(rdev, (reg), (v))
2555 #define RREG32_PCIE(reg) rv370_pcie_rreg(rdev, (reg))
2556 #define WREG32_PCIE(reg, v) rv370_pcie_wreg(rdev, (reg), (v))
2557 #define RREG32_PCIE_PORT(reg) rdev->pciep_rreg(rdev, (reg))
2558 #define WREG32_PCIE_PORT(reg, v) rdev->pciep_wreg(rdev, (reg), (v))
2559 #define RREG32_SMC(reg) tn_smc_rreg(rdev, (reg))
2560 #define WREG32_SMC(reg, v) tn_smc_wreg(rdev, (reg), (v))
2561 #define RREG32_RCU(reg) r600_rcu_rreg(rdev, (reg))
2562 #define WREG32_RCU(reg, v) r600_rcu_wreg(rdev, (reg), (v))
2563 #define RREG32_CG(reg) eg_cg_rreg(rdev, (reg))
2564 #define WREG32_CG(reg, v) eg_cg_wreg(rdev, (reg), (v))
2565 #define RREG32_PIF_PHY0(reg) eg_pif_phy0_rreg(rdev, (reg))
2566 #define WREG32_PIF_PHY0(reg, v) eg_pif_phy0_wreg(rdev, (reg), (v))
2567 #define RREG32_PIF_PHY1(reg) eg_pif_phy1_rreg(rdev, (reg))
2568 #define WREG32_PIF_PHY1(reg, v) eg_pif_phy1_wreg(rdev, (reg), (v))
2569 #define RREG32_UVD_CTX(reg) r600_uvd_ctx_rreg(rdev, (reg))
2570 #define WREG32_UVD_CTX(reg, v) r600_uvd_ctx_wreg(rdev, (reg), (v))
2571 #define RREG32_DIDT(reg) cik_didt_rreg(rdev, (reg))
2572 #define WREG32_DIDT(reg, v) cik_didt_wreg(rdev, (reg), (v))
2573 #define WREG32_P(reg, val, mask)                \
2574     do {                            \
2575         uint32_t tmp_ = RREG32(reg);            \
2576         tmp_ &= (mask);                 \
2577         tmp_ |= ((val) & ~(mask));          \
2578         WREG32(reg, tmp_);              \
2579     } while (0)
2580 #define WREG32_AND(reg, and) WREG32_P(reg, 0, and)
2581 #define WREG32_OR(reg, or) WREG32_P(reg, or, ~(or))
2582 #define WREG32_PLL_P(reg, val, mask)                \
2583     do {                            \
2584         uint32_t tmp_ = RREG32_PLL(reg);        \
2585         tmp_ &= (mask);                 \
2586         tmp_ |= ((val) & ~(mask));          \
2587         WREG32_PLL(reg, tmp_);              \
2588     } while (0)
2589 #define WREG32_SMC_P(reg, val, mask)                \
2590     do {                            \
2591         uint32_t tmp_ = RREG32_SMC(reg);        \
2592         tmp_ &= (mask);                 \
2593         tmp_ |= ((val) & ~(mask));          \
2594         WREG32_SMC(reg, tmp_);              \
2595     } while (0)
2596 #define DREG32_SYS(sqf, rdev, reg) seq_printf((sqf), #reg " : 0x%08X\n", r100_mm_rreg((rdev), (reg), false))
2597 #define RREG32_IO(reg) r100_io_rreg(rdev, (reg))
2598 #define WREG32_IO(reg, v) r100_io_wreg(rdev, (reg), (v))
2599 
2600 #define RDOORBELL32(index) cik_mm_rdoorbell(rdev, (index))
2601 #define WDOORBELL32(index, v) cik_mm_wdoorbell(rdev, (index), (v))
2602 
2603 /*
2604  * Indirect registers accessors.
2605  * They used to be inlined, but this increases code size by ~65 kbytes.
2606  * Since each performs a pair of MMIO ops
2607  * within a spin_lock_irqsave/spin_unlock_irqrestore region,
2608  * the cost of call+ret is almost negligible. MMIO and locking
2609  * costs several dozens of cycles each at best, call+ret is ~5 cycles.
2610  */
2611 uint32_t rv370_pcie_rreg(struct radeon_device *rdev, uint32_t reg);
2612 void rv370_pcie_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v);
2613 u32 tn_smc_rreg(struct radeon_device *rdev, u32 reg);
2614 void tn_smc_wreg(struct radeon_device *rdev, u32 reg, u32 v);
2615 u32 r600_rcu_rreg(struct radeon_device *rdev, u32 reg);
2616 void r600_rcu_wreg(struct radeon_device *rdev, u32 reg, u32 v);
2617 u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg);
2618 void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v);
2619 u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg);
2620 void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v);
2621 u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg);
2622 void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v);
2623 u32 r600_uvd_ctx_rreg(struct radeon_device *rdev, u32 reg);
2624 void r600_uvd_ctx_wreg(struct radeon_device *rdev, u32 reg, u32 v);
2625 u32 cik_didt_rreg(struct radeon_device *rdev, u32 reg);
2626 void cik_didt_wreg(struct radeon_device *rdev, u32 reg, u32 v);
2627 
2628 void r100_pll_errata_after_index(struct radeon_device *rdev);
2629 
2630 
2631 /*
2632  * ASICs helpers.
2633  */
2634 #define ASIC_IS_RN50(rdev) ((rdev->pdev->device == 0x515e) || \
2635                 (rdev->pdev->device == 0x5969))
2636 #define ASIC_IS_RV100(rdev) ((rdev->family == CHIP_RV100) || \
2637         (rdev->family == CHIP_RV200) || \
2638         (rdev->family == CHIP_RS100) || \
2639         (rdev->family == CHIP_RS200) || \
2640         (rdev->family == CHIP_RV250) || \
2641         (rdev->family == CHIP_RV280) || \
2642         (rdev->family == CHIP_RS300))
2643 #define ASIC_IS_R300(rdev) ((rdev->family == CHIP_R300)  || \
2644         (rdev->family == CHIP_RV350) ||         \
2645         (rdev->family == CHIP_R350)  ||         \
2646         (rdev->family == CHIP_RV380) ||         \
2647         (rdev->family == CHIP_R420)  ||         \
2648         (rdev->family == CHIP_R423)  ||         \
2649         (rdev->family == CHIP_RV410) ||         \
2650         (rdev->family == CHIP_RS400) ||         \
2651         (rdev->family == CHIP_RS480))
2652 #define ASIC_IS_X2(rdev) ((rdev->pdev->device == 0x9441) || \
2653         (rdev->pdev->device == 0x9443) || \
2654         (rdev->pdev->device == 0x944B) || \
2655         (rdev->pdev->device == 0x9506) || \
2656         (rdev->pdev->device == 0x9509) || \
2657         (rdev->pdev->device == 0x950F) || \
2658         (rdev->pdev->device == 0x689C) || \
2659         (rdev->pdev->device == 0x689D))
2660 #define ASIC_IS_AVIVO(rdev) ((rdev->family >= CHIP_RS600))
2661 #define ASIC_IS_DCE2(rdev) ((rdev->family == CHIP_RS600)  ||    \
2662                 (rdev->family == CHIP_RS690)  ||    \
2663                 (rdev->family == CHIP_RS740)  ||    \
2664                 (rdev->family >= CHIP_R600))
2665 #define ASIC_IS_DCE3(rdev) ((rdev->family >= CHIP_RV620))
2666 #define ASIC_IS_DCE32(rdev) ((rdev->family >= CHIP_RV730))
2667 #define ASIC_IS_DCE4(rdev) ((rdev->family >= CHIP_CEDAR))
2668 #define ASIC_IS_DCE41(rdev) ((rdev->family >= CHIP_PALM) && \
2669                  (rdev->flags & RADEON_IS_IGP))
2670 #define ASIC_IS_DCE5(rdev) ((rdev->family >= CHIP_BARTS))
2671 #define ASIC_IS_DCE6(rdev) ((rdev->family >= CHIP_ARUBA))
2672 #define ASIC_IS_DCE61(rdev) ((rdev->family >= CHIP_ARUBA) && \
2673                  (rdev->flags & RADEON_IS_IGP))
2674 #define ASIC_IS_DCE64(rdev) ((rdev->family == CHIP_OLAND))
2675 #define ASIC_IS_NODCE(rdev) ((rdev->family == CHIP_HAINAN))
2676 #define ASIC_IS_DCE8(rdev) ((rdev->family >= CHIP_BONAIRE))
2677 #define ASIC_IS_DCE81(rdev) ((rdev->family == CHIP_KAVERI))
2678 #define ASIC_IS_DCE82(rdev) ((rdev->family == CHIP_BONAIRE))
2679 #define ASIC_IS_DCE83(rdev) ((rdev->family == CHIP_KABINI) || \
2680                  (rdev->family == CHIP_MULLINS))
2681 
2682 #define ASIC_IS_LOMBOK(rdev) ((rdev->pdev->device == 0x6849) || \
2683                   (rdev->pdev->device == 0x6850) || \
2684                   (rdev->pdev->device == 0x6858) || \
2685                   (rdev->pdev->device == 0x6859) || \
2686                   (rdev->pdev->device == 0x6840) || \
2687                   (rdev->pdev->device == 0x6841) || \
2688                   (rdev->pdev->device == 0x6842) || \
2689                   (rdev->pdev->device == 0x6843))
2690 
2691 /*
2692  * BIOS helpers.
2693  */
2694 #define RBIOS8(i) (rdev->bios[i])
2695 #define RBIOS16(i) (RBIOS8(i) | (RBIOS8((i)+1) << 8))
2696 #define RBIOS32(i) ((RBIOS16(i)) | (RBIOS16((i)+2) << 16))
2697 
2698 int radeon_combios_init(struct radeon_device *rdev);
2699 void radeon_combios_fini(struct radeon_device *rdev);
2700 int radeon_atombios_init(struct radeon_device *rdev);
2701 void radeon_atombios_fini(struct radeon_device *rdev);
2702 
2703 
2704 /*
2705  * RING helpers.
2706  */
2707 
2708 /**
2709  * radeon_ring_write - write a value to the ring
2710  *
2711  * @ring: radeon_ring structure holding ring information
2712  * @v: dword (dw) value to write
2713  *
2714  * Write a value to the requested ring buffer (all asics).
2715  */
2716 static inline void radeon_ring_write(struct radeon_ring *ring, uint32_t v)
2717 {
2718     if (ring->count_dw <= 0)
2719         DRM_ERROR("radeon: writing more dwords to the ring than expected!\n");
2720 
2721     ring->ring[ring->wptr++] = v;
2722     ring->wptr &= ring->ptr_mask;
2723     ring->count_dw--;
2724     ring->ring_free_dw--;
2725 }
2726 
2727 /*
2728  * ASICs macro.
2729  */
2730 #define radeon_init(rdev) (rdev)->asic->init((rdev))
2731 #define radeon_fini(rdev) (rdev)->asic->fini((rdev))
2732 #define radeon_resume(rdev) (rdev)->asic->resume((rdev))
2733 #define radeon_suspend(rdev) (rdev)->asic->suspend((rdev))
2734 #define radeon_cs_parse(rdev, r, p) (rdev)->asic->ring[(r)]->cs_parse((p))
2735 #define radeon_vga_set_state(rdev, state) (rdev)->asic->vga_set_state((rdev), (state))
2736 #define radeon_asic_reset(rdev) (rdev)->asic->asic_reset((rdev), false)
2737 #define radeon_gart_tlb_flush(rdev) (rdev)->asic->gart.tlb_flush((rdev))
2738 #define radeon_gart_get_page_entry(a, f) (rdev)->asic->gart.get_page_entry((a), (f))
2739 #define radeon_gart_set_page(rdev, i, e) (rdev)->asic->gart.set_page((rdev), (i), (e))
2740 #define radeon_asic_vm_init(rdev) (rdev)->asic->vm.init((rdev))
2741 #define radeon_asic_vm_fini(rdev) (rdev)->asic->vm.fini((rdev))
2742 #define radeon_asic_vm_copy_pages(rdev, ib, pe, src, count) ((rdev)->asic->vm.copy_pages((rdev), (ib), (pe), (src), (count)))
2743 #define radeon_asic_vm_write_pages(rdev, ib, pe, addr, count, incr, flags) ((rdev)->asic->vm.write_pages((rdev), (ib), (pe), (addr), (count), (incr), (flags)))
2744 #define radeon_asic_vm_set_pages(rdev, ib, pe, addr, count, incr, flags) ((rdev)->asic->vm.set_pages((rdev), (ib), (pe), (addr), (count), (incr), (flags)))
2745 #define radeon_asic_vm_pad_ib(rdev, ib) ((rdev)->asic->vm.pad_ib((ib)))
2746 #define radeon_ring_start(rdev, r, cp) (rdev)->asic->ring[(r)]->ring_start((rdev), (cp))
2747 #define radeon_ring_test(rdev, r, cp) (rdev)->asic->ring[(r)]->ring_test((rdev), (cp))
2748 #define radeon_ib_test(rdev, r, cp) (rdev)->asic->ring[(r)]->ib_test((rdev), (cp))
2749 #define radeon_ring_ib_execute(rdev, r, ib) (rdev)->asic->ring[(r)]->ib_execute((rdev), (ib))
2750 #define radeon_ring_ib_parse(rdev, r, ib) (rdev)->asic->ring[(r)]->ib_parse((rdev), (ib))
2751 #define radeon_ring_is_lockup(rdev, r, cp) (rdev)->asic->ring[(r)]->is_lockup((rdev), (cp))
2752 #define radeon_ring_vm_flush(rdev, r, vm_id, pd_addr) (rdev)->asic->ring[(r)->idx]->vm_flush((rdev), (r), (vm_id), (pd_addr))
2753 #define radeon_ring_get_rptr(rdev, r) (rdev)->asic->ring[(r)->idx]->get_rptr((rdev), (r))
2754 #define radeon_ring_get_wptr(rdev, r) (rdev)->asic->ring[(r)->idx]->get_wptr((rdev), (r))
2755 #define radeon_ring_set_wptr(rdev, r) (rdev)->asic->ring[(r)->idx]->set_wptr((rdev), (r))
2756 #define radeon_irq_set(rdev) (rdev)->asic->irq.set((rdev))
2757 #define radeon_irq_process(rdev) (rdev)->asic->irq.process((rdev))
2758 #define radeon_get_vblank_counter(rdev, crtc) (rdev)->asic->display.get_vblank_counter((rdev), (crtc))
2759 #define radeon_set_backlight_level(rdev, e, l) (rdev)->asic->display.set_backlight_level((e), (l))
2760 #define radeon_get_backlight_level(rdev, e) (rdev)->asic->display.get_backlight_level((e))
2761 #define radeon_hdmi_enable(rdev, e, b) (rdev)->asic->display.hdmi_enable((e), (b))
2762 #define radeon_hdmi_setmode(rdev, e, m) (rdev)->asic->display.hdmi_setmode((e), (m))
2763 #define radeon_fence_ring_emit(rdev, r, fence) (rdev)->asic->ring[(r)]->emit_fence((rdev), (fence))
2764 #define radeon_semaphore_ring_emit(rdev, r, cp, semaphore, emit_wait) (rdev)->asic->ring[(r)]->emit_semaphore((rdev), (cp), (semaphore), (emit_wait))
2765 #define radeon_copy_blit(rdev, s, d, np, resv) (rdev)->asic->copy.blit((rdev), (s), (d), (np), (resv))
2766 #define radeon_copy_dma(rdev, s, d, np, resv) (rdev)->asic->copy.dma((rdev), (s), (d), (np), (resv))
2767 #define radeon_copy(rdev, s, d, np, resv) (rdev)->asic->copy.copy((rdev), (s), (d), (np), (resv))
2768 #define radeon_copy_blit_ring_index(rdev) (rdev)->asic->copy.blit_ring_index
2769 #define radeon_copy_dma_ring_index(rdev) (rdev)->asic->copy.dma_ring_index
2770 #define radeon_copy_ring_index(rdev) (rdev)->asic->copy.copy_ring_index
2771 #define radeon_get_engine_clock(rdev) (rdev)->asic->pm.get_engine_clock((rdev))
2772 #define radeon_set_engine_clock(rdev, e) (rdev)->asic->pm.set_engine_clock((rdev), (e))
2773 #define radeon_get_memory_clock(rdev) (rdev)->asic->pm.get_memory_clock((rdev))
2774 #define radeon_set_memory_clock(rdev, e) (rdev)->asic->pm.set_memory_clock((rdev), (e))
2775 #define radeon_get_pcie_lanes(rdev) (rdev)->asic->pm.get_pcie_lanes((rdev))
2776 #define radeon_set_pcie_lanes(rdev, l) (rdev)->asic->pm.set_pcie_lanes((rdev), (l))
2777 #define radeon_set_clock_gating(rdev, e) (rdev)->asic->pm.set_clock_gating((rdev), (e))
2778 #define radeon_set_uvd_clocks(rdev, v, d) (rdev)->asic->pm.set_uvd_clocks((rdev), (v), (d))
2779 #define radeon_set_vce_clocks(rdev, ev, ec) (rdev)->asic->pm.set_vce_clocks((rdev), (ev), (ec))
2780 #define radeon_get_temperature(rdev) (rdev)->asic->pm.get_temperature((rdev))
2781 #define radeon_set_surface_reg(rdev, r, f, p, o, s) ((rdev)->asic->surface.set_reg((rdev), (r), (f), (p), (o), (s)))
2782 #define radeon_clear_surface_reg(rdev, r) ((rdev)->asic->surface.clear_reg((rdev), (r)))
2783 #define radeon_bandwidth_update(rdev) (rdev)->asic->display.bandwidth_update((rdev))
2784 #define radeon_hpd_init(rdev) (rdev)->asic->hpd.init((rdev))
2785 #define radeon_hpd_fini(rdev) (rdev)->asic->hpd.fini((rdev))
2786 #define radeon_hpd_sense(rdev, h) (rdev)->asic->hpd.sense((rdev), (h))
2787 #define radeon_hpd_set_polarity(rdev, h) (rdev)->asic->hpd.set_polarity((rdev), (h))
2788 #define radeon_gui_idle(rdev) (rdev)->asic->gui_idle((rdev))
2789 #define radeon_pm_misc(rdev) (rdev)->asic->pm.misc((rdev))
2790 #define radeon_pm_prepare(rdev) (rdev)->asic->pm.prepare((rdev))
2791 #define radeon_pm_finish(rdev) (rdev)->asic->pm.finish((rdev))
2792 #define radeon_pm_init_profile(rdev) (rdev)->asic->pm.init_profile((rdev))
2793 #define radeon_pm_get_dynpm_state(rdev) (rdev)->asic->pm.get_dynpm_state((rdev))
2794 #define radeon_page_flip(rdev, crtc, base, async) (rdev)->asic->pflip.page_flip((rdev), (crtc), (base), (async))
2795 #define radeon_page_flip_pending(rdev, crtc) (rdev)->asic->pflip.page_flip_pending((rdev), (crtc))
2796 #define radeon_wait_for_vblank(rdev, crtc) (rdev)->asic->display.wait_for_vblank((rdev), (crtc))
2797 #define radeon_mc_wait_for_idle(rdev) (rdev)->asic->mc_wait_for_idle((rdev))
2798 #define radeon_get_xclk(rdev) (rdev)->asic->get_xclk((rdev))
2799 #define radeon_get_gpu_clock_counter(rdev) (rdev)->asic->get_gpu_clock_counter((rdev))
2800 #define radeon_get_allowed_info_register(rdev, r, v) (rdev)->asic->get_allowed_info_register((rdev), (r), (v))
2801 #define radeon_dpm_init(rdev) rdev->asic->dpm.init((rdev))
2802 #define radeon_dpm_setup_asic(rdev) rdev->asic->dpm.setup_asic((rdev))
2803 #define radeon_dpm_enable(rdev) rdev->asic->dpm.enable((rdev))
2804 #define radeon_dpm_late_enable(rdev) rdev->asic->dpm.late_enable((rdev))
2805 #define radeon_dpm_disable(rdev) rdev->asic->dpm.disable((rdev))
2806 #define radeon_dpm_pre_set_power_state(rdev) rdev->asic->dpm.pre_set_power_state((rdev))
2807 #define radeon_dpm_set_power_state(rdev) rdev->asic->dpm.set_power_state((rdev))
2808 #define radeon_dpm_post_set_power_state(rdev) rdev->asic->dpm.post_set_power_state((rdev))
2809 #define radeon_dpm_display_configuration_changed(rdev) rdev->asic->dpm.display_configuration_changed((rdev))
2810 #define radeon_dpm_fini(rdev) rdev->asic->dpm.fini((rdev))
2811 #define radeon_dpm_get_sclk(rdev, l) rdev->asic->dpm.get_sclk((rdev), (l))
2812 #define radeon_dpm_get_mclk(rdev, l) rdev->asic->dpm.get_mclk((rdev), (l))
2813 #define radeon_dpm_print_power_state(rdev, ps) rdev->asic->dpm.print_power_state((rdev), (ps))
2814 #define radeon_dpm_debugfs_print_current_performance_level(rdev, m) rdev->asic->dpm.debugfs_print_current_performance_level((rdev), (m))
2815 #define radeon_dpm_force_performance_level(rdev, l) rdev->asic->dpm.force_performance_level((rdev), (l))
2816 #define radeon_dpm_vblank_too_short(rdev) rdev->asic->dpm.vblank_too_short((rdev))
2817 #define radeon_dpm_powergate_uvd(rdev, g) rdev->asic->dpm.powergate_uvd((rdev), (g))
2818 #define radeon_dpm_enable_bapm(rdev, e) rdev->asic->dpm.enable_bapm((rdev), (e))
2819 #define radeon_dpm_get_current_sclk(rdev) rdev->asic->dpm.get_current_sclk((rdev))
2820 #define radeon_dpm_get_current_mclk(rdev) rdev->asic->dpm.get_current_mclk((rdev))
2821 
2822 /* Common functions */
2823 /* AGP */
2824 extern int radeon_gpu_reset(struct radeon_device *rdev);
2825 extern void radeon_pci_config_reset(struct radeon_device *rdev);
2826 extern void r600_set_bios_scratch_engine_hung(struct radeon_device *rdev, bool hung);
2827 extern void radeon_agp_disable(struct radeon_device *rdev);
2828 extern int radeon_modeset_init(struct radeon_device *rdev);
2829 extern void radeon_modeset_fini(struct radeon_device *rdev);
2830 extern bool radeon_card_posted(struct radeon_device *rdev);
2831 extern void radeon_update_bandwidth_info(struct radeon_device *rdev);
2832 extern void radeon_update_display_priority(struct radeon_device *rdev);
2833 extern bool radeon_boot_test_post_card(struct radeon_device *rdev);
2834 extern void radeon_scratch_init(struct radeon_device *rdev);
2835 extern void radeon_wb_fini(struct radeon_device *rdev);
2836 extern int radeon_wb_init(struct radeon_device *rdev);
2837 extern void radeon_wb_disable(struct radeon_device *rdev);
2838 extern void radeon_surface_init(struct radeon_device *rdev);
2839 extern int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data);
2840 extern void radeon_legacy_set_clock_gating(struct radeon_device *rdev, int enable);
2841 extern void radeon_atom_set_clock_gating(struct radeon_device *rdev, int enable);
2842 extern void radeon_ttm_placement_from_domain(struct radeon_bo *rbo, u32 domain);
2843 extern bool radeon_ttm_bo_is_radeon_bo(struct ttm_buffer_object *bo);
2844 extern int radeon_ttm_tt_set_userptr(struct radeon_device *rdev,
2845                      struct ttm_tt *ttm, uint64_t addr,
2846                      uint32_t flags);
2847 extern bool radeon_ttm_tt_has_userptr(struct radeon_device *rdev, struct ttm_tt *ttm);
2848 extern bool radeon_ttm_tt_is_readonly(struct radeon_device *rdev, struct ttm_tt *ttm);
2849 bool radeon_ttm_tt_is_bound(struct ttm_device *bdev, struct ttm_tt *ttm);
2850 extern void radeon_vram_location(struct radeon_device *rdev, struct radeon_mc *mc, u64 base);
2851 extern void radeon_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
2852 extern int radeon_resume_kms(struct drm_device *dev, bool resume, bool fbcon);
2853 extern int radeon_suspend_kms(struct drm_device *dev, bool suspend,
2854                   bool fbcon, bool freeze);
2855 extern void radeon_ttm_set_active_vram_size(struct radeon_device *rdev, u64 size);
2856 extern void radeon_program_register_sequence(struct radeon_device *rdev,
2857                          const u32 *registers,
2858                          const u32 array_size);
2859 struct radeon_device *radeon_get_rdev(struct ttm_device *bdev);
2860 
2861 /* KMS */
2862 
2863 u32 radeon_get_vblank_counter_kms(struct drm_crtc *crtc);
2864 int radeon_enable_vblank_kms(struct drm_crtc *crtc);
2865 void radeon_disable_vblank_kms(struct drm_crtc *crtc);
2866 
2867 /*
2868  * vm
2869  */
2870 int radeon_vm_manager_init(struct radeon_device *rdev);
2871 void radeon_vm_manager_fini(struct radeon_device *rdev);
2872 int radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm);
2873 void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm);
2874 struct radeon_bo_list *radeon_vm_get_bos(struct radeon_device *rdev,
2875                       struct radeon_vm *vm,
2876                                           struct list_head *head);
2877 struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev,
2878                        struct radeon_vm *vm, int ring);
2879 void radeon_vm_flush(struct radeon_device *rdev,
2880                      struct radeon_vm *vm,
2881              int ring, struct radeon_fence *fence);
2882 void radeon_vm_fence(struct radeon_device *rdev,
2883              struct radeon_vm *vm,
2884              struct radeon_fence *fence);
2885 uint64_t radeon_vm_map_gart(struct radeon_device *rdev, uint64_t addr);
2886 int radeon_vm_update_page_directory(struct radeon_device *rdev,
2887                     struct radeon_vm *vm);
2888 int radeon_vm_clear_freed(struct radeon_device *rdev,
2889               struct radeon_vm *vm);
2890 int radeon_vm_clear_invalids(struct radeon_device *rdev,
2891                  struct radeon_vm *vm);
2892 int radeon_vm_bo_update(struct radeon_device *rdev,
2893             struct radeon_bo_va *bo_va,
2894             struct ttm_resource *mem);
2895 void radeon_vm_bo_invalidate(struct radeon_device *rdev,
2896                  struct radeon_bo *bo);
2897 struct radeon_bo_va *radeon_vm_bo_find(struct radeon_vm *vm,
2898                        struct radeon_bo *bo);
2899 struct radeon_bo_va *radeon_vm_bo_add(struct radeon_device *rdev,
2900                       struct radeon_vm *vm,
2901                       struct radeon_bo *bo);
2902 int radeon_vm_bo_set_addr(struct radeon_device *rdev,
2903               struct radeon_bo_va *bo_va,
2904               uint64_t offset,
2905               uint32_t flags);
2906 void radeon_vm_bo_rmv(struct radeon_device *rdev,
2907               struct radeon_bo_va *bo_va);
2908 
2909 /* audio */
2910 void r600_audio_update_hdmi(struct work_struct *work);
2911 struct r600_audio_pin *r600_audio_get_pin(struct radeon_device *rdev);
2912 struct r600_audio_pin *dce6_audio_get_pin(struct radeon_device *rdev);
2913 void r600_audio_enable(struct radeon_device *rdev,
2914                struct r600_audio_pin *pin,
2915                u8 enable_mask);
2916 void dce6_audio_enable(struct radeon_device *rdev,
2917                struct r600_audio_pin *pin,
2918                u8 enable_mask);
2919 
2920 /*
2921  * R600 vram scratch functions
2922  */
2923 int r600_vram_scratch_init(struct radeon_device *rdev);
2924 void r600_vram_scratch_fini(struct radeon_device *rdev);
2925 
2926 /*
2927  * r600 cs checking helper
2928  */
2929 unsigned r600_mip_minify(unsigned size, unsigned level);
2930 bool r600_fmt_is_valid_color(u32 format);
2931 bool r600_fmt_is_valid_texture(u32 format, enum radeon_family family);
2932 int r600_fmt_get_blocksize(u32 format);
2933 int r600_fmt_get_nblocksx(u32 format, u32 w);
2934 int r600_fmt_get_nblocksy(u32 format, u32 h);
2935 
2936 /*
2937  * r600 functions used by radeon_encoder.c
2938  */
2939 struct radeon_hdmi_acr {
2940     u32 clock;
2941 
2942     int n_32khz;
2943     int cts_32khz;
2944 
2945     int n_44_1khz;
2946     int cts_44_1khz;
2947 
2948     int n_48khz;
2949     int cts_48khz;
2950 
2951 };
2952 
2953 extern struct radeon_hdmi_acr r600_hdmi_acr(uint32_t clock);
2954 
2955 extern u32 r6xx_remap_render_backend(struct radeon_device *rdev,
2956                      u32 tiling_pipe_num,
2957                      u32 max_rb_num,
2958                      u32 total_max_rb_num,
2959                      u32 enabled_rb_mask);
2960 
2961 /*
2962  * evergreen functions used by radeon_encoder.c
2963  */
2964 
2965 extern int ni_init_microcode(struct radeon_device *rdev);
2966 extern int ni_mc_load_microcode(struct radeon_device *rdev);
2967 
2968 /* radeon_acpi.c */
2969 #if defined(CONFIG_ACPI)
2970 extern int radeon_acpi_init(struct radeon_device *rdev);
2971 extern void radeon_acpi_fini(struct radeon_device *rdev);
2972 extern bool radeon_acpi_is_pcie_performance_request_supported(struct radeon_device *rdev);
2973 extern int radeon_acpi_pcie_performance_request(struct radeon_device *rdev,
2974                         u8 perf_req, bool advertise);
2975 extern int radeon_acpi_pcie_notify_device_ready(struct radeon_device *rdev);
2976 #else
2977 static inline int radeon_acpi_init(struct radeon_device *rdev) { return 0; }
2978 static inline void radeon_acpi_fini(struct radeon_device *rdev) { }
2979 #endif
2980 
2981 int radeon_cs_packet_parse(struct radeon_cs_parser *p,
2982                struct radeon_cs_packet *pkt,
2983                unsigned idx);
2984 bool radeon_cs_packet_next_is_pkt3_nop(struct radeon_cs_parser *p);
2985 void radeon_cs_dump_packet(struct radeon_cs_parser *p,
2986                struct radeon_cs_packet *pkt);
2987 int radeon_cs_packet_next_reloc(struct radeon_cs_parser *p,
2988                 struct radeon_bo_list **cs_reloc,
2989                 int nomm);
2990 int r600_cs_common_vline_parse(struct radeon_cs_parser *p,
2991                    uint32_t *vline_start_end,
2992                    uint32_t *vline_status);
2993 
2994 /* interrupt control register helpers */
2995 void radeon_irq_kms_set_irq_n_enabled(struct radeon_device *rdev,
2996                       u32 reg, u32 mask,
2997                       bool enable, const char *name,
2998                       unsigned n);
2999 
3000 #include "radeon_object.h"
3001 
3002 #endif
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!