uapi/linux/kfd_ioctl.h

0001 /*
0002  * Copyright 2014 Advanced Micro Devices, Inc.
0003  *
0004  * Permission is hereby granted, free of charge, to any person obtaining a
0005  * copy of this software and associated documentation files (the "Software"),
0006  * to deal in the Software without restriction, including without limitation
0007  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0008  * and/or sell copies of the Software, and to permit persons to whom the
0009  * Software is furnished to do so, subject to the following conditions:
0010  *
0011  * The above copyright notice and this permission notice shall be included in
0012  * all copies or substantial portions of the Software.
0013  *
0014  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0015  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0016  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0017  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
0018  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
0019  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
0020  * OTHER DEALINGS IN THE SOFTWARE.
0021  */
0022
0023 #ifndef KFD_IOCTL_H_INCLUDED
0024 #define KFD_IOCTL_H_INCLUDED
0025
0026 #include <drm/drm.h>
0027 #include <linux/ioctl.h>
0028
0029 /*
0030  * - 1.1 - initial version
0031  * - 1.3 - Add SMI events support
0032  * - 1.4 - Indicate new SRAM EDC bit in device properties
0033  * - 1.5 - Add SVM API
0034  * - 1.6 - Query clear flags in SVM get_attr API
0035  * - 1.7 - Checkpoint Restore (CRIU) API
0036  * - 1.8 - CRIU - Support for SDMA transfers with GTT BOs
0037  * - 1.9 - Add available memory ioctl
0038  * - 1.10 - Add SMI profiler event log
0039  * - 1.11 - Add unified memory for ctx save/restore area
0040  */
0041 #define KFD_IOCTL_MAJOR_VERSION 1
0042 #define KFD_IOCTL_MINOR_VERSION 11
0043
0044 struct kfd_ioctl_get_version_args {
0045     __u32 major_version;    /* from KFD */
0046     __u32 minor_version;    /* from KFD */
0047 };
0048
0049 /* For kfd_ioctl_create_queue_args.queue_type. */
0050 #define KFD_IOC_QUEUE_TYPE_COMPUTE      0x0
0051 #define KFD_IOC_QUEUE_TYPE_SDMA         0x1
0052 #define KFD_IOC_QUEUE_TYPE_COMPUTE_AQL      0x2
0053 #define KFD_IOC_QUEUE_TYPE_SDMA_XGMI        0x3
0054
0055 #define KFD_MAX_QUEUE_PERCENTAGE    100
0056 #define KFD_MAX_QUEUE_PRIORITY      15
0057
0058 struct kfd_ioctl_create_queue_args {
0059     __u64 ring_base_address;    /* to KFD */
0060     __u64 write_pointer_address;    /* from KFD */
0061     __u64 read_pointer_address; /* from KFD */
0062     __u64 doorbell_offset;  /* from KFD */
0063
0064     __u32 ring_size;        /* to KFD */
0065     __u32 gpu_id;       /* to KFD */
0066     __u32 queue_type;       /* to KFD */
0067     __u32 queue_percentage; /* to KFD */
0068     __u32 queue_priority;   /* to KFD */
0069     __u32 queue_id;     /* from KFD */
0070
0071     __u64 eop_buffer_address;   /* to KFD */
0072     __u64 eop_buffer_size;  /* to KFD */
0073     __u64 ctx_save_restore_address; /* to KFD */
0074     __u32 ctx_save_restore_size;    /* to KFD */
0075     __u32 ctl_stack_size;       /* to KFD */
0076 };
0077
0078 struct kfd_ioctl_destroy_queue_args {
0079     __u32 queue_id;     /* to KFD */
0080     __u32 pad;
0081 };
0082
0083 struct kfd_ioctl_update_queue_args {
0084     __u64 ring_base_address;    /* to KFD */
0085
0086     __u32 queue_id;     /* to KFD */
0087     __u32 ring_size;        /* to KFD */
0088     __u32 queue_percentage; /* to KFD */
0089     __u32 queue_priority;   /* to KFD */
0090 };
0091
0092 struct kfd_ioctl_set_cu_mask_args {
0093     __u32 queue_id;     /* to KFD */
0094     __u32 num_cu_mask;      /* to KFD */
0095     __u64 cu_mask_ptr;      /* to KFD */
0096 };
0097
0098 struct kfd_ioctl_get_queue_wave_state_args {
0099     __u64 ctl_stack_address;    /* to KFD */
0100     __u32 ctl_stack_used_size;  /* from KFD */
0101     __u32 save_area_used_size;  /* from KFD */
0102     __u32 queue_id;         /* to KFD */
0103     __u32 pad;
0104 };
0105
0106 struct kfd_ioctl_get_available_memory_args {
0107     __u64 available;    /* from KFD */
0108     __u32 gpu_id;       /* to KFD */
0109     __u32 pad;
0110 };
0111
0112 /* For kfd_ioctl_set_memory_policy_args.default_policy and alternate_policy */
0113 #define KFD_IOC_CACHE_POLICY_COHERENT 0
0114 #define KFD_IOC_CACHE_POLICY_NONCOHERENT 1
0115
0116 struct kfd_ioctl_set_memory_policy_args {
0117     __u64 alternate_aperture_base;  /* to KFD */
0118     __u64 alternate_aperture_size;  /* to KFD */
0119
0120     __u32 gpu_id;           /* to KFD */
0121     __u32 default_policy;       /* to KFD */
0122     __u32 alternate_policy;     /* to KFD */
0123     __u32 pad;
0124 };
0125
0126 /*
0127  * All counters are monotonic. They are used for profiling of compute jobs.
0128  * The profiling is done by userspace.
0129  *
0130  * In case of GPU reset, the counter should not be affected.
0131  */
0132
0133 struct kfd_ioctl_get_clock_counters_args {
0134     __u64 gpu_clock_counter;    /* from KFD */
0135     __u64 cpu_clock_counter;    /* from KFD */
0136     __u64 system_clock_counter; /* from KFD */
0137     __u64 system_clock_freq;    /* from KFD */
0138
0139     __u32 gpu_id;       /* to KFD */
0140     __u32 pad;
0141 };
0142
0143 struct kfd_process_device_apertures {
0144     __u64 lds_base;     /* from KFD */
0145     __u64 lds_limit;        /* from KFD */
0146     __u64 scratch_base;     /* from KFD */
0147     __u64 scratch_limit;        /* from KFD */
0148     __u64 gpuvm_base;       /* from KFD */
0149     __u64 gpuvm_limit;      /* from KFD */
0150     __u32 gpu_id;       /* from KFD */
0151     __u32 pad;
0152 };
0153
0154 /*
0155  * AMDKFD_IOC_GET_PROCESS_APERTURES is deprecated. Use
0156  * AMDKFD_IOC_GET_PROCESS_APERTURES_NEW instead, which supports an
0157  * unlimited number of GPUs.
0158  */
0159 #define NUM_OF_SUPPORTED_GPUS 7
0160 struct kfd_ioctl_get_process_apertures_args {
0161     struct kfd_process_device_apertures
0162             process_apertures[NUM_OF_SUPPORTED_GPUS];/* from KFD */
0163
0164     /* from KFD, should be in the range [1 - NUM_OF_SUPPORTED_GPUS] */
0165     __u32 num_of_nodes;
0166     __u32 pad;
0167 };
0168
0169 struct kfd_ioctl_get_process_apertures_new_args {
0170     /* User allocated. Pointer to struct kfd_process_device_apertures
0171      * filled in by Kernel
0172      */
0173     __u64 kfd_process_device_apertures_ptr;
0174     /* to KFD - indicates amount of memory present in
0175      *  kfd_process_device_apertures_ptr
0176      * from KFD - Number of entries filled by KFD.
0177      */
0178     __u32 num_of_nodes;
0179     __u32 pad;
0180 };
0181
0182 #define MAX_ALLOWED_NUM_POINTS    100
0183 #define MAX_ALLOWED_AW_BUFF_SIZE 4096
0184 #define MAX_ALLOWED_WAC_BUFF_SIZE  128
0185
0186 struct kfd_ioctl_dbg_register_args {
0187     __u32 gpu_id;       /* to KFD */
0188     __u32 pad;
0189 };
0190
0191 struct kfd_ioctl_dbg_unregister_args {
0192     __u32 gpu_id;       /* to KFD */
0193     __u32 pad;
0194 };
0195
0196 struct kfd_ioctl_dbg_address_watch_args {
0197     __u64 content_ptr;      /* a pointer to the actual content */
0198     __u32 gpu_id;       /* to KFD */
0199     __u32 buf_size_in_bytes;    /*including gpu_id and buf_size */
0200 };
0201
0202 struct kfd_ioctl_dbg_wave_control_args {
0203     __u64 content_ptr;      /* a pointer to the actual content */
0204     __u32 gpu_id;       /* to KFD */
0205     __u32 buf_size_in_bytes;    /*including gpu_id and buf_size */
0206 };
0207
0208 #define KFD_INVALID_FD     0xffffffff
0209
0210 /* Matching HSA_EVENTTYPE */
0211 #define KFD_IOC_EVENT_SIGNAL            0
0212 #define KFD_IOC_EVENT_NODECHANGE        1
0213 #define KFD_IOC_EVENT_DEVICESTATECHANGE     2
0214 #define KFD_IOC_EVENT_HW_EXCEPTION      3
0215 #define KFD_IOC_EVENT_SYSTEM_EVENT      4
0216 #define KFD_IOC_EVENT_DEBUG_EVENT       5
0217 #define KFD_IOC_EVENT_PROFILE_EVENT     6
0218 #define KFD_IOC_EVENT_QUEUE_EVENT       7
0219 #define KFD_IOC_EVENT_MEMORY            8
0220
0221 #define KFD_IOC_WAIT_RESULT_COMPLETE        0
0222 #define KFD_IOC_WAIT_RESULT_TIMEOUT     1
0223 #define KFD_IOC_WAIT_RESULT_FAIL        2
0224
0225 #define KFD_SIGNAL_EVENT_LIMIT          4096
0226
0227 /* For kfd_event_data.hw_exception_data.reset_type. */
0228 #define KFD_HW_EXCEPTION_WHOLE_GPU_RESET    0
0229 #define KFD_HW_EXCEPTION_PER_ENGINE_RESET   1
0230
0231 /* For kfd_event_data.hw_exception_data.reset_cause. */
0232 #define KFD_HW_EXCEPTION_GPU_HANG   0
0233 #define KFD_HW_EXCEPTION_ECC        1
0234
0235 /* For kfd_hsa_memory_exception_data.ErrorType */
0236 #define KFD_MEM_ERR_NO_RAS      0
0237 #define KFD_MEM_ERR_SRAM_ECC        1
0238 #define KFD_MEM_ERR_POISON_CONSUMED 2
0239 #define KFD_MEM_ERR_GPU_HANG        3
0240
0241 struct kfd_ioctl_create_event_args {
0242     __u64 event_page_offset;    /* from KFD */
0243     __u32 event_trigger_data;   /* from KFD - signal events only */
0244     __u32 event_type;       /* to KFD */
0245     __u32 auto_reset;       /* to KFD */
0246     __u32 node_id;      /* to KFD - only valid for certain
0247                             event types */
0248     __u32 event_id;     /* from KFD */
0249     __u32 event_slot_index; /* from KFD */
0250 };
0251
0252 struct kfd_ioctl_destroy_event_args {
0253     __u32 event_id;     /* to KFD */
0254     __u32 pad;
0255 };
0256
0257 struct kfd_ioctl_set_event_args {
0258     __u32 event_id;     /* to KFD */
0259     __u32 pad;
0260 };
0261
0262 struct kfd_ioctl_reset_event_args {
0263     __u32 event_id;     /* to KFD */
0264     __u32 pad;
0265 };
0266
0267 struct kfd_memory_exception_failure {
0268     __u32 NotPresent;   /* Page not present or supervisor privilege */
0269     __u32 ReadOnly; /* Write access to a read-only page */
0270     __u32 NoExecute;    /* Execute access to a page marked NX */
0271     __u32 imprecise;    /* Can't determine the  exact fault address */
0272 };
0273
0274 /* memory exception data */
0275 struct kfd_hsa_memory_exception_data {
0276     struct kfd_memory_exception_failure failure;
0277     __u64 va;
0278     __u32 gpu_id;
0279     __u32 ErrorType; /* 0 = no RAS error,
0280               * 1 = ECC_SRAM,
0281               * 2 = Link_SYNFLOOD (poison),
0282               * 3 = GPU hang (not attributable to a specific cause),
0283               * other values reserved
0284               */
0285 };
0286
0287 /* hw exception data */
0288 struct kfd_hsa_hw_exception_data {
0289     __u32 reset_type;
0290     __u32 reset_cause;
0291     __u32 memory_lost;
0292     __u32 gpu_id;
0293 };
0294
0295 /* Event data */
0296 struct kfd_event_data {
0297     union {
0298         struct kfd_hsa_memory_exception_data memory_exception_data;
0299         struct kfd_hsa_hw_exception_data hw_exception_data;
0300     };              /* From KFD */
0301     __u64 kfd_event_data_ext;   /* pointer to an extension structure
0302                        for future exception types */
0303     __u32 event_id;     /* to KFD */
0304     __u32 pad;
0305 };
0306
0307 struct kfd_ioctl_wait_events_args {
0308     __u64 events_ptr;       /* pointed to struct
0309                        kfd_event_data array, to KFD */
0310     __u32 num_events;       /* to KFD */
0311     __u32 wait_for_all;     /* to KFD */
0312     __u32 timeout;      /* to KFD */
0313     __u32 wait_result;      /* from KFD */
0314 };
0315
0316 struct kfd_ioctl_set_scratch_backing_va_args {
0317     __u64 va_addr;  /* to KFD */
0318     __u32 gpu_id;   /* to KFD */
0319     __u32 pad;
0320 };
0321
0322 struct kfd_ioctl_get_tile_config_args {
0323     /* to KFD: pointer to tile array */
0324     __u64 tile_config_ptr;
0325     /* to KFD: pointer to macro tile array */
0326     __u64 macro_tile_config_ptr;
0327     /* to KFD: array size allocated by user mode
0328      * from KFD: array size filled by kernel
0329      */
0330     __u32 num_tile_configs;
0331     /* to KFD: array size allocated by user mode
0332      * from KFD: array size filled by kernel
0333      */
0334     __u32 num_macro_tile_configs;
0335
0336     __u32 gpu_id;       /* to KFD */
0337     __u32 gb_addr_config;   /* from KFD */
0338     __u32 num_banks;        /* from KFD */
0339     __u32 num_ranks;        /* from KFD */
0340     /* struct size can be extended later if needed
0341      * without breaking ABI compatibility
0342      */
0343 };
0344
0345 struct kfd_ioctl_set_trap_handler_args {
0346     __u64 tba_addr;     /* to KFD */
0347     __u64 tma_addr;     /* to KFD */
0348     __u32 gpu_id;       /* to KFD */
0349     __u32 pad;
0350 };
0351
0352 struct kfd_ioctl_acquire_vm_args {
0353     __u32 drm_fd;   /* to KFD */
0354     __u32 gpu_id;   /* to KFD */
0355 };
0356
0357 /* Allocation flags: memory types */
0358 #define KFD_IOC_ALLOC_MEM_FLAGS_VRAM        (1 << 0)
0359 #define KFD_IOC_ALLOC_MEM_FLAGS_GTT     (1 << 1)
0360 #define KFD_IOC_ALLOC_MEM_FLAGS_USERPTR     (1 << 2)
0361 #define KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL    (1 << 3)
0362 #define KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP  (1 << 4)
0363 /* Allocation flags: attributes/access options */
0364 #define KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE    (1 << 31)
0365 #define KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE  (1 << 30)
0366 #define KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC      (1 << 29)
0367 #define KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE   (1 << 28)
0368 #define KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM   (1 << 27)
0369 #define KFD_IOC_ALLOC_MEM_FLAGS_COHERENT    (1 << 26)
0370 #define KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED    (1 << 25)
0371
0372 /* Allocate memory for later SVM (shared virtual memory) mapping.
0373  *
0374  * @va_addr:     virtual address of the memory to be allocated
0375  *               all later mappings on all GPUs will use this address
0376  * @size:        size in bytes
0377  * @handle:      buffer handle returned to user mode, used to refer to
0378  *               this allocation for mapping, unmapping and freeing
0379  * @mmap_offset: for CPU-mapping the allocation by mmapping a render node
0380  *               for userptrs this is overloaded to specify the CPU address
0381  * @gpu_id:      device identifier
0382  * @flags:       memory type and attributes. See KFD_IOC_ALLOC_MEM_FLAGS above
0383  */
0384 struct kfd_ioctl_alloc_memory_of_gpu_args {
0385     __u64 va_addr;      /* to KFD */
0386     __u64 size;     /* to KFD */
0387     __u64 handle;       /* from KFD */
0388     __u64 mmap_offset;  /* to KFD (userptr), from KFD (mmap offset) */
0389     __u32 gpu_id;       /* to KFD */
0390     __u32 flags;
0391 };
0392
0393 /* Free memory allocated with kfd_ioctl_alloc_memory_of_gpu
0394  *
0395  * @handle: memory handle returned by alloc
0396  */
0397 struct kfd_ioctl_free_memory_of_gpu_args {
0398     __u64 handle;       /* to KFD */
0399 };
0400
0401 /* Map memory to one or more GPUs
0402  *
0403  * @handle:                memory handle returned by alloc
0404  * @device_ids_array_ptr:  array of gpu_ids (__u32 per device)
0405  * @n_devices:             number of devices in the array
0406  * @n_success:             number of devices mapped successfully
0407  *
0408  * @n_success returns information to the caller how many devices from
0409  * the start of the array have mapped the buffer successfully. It can
0410  * be passed into a subsequent retry call to skip those devices. For
0411  * the first call the caller should initialize it to 0.
0412  *
0413  * If the ioctl completes with return code 0 (success), n_success ==
0414  * n_devices.
0415  */
0416 struct kfd_ioctl_map_memory_to_gpu_args {
0417     __u64 handle;           /* to KFD */
0418     __u64 device_ids_array_ptr; /* to KFD */
0419     __u32 n_devices;        /* to KFD */
0420     __u32 n_success;        /* to/from KFD */
0421 };
0422
0423 /* Unmap memory from one or more GPUs
0424  *
0425  * same arguments as for mapping
0426  */
0427 struct kfd_ioctl_unmap_memory_from_gpu_args {
0428     __u64 handle;           /* to KFD */
0429     __u64 device_ids_array_ptr; /* to KFD */
0430     __u32 n_devices;        /* to KFD */
0431     __u32 n_success;        /* to/from KFD */
0432 };
0433
0434 /* Allocate GWS for specific queue
0435  *
0436  * @queue_id:    queue's id that GWS is allocated for
0437  * @num_gws:     how many GWS to allocate
0438  * @first_gws:   index of the first GWS allocated.
0439  *               only support contiguous GWS allocation
0440  */
0441 struct kfd_ioctl_alloc_queue_gws_args {
0442     __u32 queue_id;     /* to KFD */
0443     __u32 num_gws;      /* to KFD */
0444     __u32 first_gws;    /* from KFD */
0445     __u32 pad;
0446 };
0447
0448 struct kfd_ioctl_get_dmabuf_info_args {
0449     __u64 size;     /* from KFD */
0450     __u64 metadata_ptr; /* to KFD */
0451     __u32 metadata_size;    /* to KFD (space allocated by user)
0452                  * from KFD (actual metadata size)
0453                  */
0454     __u32 gpu_id;   /* from KFD */
0455     __u32 flags;        /* from KFD (KFD_IOC_ALLOC_MEM_FLAGS) */
0456     __u32 dmabuf_fd;    /* to KFD */
0457 };
0458
0459 struct kfd_ioctl_import_dmabuf_args {
0460     __u64 va_addr;  /* to KFD */
0461     __u64 handle;   /* from KFD */
0462     __u32 gpu_id;   /* to KFD */
0463     __u32 dmabuf_fd;    /* to KFD */
0464 };
0465
0466 /*
0467  * KFD SMI(System Management Interface) events
0468  */
0469 enum kfd_smi_event {
0470     KFD_SMI_EVENT_NONE = 0, /* not used */
0471     KFD_SMI_EVENT_VMFAULT = 1, /* event start counting at 1 */
0472     KFD_SMI_EVENT_THERMAL_THROTTLE = 2,
0473     KFD_SMI_EVENT_GPU_PRE_RESET = 3,
0474     KFD_SMI_EVENT_GPU_POST_RESET = 4,
0475     KFD_SMI_EVENT_MIGRATE_START = 5,
0476     KFD_SMI_EVENT_MIGRATE_END = 6,
0477     KFD_SMI_EVENT_PAGE_FAULT_START = 7,
0478     KFD_SMI_EVENT_PAGE_FAULT_END = 8,
0479     KFD_SMI_EVENT_QUEUE_EVICTION = 9,
0480     KFD_SMI_EVENT_QUEUE_RESTORE = 10,
0481     KFD_SMI_EVENT_UNMAP_FROM_GPU = 11,
0482
0483     /*
0484      * max event number, as a flag bit to get events from all processes,
0485      * this requires super user permission, otherwise will not be able to
0486      * receive event from any process. Without this flag to receive events
0487      * from same process.
0488      */
0489     KFD_SMI_EVENT_ALL_PROCESS = 64
0490 };
0491
0492 enum KFD_MIGRATE_TRIGGERS {
0493     KFD_MIGRATE_TRIGGER_PREFETCH,
0494     KFD_MIGRATE_TRIGGER_PAGEFAULT_GPU,
0495     KFD_MIGRATE_TRIGGER_PAGEFAULT_CPU,
0496     KFD_MIGRATE_TRIGGER_TTM_EVICTION
0497 };
0498
0499 enum KFD_QUEUE_EVICTION_TRIGGERS {
0500     KFD_QUEUE_EVICTION_TRIGGER_SVM,
0501     KFD_QUEUE_EVICTION_TRIGGER_USERPTR,
0502     KFD_QUEUE_EVICTION_TRIGGER_TTM,
0503     KFD_QUEUE_EVICTION_TRIGGER_SUSPEND,
0504     KFD_QUEUE_EVICTION_CRIU_CHECKPOINT,
0505     KFD_QUEUE_EVICTION_CRIU_RESTORE
0506 };
0507
0508 enum KFD_SVM_UNMAP_TRIGGERS {
0509     KFD_SVM_UNMAP_TRIGGER_MMU_NOTIFY,
0510     KFD_SVM_UNMAP_TRIGGER_MMU_NOTIFY_MIGRATE,
0511     KFD_SVM_UNMAP_TRIGGER_UNMAP_FROM_CPU
0512 };
0513
0514 #define KFD_SMI_EVENT_MASK_FROM_INDEX(i) (1ULL << ((i) - 1))
0515 #define KFD_SMI_EVENT_MSG_SIZE  96
0516
0517 struct kfd_ioctl_smi_events_args {
0518     __u32 gpuid;    /* to KFD */
0519     __u32 anon_fd;  /* from KFD */
0520 };
0521
0522 /**************************************************************************************************
0523  * CRIU IOCTLs (Checkpoint Restore In Userspace)
0524  *
0525  * When checkpointing a process, the userspace application will perform:
0526  * 1. PROCESS_INFO op to determine current process information. This pauses execution and evicts
0527  *    all the queues.
0528  * 2. CHECKPOINT op to checkpoint process contents (BOs, queues, events, svm-ranges)
0529  * 3. UNPAUSE op to un-evict all the queues
0530  *
0531  * When restoring a process, the CRIU userspace application will perform:
0532  *
0533  * 1. RESTORE op to restore process contents
0534  * 2. RESUME op to start the process
0535  *
0536  * Note: Queues are forced into an evicted state after a successful PROCESS_INFO. User
0537  * application needs to perform an UNPAUSE operation after calling PROCESS_INFO.
0538  */
0539
0540 enum kfd_criu_op {
0541     KFD_CRIU_OP_PROCESS_INFO,
0542     KFD_CRIU_OP_CHECKPOINT,
0543     KFD_CRIU_OP_UNPAUSE,
0544     KFD_CRIU_OP_RESTORE,
0545     KFD_CRIU_OP_RESUME,
0546 };
0547
0548 /**
0549  * kfd_ioctl_criu_args - Arguments perform CRIU operation
0550  * @devices:        [in/out] User pointer to memory location for devices information.
0551  *          This is an array of type kfd_criu_device_bucket.
0552  * @bos:        [in/out] User pointer to memory location for BOs information
0553  *          This is an array of type kfd_criu_bo_bucket.
0554  * @priv_data:      [in/out] User pointer to memory location for private data
0555  * @priv_data_size: [in/out] Size of priv_data in bytes
0556  * @num_devices:    [in/out] Number of GPUs used by process. Size of @devices array.
0557  * @num_bos     [in/out] Number of BOs used by process. Size of @bos array.
0558  * @num_objects:    [in/out] Number of objects used by process. Objects are opaque to
0559  *               user application.
0560  * @pid:        [in/out] PID of the process being checkpointed
0561  * @op          [in] Type of operation (kfd_criu_op)
0562  *
0563  * Return: 0 on success, -errno on failure
0564  */
0565 struct kfd_ioctl_criu_args {
0566     __u64 devices;      /* Used during ops: CHECKPOINT, RESTORE */
0567     __u64 bos;      /* Used during ops: CHECKPOINT, RESTORE */
0568     __u64 priv_data;    /* Used during ops: CHECKPOINT, RESTORE */
0569     __u64 priv_data_size;   /* Used during ops: PROCESS_INFO, RESTORE */
0570     __u32 num_devices;  /* Used during ops: PROCESS_INFO, RESTORE */
0571     __u32 num_bos;      /* Used during ops: PROCESS_INFO, RESTORE */
0572     __u32 num_objects;  /* Used during ops: PROCESS_INFO, RESTORE */
0573     __u32 pid;      /* Used during ops: PROCESS_INFO, RESUME */
0574     __u32 op;
0575 };
0576
0577 struct kfd_criu_device_bucket {
0578     __u32 user_gpu_id;
0579     __u32 actual_gpu_id;
0580     __u32 drm_fd;
0581     __u32 pad;
0582 };
0583
0584 struct kfd_criu_bo_bucket {
0585     __u64 addr;
0586     __u64 size;
0587     __u64 offset;
0588     __u64 restored_offset;    /* During restore, updated offset for BO */
0589     __u32 gpu_id;             /* This is the user_gpu_id */
0590     __u32 alloc_flags;
0591     __u32 dmabuf_fd;
0592     __u32 pad;
0593 };
0594
0595 /* CRIU IOCTLs - END */
0596 /**************************************************************************************************/
0597
0598 /* Register offset inside the remapped mmio page
0599  */
0600 enum kfd_mmio_remap {
0601     KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL = 0,
0602     KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL = 4,
0603 };
0604
0605 /* Guarantee host access to memory */
0606 #define KFD_IOCTL_SVM_FLAG_HOST_ACCESS 0x00000001
0607 /* Fine grained coherency between all devices with access */
0608 #define KFD_IOCTL_SVM_FLAG_COHERENT    0x00000002
0609 /* Use any GPU in same hive as preferred device */
0610 #define KFD_IOCTL_SVM_FLAG_HIVE_LOCAL  0x00000004
0611 /* GPUs only read, allows replication */
0612 #define KFD_IOCTL_SVM_FLAG_GPU_RO      0x00000008
0613 /* Allow execution on GPU */
0614 #define KFD_IOCTL_SVM_FLAG_GPU_EXEC    0x00000010
0615 /* GPUs mostly read, may allow similar optimizations as RO, but writes fault */
0616 #define KFD_IOCTL_SVM_FLAG_GPU_READ_MOSTLY     0x00000020
0617 /* Keep GPU memory mapping always valid as if XNACK is disable */
0618 #define KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED   0x00000040
0619
0620 /**
0621  * kfd_ioctl_svm_op - SVM ioctl operations
0622  *
0623  * @KFD_IOCTL_SVM_OP_SET_ATTR: Modify one or more attributes
0624  * @KFD_IOCTL_SVM_OP_GET_ATTR: Query one or more attributes
0625  */
0626 enum kfd_ioctl_svm_op {
0627     KFD_IOCTL_SVM_OP_SET_ATTR,
0628     KFD_IOCTL_SVM_OP_GET_ATTR
0629 };
0630
0631 /** kfd_ioctl_svm_location - Enum for preferred and prefetch locations
0632  *
0633  * GPU IDs are used to specify GPUs as preferred and prefetch locations.
0634  * Below definitions are used for system memory or for leaving the preferred
0635  * location unspecified.
0636  */
0637 enum kfd_ioctl_svm_location {
0638     KFD_IOCTL_SVM_LOCATION_SYSMEM = 0,
0639     KFD_IOCTL_SVM_LOCATION_UNDEFINED = 0xffffffff
0640 };
0641
0642 /**
0643  * kfd_ioctl_svm_attr_type - SVM attribute types
0644  *
0645  * @KFD_IOCTL_SVM_ATTR_PREFERRED_LOC: gpuid of the preferred location, 0 for
0646  *                                    system memory
0647  * @KFD_IOCTL_SVM_ATTR_PREFETCH_LOC: gpuid of the prefetch location, 0 for
0648  *                                   system memory. Setting this triggers an
0649  *                                   immediate prefetch (migration).
0650  * @KFD_IOCTL_SVM_ATTR_ACCESS:
0651  * @KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE:
0652  * @KFD_IOCTL_SVM_ATTR_NO_ACCESS: specify memory access for the gpuid given
0653  *                                by the attribute value
0654  * @KFD_IOCTL_SVM_ATTR_SET_FLAGS: bitmask of flags to set (see
0655  *                                KFD_IOCTL_SVM_FLAG_...)
0656  * @KFD_IOCTL_SVM_ATTR_CLR_FLAGS: bitmask of flags to clear
0657  * @KFD_IOCTL_SVM_ATTR_GRANULARITY: migration granularity
0658  *                                  (log2 num pages)
0659  */
0660 enum kfd_ioctl_svm_attr_type {
0661     KFD_IOCTL_SVM_ATTR_PREFERRED_LOC,
0662     KFD_IOCTL_SVM_ATTR_PREFETCH_LOC,
0663     KFD_IOCTL_SVM_ATTR_ACCESS,
0664     KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE,
0665     KFD_IOCTL_SVM_ATTR_NO_ACCESS,
0666     KFD_IOCTL_SVM_ATTR_SET_FLAGS,
0667     KFD_IOCTL_SVM_ATTR_CLR_FLAGS,
0668     KFD_IOCTL_SVM_ATTR_GRANULARITY
0669 };
0670
0671 /**
0672  * kfd_ioctl_svm_attribute - Attributes as pairs of type and value
0673  *
0674  * The meaning of the @value depends on the attribute type.
0675  *
0676  * @type: attribute type (see enum @kfd_ioctl_svm_attr_type)
0677  * @value: attribute value
0678  */
0679 struct kfd_ioctl_svm_attribute {
0680     __u32 type;
0681     __u32 value;
0682 };
0683
0684 /**
0685  * kfd_ioctl_svm_args - Arguments for SVM ioctl
0686  *
0687  * @op specifies the operation to perform (see enum
0688  * @kfd_ioctl_svm_op).  @start_addr and @size are common for all
0689  * operations.
0690  *
0691  * A variable number of attributes can be given in @attrs.
0692  * @nattr specifies the number of attributes. New attributes can be
0693  * added in the future without breaking the ABI. If unknown attributes
0694  * are given, the function returns -EINVAL.
0695  *
0696  * @KFD_IOCTL_SVM_OP_SET_ATTR sets attributes for a virtual address
0697  * range. It may overlap existing virtual address ranges. If it does,
0698  * the existing ranges will be split such that the attribute changes
0699  * only apply to the specified address range.
0700  *
0701  * @KFD_IOCTL_SVM_OP_GET_ATTR returns the intersection of attributes
0702  * over all memory in the given range and returns the result as the
0703  * attribute value. If different pages have different preferred or
0704  * prefetch locations, 0xffffffff will be returned for
0705  * @KFD_IOCTL_SVM_ATTR_PREFERRED_LOC or
0706  * @KFD_IOCTL_SVM_ATTR_PREFETCH_LOC resepctively. For
0707  * @KFD_IOCTL_SVM_ATTR_SET_FLAGS, flags of all pages will be
0708  * aggregated by bitwise AND. That means, a flag will be set in the
0709  * output, if that flag is set for all pages in the range. For
0710  * @KFD_IOCTL_SVM_ATTR_CLR_FLAGS, flags of all pages will be
0711  * aggregated by bitwise NOR. That means, a flag will be set in the
0712  * output, if that flag is clear for all pages in the range.
0713  * The minimum migration granularity throughout the range will be
0714  * returned for @KFD_IOCTL_SVM_ATTR_GRANULARITY.
0715  *
0716  * Querying of accessibility attributes works by initializing the
0717  * attribute type to @KFD_IOCTL_SVM_ATTR_ACCESS and the value to the
0718  * GPUID being queried. Multiple attributes can be given to allow
0719  * querying multiple GPUIDs. The ioctl function overwrites the
0720  * attribute type to indicate the access for the specified GPU.
0721  */
0722 struct kfd_ioctl_svm_args {
0723     __u64 start_addr;
0724     __u64 size;
0725     __u32 op;
0726     __u32 nattr;
0727     /* Variable length array of attributes */
0728     struct kfd_ioctl_svm_attribute attrs[];
0729 };
0730
0731 /**
0732  * kfd_ioctl_set_xnack_mode_args - Arguments for set_xnack_mode
0733  *
0734  * @xnack_enabled:       [in/out] Whether to enable XNACK mode for this process
0735  *
0736  * @xnack_enabled indicates whether recoverable page faults should be
0737  * enabled for the current process. 0 means disabled, positive means
0738  * enabled, negative means leave unchanged. If enabled, virtual address
0739  * translations on GFXv9 and later AMD GPUs can return XNACK and retry
0740  * the access until a valid PTE is available. This is used to implement
0741  * device page faults.
0742  *
0743  * On output, @xnack_enabled returns the (new) current mode (0 or
0744  * positive). Therefore, a negative input value can be used to query
0745  * the current mode without changing it.
0746  *
0747  * The XNACK mode fundamentally changes the way SVM managed memory works
0748  * in the driver, with subtle effects on application performance and
0749  * functionality.
0750  *
0751  * Enabling XNACK mode requires shader programs to be compiled
0752  * differently. Furthermore, not all GPUs support changing the mode
0753  * per-process. Therefore changing the mode is only allowed while no
0754  * user mode queues exist in the process. This ensure that no shader
0755  * code is running that may be compiled for the wrong mode. And GPUs
0756  * that cannot change to the requested mode will prevent the XNACK
0757  * mode from occurring. All GPUs used by the process must be in the
0758  * same XNACK mode.
0759  *
0760  * GFXv8 or older GPUs do not support 48 bit virtual addresses or SVM.
0761  * Therefore those GPUs are not considered for the XNACK mode switch.
0762  *
0763  * Return: 0 on success, -errno on failure
0764  */
0765 struct kfd_ioctl_set_xnack_mode_args {
0766     __s32 xnack_enabled;
0767 };
0768
0769 #define AMDKFD_IOCTL_BASE 'K'
0770 #define AMDKFD_IO(nr)           _IO(AMDKFD_IOCTL_BASE, nr)
0771 #define AMDKFD_IOR(nr, type)        _IOR(AMDKFD_IOCTL_BASE, nr, type)
0772 #define AMDKFD_IOW(nr, type)        _IOW(AMDKFD_IOCTL_BASE, nr, type)
0773 #define AMDKFD_IOWR(nr, type)       _IOWR(AMDKFD_IOCTL_BASE, nr, type)
0774
0775 #define AMDKFD_IOC_GET_VERSION          \
0776         AMDKFD_IOR(0x01, struct kfd_ioctl_get_version_args)
0777
0778 #define AMDKFD_IOC_CREATE_QUEUE         \
0779         AMDKFD_IOWR(0x02, struct kfd_ioctl_create_queue_args)
0780
0781 #define AMDKFD_IOC_DESTROY_QUEUE        \
0782         AMDKFD_IOWR(0x03, struct kfd_ioctl_destroy_queue_args)
0783
0784 #define AMDKFD_IOC_SET_MEMORY_POLICY        \
0785         AMDKFD_IOW(0x04, struct kfd_ioctl_set_memory_policy_args)
0786
0787 #define AMDKFD_IOC_GET_CLOCK_COUNTERS       \
0788         AMDKFD_IOWR(0x05, struct kfd_ioctl_get_clock_counters_args)
0789
0790 #define AMDKFD_IOC_GET_PROCESS_APERTURES    \
0791         AMDKFD_IOR(0x06, struct kfd_ioctl_get_process_apertures_args)
0792
0793 #define AMDKFD_IOC_UPDATE_QUEUE         \
0794         AMDKFD_IOW(0x07, struct kfd_ioctl_update_queue_args)
0795
0796 #define AMDKFD_IOC_CREATE_EVENT         \
0797         AMDKFD_IOWR(0x08, struct kfd_ioctl_create_event_args)
0798
0799 #define AMDKFD_IOC_DESTROY_EVENT        \
0800         AMDKFD_IOW(0x09, struct kfd_ioctl_destroy_event_args)
0801
0802 #define AMDKFD_IOC_SET_EVENT            \
0803         AMDKFD_IOW(0x0A, struct kfd_ioctl_set_event_args)
0804
0805 #define AMDKFD_IOC_RESET_EVENT          \
0806         AMDKFD_IOW(0x0B, struct kfd_ioctl_reset_event_args)
0807
0808 #define AMDKFD_IOC_WAIT_EVENTS          \
0809         AMDKFD_IOWR(0x0C, struct kfd_ioctl_wait_events_args)
0810
0811 #define AMDKFD_IOC_DBG_REGISTER_DEPRECATED  \
0812         AMDKFD_IOW(0x0D, struct kfd_ioctl_dbg_register_args)
0813
0814 #define AMDKFD_IOC_DBG_UNREGISTER_DEPRECATED    \
0815         AMDKFD_IOW(0x0E, struct kfd_ioctl_dbg_unregister_args)
0816
0817 #define AMDKFD_IOC_DBG_ADDRESS_WATCH_DEPRECATED \
0818         AMDKFD_IOW(0x0F, struct kfd_ioctl_dbg_address_watch_args)
0819
0820 #define AMDKFD_IOC_DBG_WAVE_CONTROL_DEPRECATED  \
0821         AMDKFD_IOW(0x10, struct kfd_ioctl_dbg_wave_control_args)
0822
0823 #define AMDKFD_IOC_SET_SCRATCH_BACKING_VA   \
0824         AMDKFD_IOWR(0x11, struct kfd_ioctl_set_scratch_backing_va_args)
0825
0826 #define AMDKFD_IOC_GET_TILE_CONFIG                                      \
0827         AMDKFD_IOWR(0x12, struct kfd_ioctl_get_tile_config_args)
0828
0829 #define AMDKFD_IOC_SET_TRAP_HANDLER     \
0830         AMDKFD_IOW(0x13, struct kfd_ioctl_set_trap_handler_args)
0831
0832 #define AMDKFD_IOC_GET_PROCESS_APERTURES_NEW    \
0833         AMDKFD_IOWR(0x14,       \
0834             struct kfd_ioctl_get_process_apertures_new_args)
0835
0836 #define AMDKFD_IOC_ACQUIRE_VM           \
0837         AMDKFD_IOW(0x15, struct kfd_ioctl_acquire_vm_args)
0838
0839 #define AMDKFD_IOC_ALLOC_MEMORY_OF_GPU      \
0840         AMDKFD_IOWR(0x16, struct kfd_ioctl_alloc_memory_of_gpu_args)
0841
0842 #define AMDKFD_IOC_FREE_MEMORY_OF_GPU       \
0843         AMDKFD_IOW(0x17, struct kfd_ioctl_free_memory_of_gpu_args)
0844
0845 #define AMDKFD_IOC_MAP_MEMORY_TO_GPU        \
0846         AMDKFD_IOWR(0x18, struct kfd_ioctl_map_memory_to_gpu_args)
0847
0848 #define AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU    \
0849         AMDKFD_IOWR(0x19, struct kfd_ioctl_unmap_memory_from_gpu_args)
0850
0851 #define AMDKFD_IOC_SET_CU_MASK      \
0852         AMDKFD_IOW(0x1A, struct kfd_ioctl_set_cu_mask_args)
0853
0854 #define AMDKFD_IOC_GET_QUEUE_WAVE_STATE     \
0855         AMDKFD_IOWR(0x1B, struct kfd_ioctl_get_queue_wave_state_args)
0856
0857 #define AMDKFD_IOC_GET_DMABUF_INFO      \
0858         AMDKFD_IOWR(0x1C, struct kfd_ioctl_get_dmabuf_info_args)
0859
0860 #define AMDKFD_IOC_IMPORT_DMABUF        \
0861         AMDKFD_IOWR(0x1D, struct kfd_ioctl_import_dmabuf_args)
0862
0863 #define AMDKFD_IOC_ALLOC_QUEUE_GWS      \
0864         AMDKFD_IOWR(0x1E, struct kfd_ioctl_alloc_queue_gws_args)
0865
0866 #define AMDKFD_IOC_SMI_EVENTS           \
0867         AMDKFD_IOWR(0x1F, struct kfd_ioctl_smi_events_args)
0868
0869 #define AMDKFD_IOC_SVM  AMDKFD_IOWR(0x20, struct kfd_ioctl_svm_args)
0870
0871 #define AMDKFD_IOC_SET_XNACK_MODE       \
0872         AMDKFD_IOWR(0x21, struct kfd_ioctl_set_xnack_mode_args)
0873
0874 #define AMDKFD_IOC_CRIU_OP          \
0875         AMDKFD_IOWR(0x22, struct kfd_ioctl_criu_args)
0876
0877 #define AMDKFD_IOC_AVAILABLE_MEMORY     \
0878         AMDKFD_IOWR(0x23, struct kfd_ioctl_get_available_memory_args)
0879
0880 #define AMDKFD_COMMAND_START        0x01
0881 #define AMDKFD_COMMAND_END      0x24
0882
0883 #endif