ufs/core/ufshpb.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /*
0003  * Universal Flash Storage Host Performance Booster
0004  *
0005  * Copyright (C) 2017-2021 Samsung Electronics Co., Ltd.
0006  *
0007  * Authors:
0008  *  Yongmyung Lee <ymhungry.lee@samsung.com>
0009  *  Jinyoung Choi <j-young.choi@samsung.com>
0010  */
0011
0012 #ifndef _UFSHPB_H_
0013 #define _UFSHPB_H_
0014
0015 /* hpb response UPIU macro */
0016 #define HPB_RSP_NONE                0x0
0017 #define HPB_RSP_REQ_REGION_UPDATE       0x1
0018 #define HPB_RSP_DEV_RESET           0x2
0019 #define MAX_ACTIVE_NUM              2
0020 #define MAX_INACTIVE_NUM            2
0021 #define DEV_DATA_SEG_LEN            0x14
0022 #define DEV_SENSE_SEG_LEN           0x12
0023 #define DEV_DES_TYPE                0x80
0024 #define DEV_ADDITIONAL_LEN          0x10
0025
0026 /* hpb map & entries macro */
0027 #define HPB_RGN_SIZE_UNIT           512
0028 #define HPB_ENTRY_BLOCK_SIZE            4096
0029 #define HPB_ENTRY_SIZE              0x8
0030 #define PINNED_NOT_SET              U32_MAX
0031
0032 /* hpb support chunk size */
0033 #define HPB_LEGACY_CHUNK_HIGH           1
0034 #define HPB_MULTI_CHUNK_HIGH            255
0035
0036 /* hpb vender defined opcode */
0037 #define UFSHPB_READ             0xF8
0038 #define UFSHPB_READ_BUFFER          0xF9
0039 #define UFSHPB_READ_BUFFER_ID           0x01
0040 #define UFSHPB_WRITE_BUFFER         0xFA
0041 #define UFSHPB_WRITE_BUFFER_INACT_SINGLE_ID 0x01
0042 #define UFSHPB_WRITE_BUFFER_PREFETCH_ID     0x02
0043 #define UFSHPB_WRITE_BUFFER_INACT_ALL_ID    0x03
0044 #define HPB_WRITE_BUFFER_CMD_LENGTH     10
0045 #define MAX_HPB_READ_ID             0x7F
0046 #define HPB_READ_BUFFER_CMD_LENGTH      10
0047 #define LU_ENABLED_HPB_FUNC         0x02
0048
0049 #define HPB_RESET_REQ_RETRIES           10
0050 #define HPB_MAP_REQ_RETRIES         5
0051 #define HPB_REQUEUE_TIME_MS         0
0052
0053 #define HPB_SUPPORT_VERSION         0x200
0054 #define HPB_SUPPORT_LEGACY_VERSION      0x100
0055
0056 enum UFSHPB_MODE {
0057     HPB_HOST_CONTROL,
0058     HPB_DEVICE_CONTROL,
0059 };
0060
0061 enum UFSHPB_STATE {
0062     HPB_INIT,
0063     HPB_PRESENT,
0064     HPB_SUSPEND,
0065     HPB_FAILED,
0066     HPB_RESET,
0067 };
0068
0069 enum HPB_RGN_STATE {
0070     HPB_RGN_INACTIVE,
0071     HPB_RGN_ACTIVE,
0072     /* pinned regions are always active */
0073     HPB_RGN_PINNED,
0074 };
0075
0076 enum HPB_SRGN_STATE {
0077     HPB_SRGN_UNUSED,
0078     HPB_SRGN_INVALID,
0079     HPB_SRGN_VALID,
0080     HPB_SRGN_ISSUED,
0081 };
0082
0083 /**
0084  * struct ufshpb_lu_info - UFSHPB logical unit related info
0085  * @num_blocks: the number of logical block
0086  * @pinned_start: the start region number of pinned region
0087  * @num_pinned: the number of pinned regions
0088  * @max_active_rgns: maximum number of active regions
0089  */
0090 struct ufshpb_lu_info {
0091     int num_blocks;
0092     int pinned_start;
0093     int num_pinned;
0094     int max_active_rgns;
0095 };
0096
0097 struct ufshpb_map_ctx {
0098     struct page **m_page;
0099     unsigned long *ppn_dirty;
0100 };
0101
0102 struct ufshpb_subregion {
0103     struct ufshpb_map_ctx *mctx;
0104     enum HPB_SRGN_STATE srgn_state;
0105     int rgn_idx;
0106     int srgn_idx;
0107     bool is_last;
0108
0109     /* subregion reads - for host mode */
0110     unsigned int reads;
0111
0112     /* below information is used by rsp_list */
0113     struct list_head list_act_srgn;
0114 };
0115
0116 struct ufshpb_region {
0117     struct ufshpb_lu *hpb;
0118     struct ufshpb_subregion *srgn_tbl;
0119     enum HPB_RGN_STATE rgn_state;
0120     int rgn_idx;
0121     int srgn_cnt;
0122
0123     /* below information is used by rsp_list */
0124     struct list_head list_inact_rgn;
0125
0126     /* below information is used by lru */
0127     struct list_head list_lru_rgn;
0128     unsigned long rgn_flags;
0129 #define RGN_FLAG_DIRTY 0
0130 #define RGN_FLAG_UPDATE 1
0131
0132     /* region reads - for host mode */
0133     spinlock_t rgn_lock;
0134     unsigned int reads;
0135     /* region "cold" timer - for host mode */
0136     ktime_t read_timeout;
0137     unsigned int read_timeout_expiries;
0138     struct list_head list_expired_rgn;
0139 };
0140
0141 #define for_each_sub_region(rgn, i, srgn)               \
0142     for ((i) = 0;                           \
0143          ((i) < (rgn)->srgn_cnt) && ((srgn) = &(rgn)->srgn_tbl[i]); \
0144          (i)++)
0145
0146 /**
0147  * struct ufshpb_req - HPB related request structure (write/read buffer)
0148  * @req: block layer request structure
0149  * @bio: bio for this request
0150  * @hpb: ufshpb_lu structure that related to
0151  * @list_req: ufshpb_req mempool list
0152  * @sense: store its sense data
0153  * @mctx: L2P map information
0154  * @rgn_idx: target region index
0155  * @srgn_idx: target sub-region index
0156  * @lun: target logical unit number
0157  * @m_page: L2P map information data for pre-request
0158  * @len: length of host-side cached L2P map in m_page
0159  * @lpn: start LPN of L2P map in m_page
0160  */
0161 struct ufshpb_req {
0162     struct request *req;
0163     struct bio *bio;
0164     struct ufshpb_lu *hpb;
0165     struct list_head list_req;
0166     union {
0167         struct {
0168             struct ufshpb_map_ctx *mctx;
0169             unsigned int rgn_idx;
0170             unsigned int srgn_idx;
0171             unsigned int lun;
0172         } rb;
0173         struct {
0174             struct page *m_page;
0175             unsigned int len;
0176             unsigned long lpn;
0177         } wb;
0178     };
0179 };
0180
0181 struct victim_select_info {
0182     struct list_head lh_lru_rgn; /* LRU list of regions */
0183     int max_lru_active_cnt; /* supported hpb #region - pinned #region */
0184     atomic_t active_cnt;
0185 };
0186
0187 /**
0188  * ufshpb_params - ufs hpb parameters
0189  * @requeue_timeout_ms - requeue threshold of wb command (0x2)
0190  * @activation_thld - min reads [IOs] to activate/update a region
0191  * @normalization_factor - shift right the region's reads
0192  * @eviction_thld_enter - min reads [IOs] for the entering region in eviction
0193  * @eviction_thld_exit - max reads [IOs] for the exiting region in eviction
0194  * @read_timeout_ms - timeout [ms] from the last read IO to the region
0195  * @read_timeout_expiries - amount of allowable timeout expireis
0196  * @timeout_polling_interval_ms - frequency in which timeouts are checked
0197  * @inflight_map_req - number of inflight map requests
0198  */
0199 struct ufshpb_params {
0200     unsigned int requeue_timeout_ms;
0201     unsigned int activation_thld;
0202     unsigned int normalization_factor;
0203     unsigned int eviction_thld_enter;
0204     unsigned int eviction_thld_exit;
0205     unsigned int read_timeout_ms;
0206     unsigned int read_timeout_expiries;
0207     unsigned int timeout_polling_interval_ms;
0208     unsigned int inflight_map_req;
0209 };
0210
0211 struct ufshpb_stats {
0212     u64 hit_cnt;
0213     u64 miss_cnt;
0214     u64 rcmd_noti_cnt;
0215     u64 rcmd_active_cnt;
0216     u64 rcmd_inactive_cnt;
0217     u64 map_req_cnt;
0218     u64 pre_req_cnt;
0219     u64 umap_req_cnt;
0220 };
0221
0222 struct ufshpb_lu {
0223     int lun;
0224     struct scsi_device *sdev_ufs_lu;
0225
0226     spinlock_t rgn_state_lock; /* for protect rgn/srgn state */
0227     struct ufshpb_region *rgn_tbl;
0228
0229     atomic_t hpb_state;
0230
0231     spinlock_t rsp_list_lock;
0232     struct list_head lh_act_srgn; /* hold rsp_list_lock */
0233     struct list_head lh_inact_rgn; /* hold rsp_list_lock */
0234
0235     /* pre request information */
0236     struct ufshpb_req *pre_req;
0237     int num_inflight_pre_req;
0238     int throttle_pre_req;
0239     int num_inflight_map_req; /* hold param_lock */
0240     spinlock_t param_lock;
0241
0242     struct list_head lh_pre_req_free;
0243     int pre_req_max_tr_len;
0244
0245     /* cached L2P map management worker */
0246     struct work_struct map_work;
0247
0248     /* for selecting victim */
0249     struct victim_select_info lru_info;
0250     struct work_struct ufshpb_normalization_work;
0251     struct delayed_work ufshpb_read_to_work;
0252     unsigned long work_data_bits;
0253 #define TIMEOUT_WORK_RUNNING 0
0254
0255     /* pinned region information */
0256     u32 lu_pinned_start;
0257     u32 lu_pinned_end;
0258
0259     /* HPB related configuration */
0260     u32 rgns_per_lu;
0261     u32 srgns_per_lu;
0262     u32 last_srgn_entries;
0263     int srgns_per_rgn;
0264     u32 srgn_mem_size;
0265     u32 entries_per_rgn_mask;
0266     u32 entries_per_rgn_shift;
0267     u32 entries_per_srgn;
0268     u32 entries_per_srgn_mask;
0269     u32 entries_per_srgn_shift;
0270     u32 pages_per_srgn;
0271
0272     bool is_hcm;
0273
0274     struct ufshpb_stats stats;
0275     struct ufshpb_params params;
0276
0277     struct kmem_cache *map_req_cache;
0278     struct kmem_cache *m_page_cache;
0279
0280     struct list_head list_hpb_lu;
0281 };
0282
0283 struct ufs_hba;
0284 struct ufshcd_lrb;
0285
0286 #ifndef CONFIG_SCSI_UFS_HPB
0287 static int ufshpb_prep(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) { return 0; }
0288 static void ufshpb_rsp_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) {}
0289 static void ufshpb_resume(struct ufs_hba *hba) {}
0290 static void ufshpb_suspend(struct ufs_hba *hba) {}
0291 static void ufshpb_toggle_state(struct ufs_hba *hba, enum UFSHPB_STATE src, enum UFSHPB_STATE dest) {}
0292 static void ufshpb_init(struct ufs_hba *hba) {}
0293 static void ufshpb_init_hpb_lu(struct ufs_hba *hba, struct scsi_device *sdev) {}
0294 static void ufshpb_destroy_lu(struct ufs_hba *hba, struct scsi_device *sdev) {}
0295 static void ufshpb_remove(struct ufs_hba *hba) {}
0296 static bool ufshpb_is_allowed(struct ufs_hba *hba) { return false; }
0297 static void ufshpb_get_geo_info(struct ufs_hba *hba, u8 *geo_buf) {}
0298 static void ufshpb_get_dev_info(struct ufs_hba *hba, u8 *desc_buf) {}
0299 static bool ufshpb_is_legacy(struct ufs_hba *hba) { return false; }
0300 #else
0301 int ufshpb_prep(struct ufs_hba *hba, struct ufshcd_lrb *lrbp);
0302 void ufshpb_rsp_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp);
0303 void ufshpb_resume(struct ufs_hba *hba);
0304 void ufshpb_suspend(struct ufs_hba *hba);
0305 void ufshpb_toggle_state(struct ufs_hba *hba, enum UFSHPB_STATE src, enum UFSHPB_STATE dest);
0306 void ufshpb_init(struct ufs_hba *hba);
0307 void ufshpb_init_hpb_lu(struct ufs_hba *hba, struct scsi_device *sdev);
0308 void ufshpb_destroy_lu(struct ufs_hba *hba, struct scsi_device *sdev);
0309 void ufshpb_remove(struct ufs_hba *hba);
0310 bool ufshpb_is_allowed(struct ufs_hba *hba);
0311 void ufshpb_get_geo_info(struct ufs_hba *hba, u8 *geo_buf);
0312 void ufshpb_get_dev_info(struct ufs_hba *hba, u8 *desc_buf);
0313 bool ufshpb_is_legacy(struct ufs_hba *hba);
0314 extern struct attribute_group ufs_sysfs_hpb_stat_group;
0315 extern struct attribute_group ufs_sysfs_hpb_param_group;
0316 #endif
0317
0318 #endif /* End of Header */