cpu/resctrl/internal.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 #ifndef _ASM_X86_RESCTRL_INTERNAL_H
0003 #define _ASM_X86_RESCTRL_INTERNAL_H
0004
0005 #include <linux/resctrl.h>
0006 #include <linux/sched.h>
0007 #include <linux/kernfs.h>
0008 #include <linux/fs_context.h>
0009 #include <linux/jump_label.h>
0010
0011 #define MSR_IA32_L3_QOS_CFG     0xc81
0012 #define MSR_IA32_L2_QOS_CFG     0xc82
0013 #define MSR_IA32_L3_CBM_BASE        0xc90
0014 #define MSR_IA32_L2_CBM_BASE        0xd10
0015 #define MSR_IA32_MBA_THRTL_BASE     0xd50
0016 #define MSR_IA32_MBA_BW_BASE        0xc0000200
0017
0018 #define MSR_IA32_QM_CTR         0x0c8e
0019 #define MSR_IA32_QM_EVTSEL      0x0c8d
0020
0021 #define L3_QOS_CDP_ENABLE       0x01ULL
0022
0023 #define L2_QOS_CDP_ENABLE       0x01ULL
0024
0025 /*
0026  * Event IDs are used to program IA32_QM_EVTSEL before reading event
0027  * counter from IA32_QM_CTR
0028  */
0029 #define QOS_L3_OCCUP_EVENT_ID       0x01
0030 #define QOS_L3_MBM_TOTAL_EVENT_ID   0x02
0031 #define QOS_L3_MBM_LOCAL_EVENT_ID   0x03
0032
0033 #define CQM_LIMBOCHECK_INTERVAL 1000
0034
0035 #define MBM_CNTR_WIDTH_BASE     24
0036 #define MBM_OVERFLOW_INTERVAL       1000
0037 #define MAX_MBA_BW          100u
0038 #define MBA_IS_LINEAR           0x4
0039 #define MBA_MAX_MBPS            U32_MAX
0040 #define MAX_MBA_BW_AMD          0x800
0041 #define MBM_CNTR_WIDTH_OFFSET_AMD   20
0042
0043 #define RMID_VAL_ERROR          BIT_ULL(63)
0044 #define RMID_VAL_UNAVAIL        BIT_ULL(62)
0045 /*
0046  * With the above fields in use 62 bits remain in MSR_IA32_QM_CTR for
0047  * data to be returned. The counter width is discovered from the hardware
0048  * as an offset from MBM_CNTR_WIDTH_BASE.
0049  */
0050 #define MBM_CNTR_WIDTH_OFFSET_MAX (62 - MBM_CNTR_WIDTH_BASE)
0051
0052
0053 struct rdt_fs_context {
0054     struct kernfs_fs_context    kfc;
0055     bool                enable_cdpl2;
0056     bool                enable_cdpl3;
0057     bool                enable_mba_mbps;
0058 };
0059
0060 static inline struct rdt_fs_context *rdt_fc2context(struct fs_context *fc)
0061 {
0062     struct kernfs_fs_context *kfc = fc->fs_private;
0063
0064     return container_of(kfc, struct rdt_fs_context, kfc);
0065 }
0066
0067 DECLARE_STATIC_KEY_FALSE(rdt_enable_key);
0068 DECLARE_STATIC_KEY_FALSE(rdt_mon_enable_key);
0069
0070 /**
0071  * struct mon_evt - Entry in the event list of a resource
0072  * @evtid:      event id
0073  * @name:       name of the event
0074  * @list:       entry in &rdt_resource->evt_list
0075  */
0076 struct mon_evt {
0077     u32         evtid;
0078     char            *name;
0079     struct list_head    list;
0080 };
0081
0082 /**
0083  * union mon_data_bits - Monitoring details for each event file
0084  * @priv:              Used to store monitoring event data in @u
0085  *                     as kernfs private data
0086  * @rid:               Resource id associated with the event file
0087  * @evtid:             Event id associated with the event file
0088  * @domid:             The domain to which the event file belongs
0089  * @u:                 Name of the bit fields struct
0090  */
0091 union mon_data_bits {
0092     void *priv;
0093     struct {
0094         unsigned int rid    : 10;
0095         unsigned int evtid  : 8;
0096         unsigned int domid  : 14;
0097     } u;
0098 };
0099
0100 struct rmid_read {
0101     struct rdtgroup     *rgrp;
0102     struct rdt_resource *r;
0103     struct rdt_domain   *d;
0104     int         evtid;
0105     bool            first;
0106     u64         val;
0107 };
0108
0109 extern unsigned int resctrl_cqm_threshold;
0110 extern bool rdt_alloc_capable;
0111 extern bool rdt_mon_capable;
0112 extern unsigned int rdt_mon_features;
0113 extern struct list_head resctrl_schema_all;
0114
0115 enum rdt_group_type {
0116     RDTCTRL_GROUP = 0,
0117     RDTMON_GROUP,
0118     RDT_NUM_GROUP,
0119 };
0120
0121 /**
0122  * enum rdtgrp_mode - Mode of a RDT resource group
0123  * @RDT_MODE_SHAREABLE: This resource group allows sharing of its allocations
0124  * @RDT_MODE_EXCLUSIVE: No sharing of this resource group's allocations allowed
0125  * @RDT_MODE_PSEUDO_LOCKSETUP: Resource group will be used for Pseudo-Locking
0126  * @RDT_MODE_PSEUDO_LOCKED: No sharing of this resource group's allocations
0127  *                          allowed AND the allocations are Cache Pseudo-Locked
0128  * @RDT_NUM_MODES: Total number of modes
0129  *
0130  * The mode of a resource group enables control over the allowed overlap
0131  * between allocations associated with different resource groups (classes
0132  * of service). User is able to modify the mode of a resource group by
0133  * writing to the "mode" resctrl file associated with the resource group.
0134  *
0135  * The "shareable", "exclusive", and "pseudo-locksetup" modes are set by
0136  * writing the appropriate text to the "mode" file. A resource group enters
0137  * "pseudo-locked" mode after the schemata is written while the resource
0138  * group is in "pseudo-locksetup" mode.
0139  */
0140 enum rdtgrp_mode {
0141     RDT_MODE_SHAREABLE = 0,
0142     RDT_MODE_EXCLUSIVE,
0143     RDT_MODE_PSEUDO_LOCKSETUP,
0144     RDT_MODE_PSEUDO_LOCKED,
0145
0146     /* Must be last */
0147     RDT_NUM_MODES,
0148 };
0149
0150 /**
0151  * struct mongroup - store mon group's data in resctrl fs.
0152  * @mon_data_kn:        kernfs node for the mon_data directory
0153  * @parent:         parent rdtgrp
0154  * @crdtgrp_list:       child rdtgroup node list
0155  * @rmid:           rmid for this rdtgroup
0156  */
0157 struct mongroup {
0158     struct kernfs_node  *mon_data_kn;
0159     struct rdtgroup     *parent;
0160     struct list_head    crdtgrp_list;
0161     u32         rmid;
0162 };
0163
0164 /**
0165  * struct pseudo_lock_region - pseudo-lock region information
0166  * @s:          Resctrl schema for the resource to which this
0167  *          pseudo-locked region belongs
0168  * @d:          RDT domain to which this pseudo-locked region
0169  *          belongs
0170  * @cbm:        bitmask of the pseudo-locked region
0171  * @lock_thread_wq: waitqueue used to wait on the pseudo-locking thread
0172  *          completion
0173  * @thread_done:    variable used by waitqueue to test if pseudo-locking
0174  *          thread completed
0175  * @cpu:        core associated with the cache on which the setup code
0176  *          will be run
0177  * @line_size:      size of the cache lines
0178  * @size:       size of pseudo-locked region in bytes
0179  * @kmem:       the kernel memory associated with pseudo-locked region
0180  * @minor:      minor number of character device associated with this
0181  *          region
0182  * @debugfs_dir:    pointer to this region's directory in the debugfs
0183  *          filesystem
0184  * @pm_reqs:        Power management QoS requests related to this region
0185  */
0186 struct pseudo_lock_region {
0187     struct resctrl_schema   *s;
0188     struct rdt_domain   *d;
0189     u32         cbm;
0190     wait_queue_head_t   lock_thread_wq;
0191     int         thread_done;
0192     int         cpu;
0193     unsigned int        line_size;
0194     unsigned int        size;
0195     void            *kmem;
0196     unsigned int        minor;
0197     struct dentry       *debugfs_dir;
0198     struct list_head    pm_reqs;
0199 };
0200
0201 /**
0202  * struct rdtgroup - store rdtgroup's data in resctrl file system.
0203  * @kn:             kernfs node
0204  * @rdtgroup_list:      linked list for all rdtgroups
0205  * @closid:         closid for this rdtgroup
0206  * @cpu_mask:           CPUs assigned to this rdtgroup
0207  * @flags:          status bits
0208  * @waitcount:          how many cpus expect to find this
0209  *              group when they acquire rdtgroup_mutex
0210  * @type:           indicates type of this rdtgroup - either
0211  *              monitor only or ctrl_mon group
0212  * @mon:            mongroup related data
0213  * @mode:           mode of resource group
0214  * @plr:            pseudo-locked region
0215  */
0216 struct rdtgroup {
0217     struct kernfs_node      *kn;
0218     struct list_head        rdtgroup_list;
0219     u32             closid;
0220     struct cpumask          cpu_mask;
0221     int             flags;
0222     atomic_t            waitcount;
0223     enum rdt_group_type     type;
0224     struct mongroup         mon;
0225     enum rdtgrp_mode        mode;
0226     struct pseudo_lock_region   *plr;
0227 };
0228
0229 /* rdtgroup.flags */
0230 #define RDT_DELETED     1
0231
0232 /* rftype.flags */
0233 #define RFTYPE_FLAGS_CPUS_LIST  1
0234
0235 /*
0236  * Define the file type flags for base and info directories.
0237  */
0238 #define RFTYPE_INFO         BIT(0)
0239 #define RFTYPE_BASE         BIT(1)
0240 #define RF_CTRLSHIFT            4
0241 #define RF_MONSHIFT         5
0242 #define RF_TOPSHIFT         6
0243 #define RFTYPE_CTRL         BIT(RF_CTRLSHIFT)
0244 #define RFTYPE_MON          BIT(RF_MONSHIFT)
0245 #define RFTYPE_TOP          BIT(RF_TOPSHIFT)
0246 #define RFTYPE_RES_CACHE        BIT(8)
0247 #define RFTYPE_RES_MB           BIT(9)
0248 #define RF_CTRL_INFO            (RFTYPE_INFO | RFTYPE_CTRL)
0249 #define RF_MON_INFO         (RFTYPE_INFO | RFTYPE_MON)
0250 #define RF_TOP_INFO         (RFTYPE_INFO | RFTYPE_TOP)
0251 #define RF_CTRL_BASE            (RFTYPE_BASE | RFTYPE_CTRL)
0252
0253 /* List of all resource groups */
0254 extern struct list_head rdt_all_groups;
0255
0256 extern int max_name_width, max_data_width;
0257
0258 int __init rdtgroup_init(void);
0259 void __exit rdtgroup_exit(void);
0260
0261 /**
0262  * struct rftype - describe each file in the resctrl file system
0263  * @name:   File name
0264  * @mode:   Access mode
0265  * @kf_ops: File operations
0266  * @flags:  File specific RFTYPE_FLAGS_* flags
0267  * @fflags: File specific RF_* or RFTYPE_* flags
0268  * @seq_show:   Show content of the file
0269  * @write:  Write to the file
0270  */
0271 struct rftype {
0272     char            *name;
0273     umode_t         mode;
0274     const struct kernfs_ops *kf_ops;
0275     unsigned long       flags;
0276     unsigned long       fflags;
0277
0278     int (*seq_show)(struct kernfs_open_file *of,
0279             struct seq_file *sf, void *v);
0280     /*
0281      * write() is the generic write callback which maps directly to
0282      * kernfs write operation and overrides all other operations.
0283      * Maximum write size is determined by ->max_write_len.
0284      */
0285     ssize_t (*write)(struct kernfs_open_file *of,
0286              char *buf, size_t nbytes, loff_t off);
0287 };
0288
0289 /**
0290  * struct mbm_state - status for each MBM counter in each domain
0291  * @chunks: Total data moved (multiply by rdt_group.mon_scale to get bytes)
0292  * @prev_msr:   Value of IA32_QM_CTR for this RMID last time we read it
0293  * @prev_bw_msr:Value of previous IA32_QM_CTR for bandwidth counting
0294  * @prev_bw:    The most recent bandwidth in MBps
0295  * @delta_bw:   Difference between the current and previous bandwidth
0296  * @delta_comp: Indicates whether to compute the delta_bw
0297  */
0298 struct mbm_state {
0299     u64 chunks;
0300     u64 prev_msr;
0301     u64 prev_bw_msr;
0302     u32 prev_bw;
0303     u32 delta_bw;
0304     bool    delta_comp;
0305 };
0306
0307 /**
0308  * struct rdt_hw_domain - Arch private attributes of a set of CPUs that share
0309  *            a resource
0310  * @d_resctrl:  Properties exposed to the resctrl file system
0311  * @ctrl_val:   array of cache or mem ctrl values (indexed by CLOSID)
0312  * @mbps_val:   When mba_sc is enabled, this holds the bandwidth in MBps
0313  *
0314  * Members of this structure are accessed via helpers that provide abstraction.
0315  */
0316 struct rdt_hw_domain {
0317     struct rdt_domain       d_resctrl;
0318     u32             *ctrl_val;
0319     u32             *mbps_val;
0320 };
0321
0322 static inline struct rdt_hw_domain *resctrl_to_arch_dom(struct rdt_domain *r)
0323 {
0324     return container_of(r, struct rdt_hw_domain, d_resctrl);
0325 }
0326
0327 /**
0328  * struct msr_param - set a range of MSRs from a domain
0329  * @res:       The resource to use
0330  * @low:       Beginning index from base MSR
0331  * @high:      End index
0332  */
0333 struct msr_param {
0334     struct rdt_resource *res;
0335     u32         low;
0336     u32         high;
0337 };
0338
0339 static inline bool is_llc_occupancy_enabled(void)
0340 {
0341     return (rdt_mon_features & (1 << QOS_L3_OCCUP_EVENT_ID));
0342 }
0343
0344 static inline bool is_mbm_total_enabled(void)
0345 {
0346     return (rdt_mon_features & (1 << QOS_L3_MBM_TOTAL_EVENT_ID));
0347 }
0348
0349 static inline bool is_mbm_local_enabled(void)
0350 {
0351     return (rdt_mon_features & (1 << QOS_L3_MBM_LOCAL_EVENT_ID));
0352 }
0353
0354 static inline bool is_mbm_enabled(void)
0355 {
0356     return (is_mbm_total_enabled() || is_mbm_local_enabled());
0357 }
0358
0359 static inline bool is_mbm_event(int e)
0360 {
0361     return (e >= QOS_L3_MBM_TOTAL_EVENT_ID &&
0362         e <= QOS_L3_MBM_LOCAL_EVENT_ID);
0363 }
0364
0365 struct rdt_parse_data {
0366     struct rdtgroup     *rdtgrp;
0367     char            *buf;
0368 };
0369
0370 /**
0371  * struct rdt_hw_resource - arch private attributes of a resctrl resource
0372  * @r_resctrl:      Attributes of the resource used directly by resctrl.
0373  * @num_closid:     Maximum number of closid this hardware can support,
0374  *          regardless of CDP. This is exposed via
0375  *          resctrl_arch_get_num_closid() to avoid confusion
0376  *          with struct resctrl_schema's property of the same name,
0377  *          which has been corrected for features like CDP.
0378  * @msr_base:       Base MSR address for CBMs
0379  * @msr_update:     Function pointer to update QOS MSRs
0380  * @mon_scale:      cqm counter * mon_scale = occupancy in bytes
0381  * @mbm_width:      Monitor width, to detect and correct for overflow.
0382  * @cdp_enabled:    CDP state of this resource
0383  *
0384  * Members of this structure are either private to the architecture
0385  * e.g. mbm_width, or accessed via helpers that provide abstraction. e.g.
0386  * msr_update and msr_base.
0387  */
0388 struct rdt_hw_resource {
0389     struct rdt_resource r_resctrl;
0390     u32         num_closid;
0391     unsigned int        msr_base;
0392     void (*msr_update)  (struct rdt_domain *d, struct msr_param *m,
0393                  struct rdt_resource *r);
0394     unsigned int        mon_scale;
0395     unsigned int        mbm_width;
0396     bool            cdp_enabled;
0397 };
0398
0399 static inline struct rdt_hw_resource *resctrl_to_arch_res(struct rdt_resource *r)
0400 {
0401     return container_of(r, struct rdt_hw_resource, r_resctrl);
0402 }
0403
0404 int parse_cbm(struct rdt_parse_data *data, struct resctrl_schema *s,
0405           struct rdt_domain *d);
0406 int parse_bw(struct rdt_parse_data *data, struct resctrl_schema *s,
0407          struct rdt_domain *d);
0408
0409 extern struct mutex rdtgroup_mutex;
0410
0411 extern struct rdt_hw_resource rdt_resources_all[];
0412 extern struct rdtgroup rdtgroup_default;
0413 DECLARE_STATIC_KEY_FALSE(rdt_alloc_enable_key);
0414
0415 extern struct dentry *debugfs_resctrl;
0416
0417 enum resctrl_res_level {
0418     RDT_RESOURCE_L3,
0419     RDT_RESOURCE_L2,
0420     RDT_RESOURCE_MBA,
0421
0422     /* Must be the last */
0423     RDT_NUM_RESOURCES,
0424 };
0425
0426 static inline struct rdt_resource *resctrl_inc(struct rdt_resource *res)
0427 {
0428     struct rdt_hw_resource *hw_res = resctrl_to_arch_res(res);
0429
0430     hw_res++;
0431     return &hw_res->r_resctrl;
0432 }
0433
0434 static inline bool resctrl_arch_get_cdp_enabled(enum resctrl_res_level l)
0435 {
0436     return rdt_resources_all[l].cdp_enabled;
0437 }
0438
0439 int resctrl_arch_set_cdp_enabled(enum resctrl_res_level l, bool enable);
0440
0441 /*
0442  * To return the common struct rdt_resource, which is contained in struct
0443  * rdt_hw_resource, walk the resctrl member of struct rdt_hw_resource.
0444  */
0445 #define for_each_rdt_resource(r)                          \
0446     for (r = &rdt_resources_all[0].r_resctrl;                 \
0447          r <= &rdt_resources_all[RDT_NUM_RESOURCES - 1].r_resctrl;        \
0448          r = resctrl_inc(r))
0449
0450 #define for_each_capable_rdt_resource(r)                      \
0451     for_each_rdt_resource(r)                          \
0452         if (r->alloc_capable || r->mon_capable)
0453
0454 #define for_each_alloc_capable_rdt_resource(r)                    \
0455     for_each_rdt_resource(r)                          \
0456         if (r->alloc_capable)
0457
0458 #define for_each_mon_capable_rdt_resource(r)                      \
0459     for_each_rdt_resource(r)                          \
0460         if (r->mon_capable)
0461
0462 #define for_each_alloc_enabled_rdt_resource(r)                    \
0463     for_each_rdt_resource(r)                          \
0464         if (r->alloc_enabled)
0465
0466 #define for_each_mon_enabled_rdt_resource(r)                      \
0467     for_each_rdt_resource(r)                          \
0468         if (r->mon_enabled)
0469
0470 /* CPUID.(EAX=10H, ECX=ResID=1).EAX */
0471 union cpuid_0x10_1_eax {
0472     struct {
0473         unsigned int cbm_len:5;
0474     } split;
0475     unsigned int full;
0476 };
0477
0478 /* CPUID.(EAX=10H, ECX=ResID=3).EAX */
0479 union cpuid_0x10_3_eax {
0480     struct {
0481         unsigned int max_delay:12;
0482     } split;
0483     unsigned int full;
0484 };
0485
0486 /* CPUID.(EAX=10H, ECX=ResID).EDX */
0487 union cpuid_0x10_x_edx {
0488     struct {
0489         unsigned int cos_max:16;
0490     } split;
0491     unsigned int full;
0492 };
0493
0494 void rdt_last_cmd_clear(void);
0495 void rdt_last_cmd_puts(const char *s);
0496 __printf(1, 2)
0497 void rdt_last_cmd_printf(const char *fmt, ...);
0498
0499 void rdt_ctrl_update(void *arg);
0500 struct rdtgroup *rdtgroup_kn_lock_live(struct kernfs_node *kn);
0501 void rdtgroup_kn_unlock(struct kernfs_node *kn);
0502 int rdtgroup_kn_mode_restrict(struct rdtgroup *r, const char *name);
0503 int rdtgroup_kn_mode_restore(struct rdtgroup *r, const char *name,
0504                  umode_t mask);
0505 struct rdt_domain *rdt_find_domain(struct rdt_resource *r, int id,
0506                    struct list_head **pos);
0507 ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of,
0508                 char *buf, size_t nbytes, loff_t off);
0509 int rdtgroup_schemata_show(struct kernfs_open_file *of,
0510                struct seq_file *s, void *v);
0511 bool rdtgroup_cbm_overlaps(struct resctrl_schema *s, struct rdt_domain *d,
0512                unsigned long cbm, int closid, bool exclusive);
0513 unsigned int rdtgroup_cbm_to_size(struct rdt_resource *r, struct rdt_domain *d,
0514                   unsigned long cbm);
0515 enum rdtgrp_mode rdtgroup_mode_by_closid(int closid);
0516 int rdtgroup_tasks_assigned(struct rdtgroup *r);
0517 int rdtgroup_locksetup_enter(struct rdtgroup *rdtgrp);
0518 int rdtgroup_locksetup_exit(struct rdtgroup *rdtgrp);
0519 bool rdtgroup_cbm_overlaps_pseudo_locked(struct rdt_domain *d, unsigned long cbm);
0520 bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_domain *d);
0521 int rdt_pseudo_lock_init(void);
0522 void rdt_pseudo_lock_release(void);
0523 int rdtgroup_pseudo_lock_create(struct rdtgroup *rdtgrp);
0524 void rdtgroup_pseudo_lock_remove(struct rdtgroup *rdtgrp);
0525 struct rdt_domain *get_domain_from_cpu(int cpu, struct rdt_resource *r);
0526 int closids_supported(void);
0527 void closid_free(int closid);
0528 int alloc_rmid(void);
0529 void free_rmid(u32 rmid);
0530 int rdt_get_mon_l3_config(struct rdt_resource *r);
0531 void mon_event_count(void *info);
0532 int rdtgroup_mondata_show(struct seq_file *m, void *arg);
0533 void rmdir_mondata_subdir_allrdtgrp(struct rdt_resource *r,
0534                     unsigned int dom_id);
0535 void mkdir_mondata_subdir_allrdtgrp(struct rdt_resource *r,
0536                     struct rdt_domain *d);
0537 void mon_event_read(struct rmid_read *rr, struct rdt_resource *r,
0538             struct rdt_domain *d, struct rdtgroup *rdtgrp,
0539             int evtid, int first);
0540 void mbm_setup_overflow_handler(struct rdt_domain *dom,
0541                 unsigned long delay_ms);
0542 void mbm_handle_overflow(struct work_struct *work);
0543 void __init intel_rdt_mbm_apply_quirk(void);
0544 bool is_mba_sc(struct rdt_resource *r);
0545 void setup_default_ctrlval(struct rdt_resource *r, u32 *dc, u32 *dm);
0546 u32 delay_bw_map(unsigned long bw, struct rdt_resource *r);
0547 void cqm_setup_limbo_handler(struct rdt_domain *dom, unsigned long delay_ms);
0548 void cqm_handle_limbo(struct work_struct *work);
0549 bool has_busy_rmid(struct rdt_resource *r, struct rdt_domain *d);
0550 void __check_limbo(struct rdt_domain *d, bool force_free);
0551 void rdt_domain_reconfigure_cdp(struct rdt_resource *r);
0552 void __init thread_throttle_mode_init(void);
0553
0554 #endif /* _ASM_X86_RESCTRL_INTERNAL_H */