Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 /* Nehalem-EX/Westmere-EX uncore support */
0003 #include "uncore.h"
0004 
0005 /* NHM-EX event control */
0006 #define NHMEX_PMON_CTL_EV_SEL_MASK  0x000000ff
0007 #define NHMEX_PMON_CTL_UMASK_MASK   0x0000ff00
0008 #define NHMEX_PMON_CTL_EN_BIT0      (1 << 0)
0009 #define NHMEX_PMON_CTL_EDGE_DET     (1 << 18)
0010 #define NHMEX_PMON_CTL_PMI_EN       (1 << 20)
0011 #define NHMEX_PMON_CTL_EN_BIT22     (1 << 22)
0012 #define NHMEX_PMON_CTL_INVERT       (1 << 23)
0013 #define NHMEX_PMON_CTL_TRESH_MASK   0xff000000
0014 #define NHMEX_PMON_RAW_EVENT_MASK   (NHMEX_PMON_CTL_EV_SEL_MASK | \
0015                      NHMEX_PMON_CTL_UMASK_MASK | \
0016                      NHMEX_PMON_CTL_EDGE_DET | \
0017                      NHMEX_PMON_CTL_INVERT | \
0018                      NHMEX_PMON_CTL_TRESH_MASK)
0019 
0020 /* NHM-EX Ubox */
0021 #define NHMEX_U_MSR_PMON_GLOBAL_CTL     0xc00
0022 #define NHMEX_U_MSR_PMON_CTR            0xc11
0023 #define NHMEX_U_MSR_PMON_EV_SEL         0xc10
0024 
0025 #define NHMEX_U_PMON_GLOBAL_EN          (1 << 0)
0026 #define NHMEX_U_PMON_GLOBAL_PMI_CORE_SEL    0x0000001e
0027 #define NHMEX_U_PMON_GLOBAL_EN_ALL      (1 << 28)
0028 #define NHMEX_U_PMON_GLOBAL_RST_ALL     (1 << 29)
0029 #define NHMEX_U_PMON_GLOBAL_FRZ_ALL     (1 << 31)
0030 
0031 #define NHMEX_U_PMON_RAW_EVENT_MASK     \
0032         (NHMEX_PMON_CTL_EV_SEL_MASK |   \
0033          NHMEX_PMON_CTL_EDGE_DET)
0034 
0035 /* NHM-EX Cbox */
0036 #define NHMEX_C0_MSR_PMON_GLOBAL_CTL        0xd00
0037 #define NHMEX_C0_MSR_PMON_CTR0          0xd11
0038 #define NHMEX_C0_MSR_PMON_EV_SEL0       0xd10
0039 #define NHMEX_C_MSR_OFFSET          0x20
0040 
0041 /* NHM-EX Bbox */
0042 #define NHMEX_B0_MSR_PMON_GLOBAL_CTL        0xc20
0043 #define NHMEX_B0_MSR_PMON_CTR0          0xc31
0044 #define NHMEX_B0_MSR_PMON_CTL0          0xc30
0045 #define NHMEX_B_MSR_OFFSET          0x40
0046 #define NHMEX_B0_MSR_MATCH          0xe45
0047 #define NHMEX_B0_MSR_MASK           0xe46
0048 #define NHMEX_B1_MSR_MATCH          0xe4d
0049 #define NHMEX_B1_MSR_MASK           0xe4e
0050 
0051 #define NHMEX_B_PMON_CTL_EN         (1 << 0)
0052 #define NHMEX_B_PMON_CTL_EV_SEL_SHIFT       1
0053 #define NHMEX_B_PMON_CTL_EV_SEL_MASK        \
0054         (0x1f << NHMEX_B_PMON_CTL_EV_SEL_SHIFT)
0055 #define NHMEX_B_PMON_CTR_SHIFT      6
0056 #define NHMEX_B_PMON_CTR_MASK       \
0057         (0x3 << NHMEX_B_PMON_CTR_SHIFT)
0058 #define NHMEX_B_PMON_RAW_EVENT_MASK     \
0059         (NHMEX_B_PMON_CTL_EV_SEL_MASK | \
0060          NHMEX_B_PMON_CTR_MASK)
0061 
0062 /* NHM-EX Sbox */
0063 #define NHMEX_S0_MSR_PMON_GLOBAL_CTL        0xc40
0064 #define NHMEX_S0_MSR_PMON_CTR0          0xc51
0065 #define NHMEX_S0_MSR_PMON_CTL0          0xc50
0066 #define NHMEX_S_MSR_OFFSET          0x80
0067 #define NHMEX_S0_MSR_MM_CFG         0xe48
0068 #define NHMEX_S0_MSR_MATCH          0xe49
0069 #define NHMEX_S0_MSR_MASK           0xe4a
0070 #define NHMEX_S1_MSR_MM_CFG         0xe58
0071 #define NHMEX_S1_MSR_MATCH          0xe59
0072 #define NHMEX_S1_MSR_MASK           0xe5a
0073 
0074 #define NHMEX_S_PMON_MM_CFG_EN          (0x1ULL << 63)
0075 #define NHMEX_S_EVENT_TO_R_PROG_EV      0
0076 
0077 /* NHM-EX Mbox */
0078 #define NHMEX_M0_MSR_GLOBAL_CTL         0xca0
0079 #define NHMEX_M0_MSR_PMU_DSP            0xca5
0080 #define NHMEX_M0_MSR_PMU_ISS            0xca6
0081 #define NHMEX_M0_MSR_PMU_MAP            0xca7
0082 #define NHMEX_M0_MSR_PMU_MSC_THR        0xca8
0083 #define NHMEX_M0_MSR_PMU_PGT            0xca9
0084 #define NHMEX_M0_MSR_PMU_PLD            0xcaa
0085 #define NHMEX_M0_MSR_PMU_ZDP_CTL_FVC        0xcab
0086 #define NHMEX_M0_MSR_PMU_CTL0           0xcb0
0087 #define NHMEX_M0_MSR_PMU_CNT0           0xcb1
0088 #define NHMEX_M_MSR_OFFSET          0x40
0089 #define NHMEX_M0_MSR_PMU_MM_CFG         0xe54
0090 #define NHMEX_M1_MSR_PMU_MM_CFG         0xe5c
0091 
0092 #define NHMEX_M_PMON_MM_CFG_EN          (1ULL << 63)
0093 #define NHMEX_M_PMON_ADDR_MATCH_MASK        0x3ffffffffULL
0094 #define NHMEX_M_PMON_ADDR_MASK_MASK     0x7ffffffULL
0095 #define NHMEX_M_PMON_ADDR_MASK_SHIFT        34
0096 
0097 #define NHMEX_M_PMON_CTL_EN         (1 << 0)
0098 #define NHMEX_M_PMON_CTL_PMI_EN         (1 << 1)
0099 #define NHMEX_M_PMON_CTL_COUNT_MODE_SHIFT   2
0100 #define NHMEX_M_PMON_CTL_COUNT_MODE_MASK    \
0101     (0x3 << NHMEX_M_PMON_CTL_COUNT_MODE_SHIFT)
0102 #define NHMEX_M_PMON_CTL_STORAGE_MODE_SHIFT 4
0103 #define NHMEX_M_PMON_CTL_STORAGE_MODE_MASK  \
0104     (0x3 << NHMEX_M_PMON_CTL_STORAGE_MODE_SHIFT)
0105 #define NHMEX_M_PMON_CTL_WRAP_MODE      (1 << 6)
0106 #define NHMEX_M_PMON_CTL_FLAG_MODE      (1 << 7)
0107 #define NHMEX_M_PMON_CTL_INC_SEL_SHIFT      9
0108 #define NHMEX_M_PMON_CTL_INC_SEL_MASK       \
0109     (0x1f << NHMEX_M_PMON_CTL_INC_SEL_SHIFT)
0110 #define NHMEX_M_PMON_CTL_SET_FLAG_SEL_SHIFT 19
0111 #define NHMEX_M_PMON_CTL_SET_FLAG_SEL_MASK  \
0112     (0x7 << NHMEX_M_PMON_CTL_SET_FLAG_SEL_SHIFT)
0113 #define NHMEX_M_PMON_RAW_EVENT_MASK         \
0114         (NHMEX_M_PMON_CTL_COUNT_MODE_MASK | \
0115          NHMEX_M_PMON_CTL_STORAGE_MODE_MASK |   \
0116          NHMEX_M_PMON_CTL_WRAP_MODE |       \
0117          NHMEX_M_PMON_CTL_FLAG_MODE |       \
0118          NHMEX_M_PMON_CTL_INC_SEL_MASK |    \
0119          NHMEX_M_PMON_CTL_SET_FLAG_SEL_MASK)
0120 
0121 #define NHMEX_M_PMON_ZDP_CTL_FVC_MASK       (((1 << 11) - 1) | (1 << 23))
0122 #define NHMEX_M_PMON_ZDP_CTL_FVC_EVENT_MASK(n)  (0x7ULL << (11 + 3 * (n)))
0123 
0124 #define WSMEX_M_PMON_ZDP_CTL_FVC_MASK       (((1 << 12) - 1) | (1 << 24))
0125 #define WSMEX_M_PMON_ZDP_CTL_FVC_EVENT_MASK(n)  (0x7ULL << (12 + 3 * (n)))
0126 
0127 /*
0128  * use the 9~13 bits to select event If the 7th bit is not set,
0129  * otherwise use the 19~21 bits to select event.
0130  */
0131 #define MBOX_INC_SEL(x) ((x) << NHMEX_M_PMON_CTL_INC_SEL_SHIFT)
0132 #define MBOX_SET_FLAG_SEL(x) (((x) << NHMEX_M_PMON_CTL_SET_FLAG_SEL_SHIFT) | \
0133                 NHMEX_M_PMON_CTL_FLAG_MODE)
0134 #define MBOX_INC_SEL_MASK (NHMEX_M_PMON_CTL_INC_SEL_MASK | \
0135                NHMEX_M_PMON_CTL_FLAG_MODE)
0136 #define MBOX_SET_FLAG_SEL_MASK (NHMEX_M_PMON_CTL_SET_FLAG_SEL_MASK | \
0137                 NHMEX_M_PMON_CTL_FLAG_MODE)
0138 #define MBOX_INC_SEL_EXTAR_REG(c, r) \
0139         EVENT_EXTRA_REG(MBOX_INC_SEL(c), NHMEX_M0_MSR_PMU_##r, \
0140                 MBOX_INC_SEL_MASK, (u64)-1, NHMEX_M_##r)
0141 #define MBOX_SET_FLAG_SEL_EXTRA_REG(c, r) \
0142         EVENT_EXTRA_REG(MBOX_SET_FLAG_SEL(c), NHMEX_M0_MSR_PMU_##r, \
0143                 MBOX_SET_FLAG_SEL_MASK, \
0144                 (u64)-1, NHMEX_M_##r)
0145 
0146 /* NHM-EX Rbox */
0147 #define NHMEX_R_MSR_GLOBAL_CTL          0xe00
0148 #define NHMEX_R_MSR_PMON_CTL0           0xe10
0149 #define NHMEX_R_MSR_PMON_CNT0           0xe11
0150 #define NHMEX_R_MSR_OFFSET          0x20
0151 
0152 #define NHMEX_R_MSR_PORTN_QLX_CFG(n)        \
0153         ((n) < 4 ? (0xe0c + (n)) : (0xe2c + (n) - 4))
0154 #define NHMEX_R_MSR_PORTN_IPERF_CFG0(n)     (0xe04 + (n))
0155 #define NHMEX_R_MSR_PORTN_IPERF_CFG1(n)     (0xe24 + (n))
0156 #define NHMEX_R_MSR_PORTN_XBR_OFFSET(n)     \
0157         (((n) < 4 ? 0 : 0x10) + (n) * 4)
0158 #define NHMEX_R_MSR_PORTN_XBR_SET1_MM_CFG(n)    \
0159         (0xe60 + NHMEX_R_MSR_PORTN_XBR_OFFSET(n))
0160 #define NHMEX_R_MSR_PORTN_XBR_SET1_MATCH(n) \
0161         (NHMEX_R_MSR_PORTN_XBR_SET1_MM_CFG(n) + 1)
0162 #define NHMEX_R_MSR_PORTN_XBR_SET1_MASK(n)  \
0163         (NHMEX_R_MSR_PORTN_XBR_SET1_MM_CFG(n) + 2)
0164 #define NHMEX_R_MSR_PORTN_XBR_SET2_MM_CFG(n)    \
0165         (0xe70 + NHMEX_R_MSR_PORTN_XBR_OFFSET(n))
0166 #define NHMEX_R_MSR_PORTN_XBR_SET2_MATCH(n) \
0167         (NHMEX_R_MSR_PORTN_XBR_SET2_MM_CFG(n) + 1)
0168 #define NHMEX_R_MSR_PORTN_XBR_SET2_MASK(n)  \
0169         (NHMEX_R_MSR_PORTN_XBR_SET2_MM_CFG(n) + 2)
0170 
0171 #define NHMEX_R_PMON_CTL_EN         (1 << 0)
0172 #define NHMEX_R_PMON_CTL_EV_SEL_SHIFT       1
0173 #define NHMEX_R_PMON_CTL_EV_SEL_MASK        \
0174         (0x1f << NHMEX_R_PMON_CTL_EV_SEL_SHIFT)
0175 #define NHMEX_R_PMON_CTL_PMI_EN         (1 << 6)
0176 #define NHMEX_R_PMON_RAW_EVENT_MASK     NHMEX_R_PMON_CTL_EV_SEL_MASK
0177 
0178 /* NHM-EX Wbox */
0179 #define NHMEX_W_MSR_GLOBAL_CTL          0xc80
0180 #define NHMEX_W_MSR_PMON_CNT0           0xc90
0181 #define NHMEX_W_MSR_PMON_EVT_SEL0       0xc91
0182 #define NHMEX_W_MSR_PMON_FIXED_CTR      0x394
0183 #define NHMEX_W_MSR_PMON_FIXED_CTL      0x395
0184 
0185 #define NHMEX_W_PMON_GLOBAL_FIXED_EN        (1ULL << 31)
0186 
0187 #define __BITS_VALUE(x, i, n)  ((typeof(x))(((x) >> ((i) * (n))) & \
0188                 ((1ULL << (n)) - 1)))
0189 
0190 DEFINE_UNCORE_FORMAT_ATTR(event, event, "config:0-7");
0191 DEFINE_UNCORE_FORMAT_ATTR(event5, event, "config:1-5");
0192 DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15");
0193 DEFINE_UNCORE_FORMAT_ATTR(edge, edge, "config:18");
0194 DEFINE_UNCORE_FORMAT_ATTR(inv, inv, "config:23");
0195 DEFINE_UNCORE_FORMAT_ATTR(thresh8, thresh, "config:24-31");
0196 DEFINE_UNCORE_FORMAT_ATTR(counter, counter, "config:6-7");
0197 DEFINE_UNCORE_FORMAT_ATTR(match, match, "config1:0-63");
0198 DEFINE_UNCORE_FORMAT_ATTR(mask, mask, "config2:0-63");
0199 
0200 static void nhmex_uncore_msr_init_box(struct intel_uncore_box *box)
0201 {
0202     wrmsrl(NHMEX_U_MSR_PMON_GLOBAL_CTL, NHMEX_U_PMON_GLOBAL_EN_ALL);
0203 }
0204 
0205 static void nhmex_uncore_msr_exit_box(struct intel_uncore_box *box)
0206 {
0207     wrmsrl(NHMEX_U_MSR_PMON_GLOBAL_CTL, 0);
0208 }
0209 
0210 static void nhmex_uncore_msr_disable_box(struct intel_uncore_box *box)
0211 {
0212     unsigned msr = uncore_msr_box_ctl(box);
0213     u64 config;
0214 
0215     if (msr) {
0216         rdmsrl(msr, config);
0217         config &= ~((1ULL << uncore_num_counters(box)) - 1);
0218         /* WBox has a fixed counter */
0219         if (uncore_msr_fixed_ctl(box))
0220             config &= ~NHMEX_W_PMON_GLOBAL_FIXED_EN;
0221         wrmsrl(msr, config);
0222     }
0223 }
0224 
0225 static void nhmex_uncore_msr_enable_box(struct intel_uncore_box *box)
0226 {
0227     unsigned msr = uncore_msr_box_ctl(box);
0228     u64 config;
0229 
0230     if (msr) {
0231         rdmsrl(msr, config);
0232         config |= (1ULL << uncore_num_counters(box)) - 1;
0233         /* WBox has a fixed counter */
0234         if (uncore_msr_fixed_ctl(box))
0235             config |= NHMEX_W_PMON_GLOBAL_FIXED_EN;
0236         wrmsrl(msr, config);
0237     }
0238 }
0239 
0240 static void nhmex_uncore_msr_disable_event(struct intel_uncore_box *box, struct perf_event *event)
0241 {
0242     wrmsrl(event->hw.config_base, 0);
0243 }
0244 
0245 static void nhmex_uncore_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
0246 {
0247     struct hw_perf_event *hwc = &event->hw;
0248 
0249     if (hwc->idx == UNCORE_PMC_IDX_FIXED)
0250         wrmsrl(hwc->config_base, NHMEX_PMON_CTL_EN_BIT0);
0251     else if (box->pmu->type->event_mask & NHMEX_PMON_CTL_EN_BIT0)
0252         wrmsrl(hwc->config_base, hwc->config | NHMEX_PMON_CTL_EN_BIT22);
0253     else
0254         wrmsrl(hwc->config_base, hwc->config | NHMEX_PMON_CTL_EN_BIT0);
0255 }
0256 
0257 #define NHMEX_UNCORE_OPS_COMMON_INIT()              \
0258     .init_box   = nhmex_uncore_msr_init_box,        \
0259     .exit_box   = nhmex_uncore_msr_exit_box,        \
0260     .disable_box    = nhmex_uncore_msr_disable_box,     \
0261     .enable_box = nhmex_uncore_msr_enable_box,      \
0262     .disable_event  = nhmex_uncore_msr_disable_event,   \
0263     .read_counter   = uncore_msr_read_counter
0264 
0265 static struct intel_uncore_ops nhmex_uncore_ops = {
0266     NHMEX_UNCORE_OPS_COMMON_INIT(),
0267     .enable_event   = nhmex_uncore_msr_enable_event,
0268 };
0269 
0270 static struct attribute *nhmex_uncore_ubox_formats_attr[] = {
0271     &format_attr_event.attr,
0272     &format_attr_edge.attr,
0273     NULL,
0274 };
0275 
0276 static const struct attribute_group nhmex_uncore_ubox_format_group = {
0277     .name       = "format",
0278     .attrs      = nhmex_uncore_ubox_formats_attr,
0279 };
0280 
0281 static struct intel_uncore_type nhmex_uncore_ubox = {
0282     .name       = "ubox",
0283     .num_counters   = 1,
0284     .num_boxes  = 1,
0285     .perf_ctr_bits  = 48,
0286     .event_ctl  = NHMEX_U_MSR_PMON_EV_SEL,
0287     .perf_ctr   = NHMEX_U_MSR_PMON_CTR,
0288     .event_mask = NHMEX_U_PMON_RAW_EVENT_MASK,
0289     .box_ctl    = NHMEX_U_MSR_PMON_GLOBAL_CTL,
0290     .ops        = &nhmex_uncore_ops,
0291     .format_group   = &nhmex_uncore_ubox_format_group
0292 };
0293 
0294 static struct attribute *nhmex_uncore_cbox_formats_attr[] = {
0295     &format_attr_event.attr,
0296     &format_attr_umask.attr,
0297     &format_attr_edge.attr,
0298     &format_attr_inv.attr,
0299     &format_attr_thresh8.attr,
0300     NULL,
0301 };
0302 
0303 static const struct attribute_group nhmex_uncore_cbox_format_group = {
0304     .name = "format",
0305     .attrs = nhmex_uncore_cbox_formats_attr,
0306 };
0307 
0308 /* msr offset for each instance of cbox */
0309 static unsigned nhmex_cbox_msr_offsets[] = {
0310     0x0, 0x80, 0x40, 0xc0, 0x20, 0xa0, 0x60, 0xe0, 0x240, 0x2c0,
0311 };
0312 
0313 static struct intel_uncore_type nhmex_uncore_cbox = {
0314     .name           = "cbox",
0315     .num_counters       = 6,
0316     .num_boxes      = 10,
0317     .perf_ctr_bits      = 48,
0318     .event_ctl      = NHMEX_C0_MSR_PMON_EV_SEL0,
0319     .perf_ctr       = NHMEX_C0_MSR_PMON_CTR0,
0320     .event_mask     = NHMEX_PMON_RAW_EVENT_MASK,
0321     .box_ctl        = NHMEX_C0_MSR_PMON_GLOBAL_CTL,
0322     .msr_offsets        = nhmex_cbox_msr_offsets,
0323     .pair_ctr_ctl       = 1,
0324     .ops            = &nhmex_uncore_ops,
0325     .format_group       = &nhmex_uncore_cbox_format_group
0326 };
0327 
0328 static struct uncore_event_desc nhmex_uncore_wbox_events[] = {
0329     INTEL_UNCORE_EVENT_DESC(clockticks, "event=0xff,umask=0"),
0330     { /* end: all zeroes */ },
0331 };
0332 
0333 static struct intel_uncore_type nhmex_uncore_wbox = {
0334     .name           = "wbox",
0335     .num_counters       = 4,
0336     .num_boxes      = 1,
0337     .perf_ctr_bits      = 48,
0338     .event_ctl      = NHMEX_W_MSR_PMON_CNT0,
0339     .perf_ctr       = NHMEX_W_MSR_PMON_EVT_SEL0,
0340     .fixed_ctr      = NHMEX_W_MSR_PMON_FIXED_CTR,
0341     .fixed_ctl      = NHMEX_W_MSR_PMON_FIXED_CTL,
0342     .event_mask     = NHMEX_PMON_RAW_EVENT_MASK,
0343     .box_ctl        = NHMEX_W_MSR_GLOBAL_CTL,
0344     .pair_ctr_ctl       = 1,
0345     .event_descs        = nhmex_uncore_wbox_events,
0346     .ops            = &nhmex_uncore_ops,
0347     .format_group       = &nhmex_uncore_cbox_format_group
0348 };
0349 
0350 static int nhmex_bbox_hw_config(struct intel_uncore_box *box, struct perf_event *event)
0351 {
0352     struct hw_perf_event *hwc = &event->hw;
0353     struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
0354     struct hw_perf_event_extra *reg2 = &hwc->branch_reg;
0355     int ctr, ev_sel;
0356 
0357     ctr = (hwc->config & NHMEX_B_PMON_CTR_MASK) >>
0358         NHMEX_B_PMON_CTR_SHIFT;
0359     ev_sel = (hwc->config & NHMEX_B_PMON_CTL_EV_SEL_MASK) >>
0360           NHMEX_B_PMON_CTL_EV_SEL_SHIFT;
0361 
0362     /* events that do not use the match/mask registers */
0363     if ((ctr == 0 && ev_sel > 0x3) || (ctr == 1 && ev_sel > 0x6) ||
0364         (ctr == 2 && ev_sel != 0x4) || ctr == 3)
0365         return 0;
0366 
0367     if (box->pmu->pmu_idx == 0)
0368         reg1->reg = NHMEX_B0_MSR_MATCH;
0369     else
0370         reg1->reg = NHMEX_B1_MSR_MATCH;
0371     reg1->idx = 0;
0372     reg1->config = event->attr.config1;
0373     reg2->config = event->attr.config2;
0374     return 0;
0375 }
0376 
0377 static void nhmex_bbox_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
0378 {
0379     struct hw_perf_event *hwc = &event->hw;
0380     struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
0381     struct hw_perf_event_extra *reg2 = &hwc->branch_reg;
0382 
0383     if (reg1->idx != EXTRA_REG_NONE) {
0384         wrmsrl(reg1->reg, reg1->config);
0385         wrmsrl(reg1->reg + 1, reg2->config);
0386     }
0387     wrmsrl(hwc->config_base, NHMEX_PMON_CTL_EN_BIT0 |
0388         (hwc->config & NHMEX_B_PMON_CTL_EV_SEL_MASK));
0389 }
0390 
0391 /*
0392  * The Bbox has 4 counters, but each counter monitors different events.
0393  * Use bits 6-7 in the event config to select counter.
0394  */
0395 static struct event_constraint nhmex_uncore_bbox_constraints[] = {
0396     EVENT_CONSTRAINT(0 , 1, 0xc0),
0397     EVENT_CONSTRAINT(0x40, 2, 0xc0),
0398     EVENT_CONSTRAINT(0x80, 4, 0xc0),
0399     EVENT_CONSTRAINT(0xc0, 8, 0xc0),
0400     EVENT_CONSTRAINT_END,
0401 };
0402 
0403 static struct attribute *nhmex_uncore_bbox_formats_attr[] = {
0404     &format_attr_event5.attr,
0405     &format_attr_counter.attr,
0406     &format_attr_match.attr,
0407     &format_attr_mask.attr,
0408     NULL,
0409 };
0410 
0411 static const struct attribute_group nhmex_uncore_bbox_format_group = {
0412     .name = "format",
0413     .attrs = nhmex_uncore_bbox_formats_attr,
0414 };
0415 
0416 static struct intel_uncore_ops nhmex_uncore_bbox_ops = {
0417     NHMEX_UNCORE_OPS_COMMON_INIT(),
0418     .enable_event       = nhmex_bbox_msr_enable_event,
0419     .hw_config      = nhmex_bbox_hw_config,
0420     .get_constraint     = uncore_get_constraint,
0421     .put_constraint     = uncore_put_constraint,
0422 };
0423 
0424 static struct intel_uncore_type nhmex_uncore_bbox = {
0425     .name           = "bbox",
0426     .num_counters       = 4,
0427     .num_boxes      = 2,
0428     .perf_ctr_bits      = 48,
0429     .event_ctl      = NHMEX_B0_MSR_PMON_CTL0,
0430     .perf_ctr       = NHMEX_B0_MSR_PMON_CTR0,
0431     .event_mask     = NHMEX_B_PMON_RAW_EVENT_MASK,
0432     .box_ctl        = NHMEX_B0_MSR_PMON_GLOBAL_CTL,
0433     .msr_offset     = NHMEX_B_MSR_OFFSET,
0434     .pair_ctr_ctl       = 1,
0435     .num_shared_regs    = 1,
0436     .constraints        = nhmex_uncore_bbox_constraints,
0437     .ops            = &nhmex_uncore_bbox_ops,
0438     .format_group       = &nhmex_uncore_bbox_format_group
0439 };
0440 
0441 static int nhmex_sbox_hw_config(struct intel_uncore_box *box, struct perf_event *event)
0442 {
0443     struct hw_perf_event *hwc = &event->hw;
0444     struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
0445     struct hw_perf_event_extra *reg2 = &hwc->branch_reg;
0446 
0447     /* only TO_R_PROG_EV event uses the match/mask register */
0448     if ((hwc->config & NHMEX_PMON_CTL_EV_SEL_MASK) !=
0449         NHMEX_S_EVENT_TO_R_PROG_EV)
0450         return 0;
0451 
0452     if (box->pmu->pmu_idx == 0)
0453         reg1->reg = NHMEX_S0_MSR_MM_CFG;
0454     else
0455         reg1->reg = NHMEX_S1_MSR_MM_CFG;
0456     reg1->idx = 0;
0457     reg1->config = event->attr.config1;
0458     reg2->config = event->attr.config2;
0459     return 0;
0460 }
0461 
0462 static void nhmex_sbox_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
0463 {
0464     struct hw_perf_event *hwc = &event->hw;
0465     struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
0466     struct hw_perf_event_extra *reg2 = &hwc->branch_reg;
0467 
0468     if (reg1->idx != EXTRA_REG_NONE) {
0469         wrmsrl(reg1->reg, 0);
0470         wrmsrl(reg1->reg + 1, reg1->config);
0471         wrmsrl(reg1->reg + 2, reg2->config);
0472         wrmsrl(reg1->reg, NHMEX_S_PMON_MM_CFG_EN);
0473     }
0474     wrmsrl(hwc->config_base, hwc->config | NHMEX_PMON_CTL_EN_BIT22);
0475 }
0476 
0477 static struct attribute *nhmex_uncore_sbox_formats_attr[] = {
0478     &format_attr_event.attr,
0479     &format_attr_umask.attr,
0480     &format_attr_edge.attr,
0481     &format_attr_inv.attr,
0482     &format_attr_thresh8.attr,
0483     &format_attr_match.attr,
0484     &format_attr_mask.attr,
0485     NULL,
0486 };
0487 
0488 static const struct attribute_group nhmex_uncore_sbox_format_group = {
0489     .name           = "format",
0490     .attrs          = nhmex_uncore_sbox_formats_attr,
0491 };
0492 
0493 static struct intel_uncore_ops nhmex_uncore_sbox_ops = {
0494     NHMEX_UNCORE_OPS_COMMON_INIT(),
0495     .enable_event       = nhmex_sbox_msr_enable_event,
0496     .hw_config      = nhmex_sbox_hw_config,
0497     .get_constraint     = uncore_get_constraint,
0498     .put_constraint     = uncore_put_constraint,
0499 };
0500 
0501 static struct intel_uncore_type nhmex_uncore_sbox = {
0502     .name           = "sbox",
0503     .num_counters       = 4,
0504     .num_boxes      = 2,
0505     .perf_ctr_bits      = 48,
0506     .event_ctl      = NHMEX_S0_MSR_PMON_CTL0,
0507     .perf_ctr       = NHMEX_S0_MSR_PMON_CTR0,
0508     .event_mask     = NHMEX_PMON_RAW_EVENT_MASK,
0509     .box_ctl        = NHMEX_S0_MSR_PMON_GLOBAL_CTL,
0510     .msr_offset     = NHMEX_S_MSR_OFFSET,
0511     .pair_ctr_ctl       = 1,
0512     .num_shared_regs    = 1,
0513     .ops            = &nhmex_uncore_sbox_ops,
0514     .format_group       = &nhmex_uncore_sbox_format_group
0515 };
0516 
0517 enum {
0518     EXTRA_REG_NHMEX_M_FILTER,
0519     EXTRA_REG_NHMEX_M_DSP,
0520     EXTRA_REG_NHMEX_M_ISS,
0521     EXTRA_REG_NHMEX_M_MAP,
0522     EXTRA_REG_NHMEX_M_MSC_THR,
0523     EXTRA_REG_NHMEX_M_PGT,
0524     EXTRA_REG_NHMEX_M_PLD,
0525     EXTRA_REG_NHMEX_M_ZDP_CTL_FVC,
0526 };
0527 
0528 static struct extra_reg nhmex_uncore_mbox_extra_regs[] = {
0529     MBOX_INC_SEL_EXTAR_REG(0x0, DSP),
0530     MBOX_INC_SEL_EXTAR_REG(0x4, MSC_THR),
0531     MBOX_INC_SEL_EXTAR_REG(0x5, MSC_THR),
0532     MBOX_INC_SEL_EXTAR_REG(0x9, ISS),
0533     /* event 0xa uses two extra registers */
0534     MBOX_INC_SEL_EXTAR_REG(0xa, ISS),
0535     MBOX_INC_SEL_EXTAR_REG(0xa, PLD),
0536     MBOX_INC_SEL_EXTAR_REG(0xb, PLD),
0537     /* events 0xd ~ 0x10 use the same extra register */
0538     MBOX_INC_SEL_EXTAR_REG(0xd, ZDP_CTL_FVC),
0539     MBOX_INC_SEL_EXTAR_REG(0xe, ZDP_CTL_FVC),
0540     MBOX_INC_SEL_EXTAR_REG(0xf, ZDP_CTL_FVC),
0541     MBOX_INC_SEL_EXTAR_REG(0x10, ZDP_CTL_FVC),
0542     MBOX_INC_SEL_EXTAR_REG(0x16, PGT),
0543     MBOX_SET_FLAG_SEL_EXTRA_REG(0x0, DSP),
0544     MBOX_SET_FLAG_SEL_EXTRA_REG(0x1, ISS),
0545     MBOX_SET_FLAG_SEL_EXTRA_REG(0x5, PGT),
0546     MBOX_SET_FLAG_SEL_EXTRA_REG(0x6, MAP),
0547     EVENT_EXTRA_END
0548 };
0549 
0550 /* Nehalem-EX or Westmere-EX ? */
0551 static bool uncore_nhmex;
0552 
0553 static bool nhmex_mbox_get_shared_reg(struct intel_uncore_box *box, int idx, u64 config)
0554 {
0555     struct intel_uncore_extra_reg *er;
0556     unsigned long flags;
0557     bool ret = false;
0558     u64 mask;
0559 
0560     if (idx < EXTRA_REG_NHMEX_M_ZDP_CTL_FVC) {
0561         er = &box->shared_regs[idx];
0562         raw_spin_lock_irqsave(&er->lock, flags);
0563         if (!atomic_read(&er->ref) || er->config == config) {
0564             atomic_inc(&er->ref);
0565             er->config = config;
0566             ret = true;
0567         }
0568         raw_spin_unlock_irqrestore(&er->lock, flags);
0569 
0570         return ret;
0571     }
0572     /*
0573      * The ZDP_CTL_FVC MSR has 4 fields which are used to control
0574      * events 0xd ~ 0x10. Besides these 4 fields, there are additional
0575      * fields which are shared.
0576      */
0577     idx -= EXTRA_REG_NHMEX_M_ZDP_CTL_FVC;
0578     if (WARN_ON_ONCE(idx >= 4))
0579         return false;
0580 
0581     /* mask of the shared fields */
0582     if (uncore_nhmex)
0583         mask = NHMEX_M_PMON_ZDP_CTL_FVC_MASK;
0584     else
0585         mask = WSMEX_M_PMON_ZDP_CTL_FVC_MASK;
0586     er = &box->shared_regs[EXTRA_REG_NHMEX_M_ZDP_CTL_FVC];
0587 
0588     raw_spin_lock_irqsave(&er->lock, flags);
0589     /* add mask of the non-shared field if it's in use */
0590     if (__BITS_VALUE(atomic_read(&er->ref), idx, 8)) {
0591         if (uncore_nhmex)
0592             mask |= NHMEX_M_PMON_ZDP_CTL_FVC_EVENT_MASK(idx);
0593         else
0594             mask |= WSMEX_M_PMON_ZDP_CTL_FVC_EVENT_MASK(idx);
0595     }
0596 
0597     if (!atomic_read(&er->ref) || !((er->config ^ config) & mask)) {
0598         atomic_add(1 << (idx * 8), &er->ref);
0599         if (uncore_nhmex)
0600             mask = NHMEX_M_PMON_ZDP_CTL_FVC_MASK |
0601                 NHMEX_M_PMON_ZDP_CTL_FVC_EVENT_MASK(idx);
0602         else
0603             mask = WSMEX_M_PMON_ZDP_CTL_FVC_MASK |
0604                 WSMEX_M_PMON_ZDP_CTL_FVC_EVENT_MASK(idx);
0605         er->config &= ~mask;
0606         er->config |= (config & mask);
0607         ret = true;
0608     }
0609     raw_spin_unlock_irqrestore(&er->lock, flags);
0610 
0611     return ret;
0612 }
0613 
0614 static void nhmex_mbox_put_shared_reg(struct intel_uncore_box *box, int idx)
0615 {
0616     struct intel_uncore_extra_reg *er;
0617 
0618     if (idx < EXTRA_REG_NHMEX_M_ZDP_CTL_FVC) {
0619         er = &box->shared_regs[idx];
0620         atomic_dec(&er->ref);
0621         return;
0622     }
0623 
0624     idx -= EXTRA_REG_NHMEX_M_ZDP_CTL_FVC;
0625     er = &box->shared_regs[EXTRA_REG_NHMEX_M_ZDP_CTL_FVC];
0626     atomic_sub(1 << (idx * 8), &er->ref);
0627 }
0628 
0629 static u64 nhmex_mbox_alter_er(struct perf_event *event, int new_idx, bool modify)
0630 {
0631     struct hw_perf_event *hwc = &event->hw;
0632     struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
0633     u64 idx, orig_idx = __BITS_VALUE(reg1->idx, 0, 8);
0634     u64 config = reg1->config;
0635 
0636     /* get the non-shared control bits and shift them */
0637     idx = orig_idx - EXTRA_REG_NHMEX_M_ZDP_CTL_FVC;
0638     if (uncore_nhmex)
0639         config &= NHMEX_M_PMON_ZDP_CTL_FVC_EVENT_MASK(idx);
0640     else
0641         config &= WSMEX_M_PMON_ZDP_CTL_FVC_EVENT_MASK(idx);
0642     if (new_idx > orig_idx) {
0643         idx = new_idx - orig_idx;
0644         config <<= 3 * idx;
0645     } else {
0646         idx = orig_idx - new_idx;
0647         config >>= 3 * idx;
0648     }
0649 
0650     /* add the shared control bits back */
0651     if (uncore_nhmex)
0652         config |= NHMEX_M_PMON_ZDP_CTL_FVC_MASK & reg1->config;
0653     else
0654         config |= WSMEX_M_PMON_ZDP_CTL_FVC_MASK & reg1->config;
0655     config |= NHMEX_M_PMON_ZDP_CTL_FVC_MASK & reg1->config;
0656     if (modify) {
0657         /* adjust the main event selector */
0658         if (new_idx > orig_idx)
0659             hwc->config += idx << NHMEX_M_PMON_CTL_INC_SEL_SHIFT;
0660         else
0661             hwc->config -= idx << NHMEX_M_PMON_CTL_INC_SEL_SHIFT;
0662         reg1->config = config;
0663         reg1->idx = ~0xff | new_idx;
0664     }
0665     return config;
0666 }
0667 
0668 static struct event_constraint *
0669 nhmex_mbox_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
0670 {
0671     struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
0672     struct hw_perf_event_extra *reg2 = &event->hw.branch_reg;
0673     int i, idx[2], alloc = 0;
0674     u64 config1 = reg1->config;
0675 
0676     idx[0] = __BITS_VALUE(reg1->idx, 0, 8);
0677     idx[1] = __BITS_VALUE(reg1->idx, 1, 8);
0678 again:
0679     for (i = 0; i < 2; i++) {
0680         if (!uncore_box_is_fake(box) && (reg1->alloc & (0x1 << i)))
0681             idx[i] = 0xff;
0682 
0683         if (idx[i] == 0xff)
0684             continue;
0685 
0686         if (!nhmex_mbox_get_shared_reg(box, idx[i],
0687                 __BITS_VALUE(config1, i, 32)))
0688             goto fail;
0689         alloc |= (0x1 << i);
0690     }
0691 
0692     /* for the match/mask registers */
0693     if (reg2->idx != EXTRA_REG_NONE &&
0694         (uncore_box_is_fake(box) || !reg2->alloc) &&
0695         !nhmex_mbox_get_shared_reg(box, reg2->idx, reg2->config))
0696         goto fail;
0697 
0698     /*
0699      * If it's a fake box -- as per validate_{group,event}() we
0700      * shouldn't touch event state and we can avoid doing so
0701      * since both will only call get_event_constraints() once
0702      * on each event, this avoids the need for reg->alloc.
0703      */
0704     if (!uncore_box_is_fake(box)) {
0705         if (idx[0] != 0xff && idx[0] != __BITS_VALUE(reg1->idx, 0, 8))
0706             nhmex_mbox_alter_er(event, idx[0], true);
0707         reg1->alloc |= alloc;
0708         if (reg2->idx != EXTRA_REG_NONE)
0709             reg2->alloc = 1;
0710     }
0711     return NULL;
0712 fail:
0713     if (idx[0] != 0xff && !(alloc & 0x1) &&
0714         idx[0] >= EXTRA_REG_NHMEX_M_ZDP_CTL_FVC) {
0715         /*
0716          * events 0xd ~ 0x10 are functional identical, but are
0717          * controlled by different fields in the ZDP_CTL_FVC
0718          * register. If we failed to take one field, try the
0719          * rest 3 choices.
0720          */
0721         BUG_ON(__BITS_VALUE(reg1->idx, 1, 8) != 0xff);
0722         idx[0] -= EXTRA_REG_NHMEX_M_ZDP_CTL_FVC;
0723         idx[0] = (idx[0] + 1) % 4;
0724         idx[0] += EXTRA_REG_NHMEX_M_ZDP_CTL_FVC;
0725         if (idx[0] != __BITS_VALUE(reg1->idx, 0, 8)) {
0726             config1 = nhmex_mbox_alter_er(event, idx[0], false);
0727             goto again;
0728         }
0729     }
0730 
0731     if (alloc & 0x1)
0732         nhmex_mbox_put_shared_reg(box, idx[0]);
0733     if (alloc & 0x2)
0734         nhmex_mbox_put_shared_reg(box, idx[1]);
0735     return &uncore_constraint_empty;
0736 }
0737 
0738 static void nhmex_mbox_put_constraint(struct intel_uncore_box *box, struct perf_event *event)
0739 {
0740     struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
0741     struct hw_perf_event_extra *reg2 = &event->hw.branch_reg;
0742 
0743     if (uncore_box_is_fake(box))
0744         return;
0745 
0746     if (reg1->alloc & 0x1)
0747         nhmex_mbox_put_shared_reg(box, __BITS_VALUE(reg1->idx, 0, 8));
0748     if (reg1->alloc & 0x2)
0749         nhmex_mbox_put_shared_reg(box, __BITS_VALUE(reg1->idx, 1, 8));
0750     reg1->alloc = 0;
0751 
0752     if (reg2->alloc) {
0753         nhmex_mbox_put_shared_reg(box, reg2->idx);
0754         reg2->alloc = 0;
0755     }
0756 }
0757 
0758 static int nhmex_mbox_extra_reg_idx(struct extra_reg *er)
0759 {
0760     if (er->idx < EXTRA_REG_NHMEX_M_ZDP_CTL_FVC)
0761         return er->idx;
0762     return er->idx + (er->event >> NHMEX_M_PMON_CTL_INC_SEL_SHIFT) - 0xd;
0763 }
0764 
0765 static int nhmex_mbox_hw_config(struct intel_uncore_box *box, struct perf_event *event)
0766 {
0767     struct intel_uncore_type *type = box->pmu->type;
0768     struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
0769     struct hw_perf_event_extra *reg2 = &event->hw.branch_reg;
0770     struct extra_reg *er;
0771     unsigned msr;
0772     int reg_idx = 0;
0773     /*
0774      * The mbox events may require 2 extra MSRs at the most. But only
0775      * the lower 32 bits in these MSRs are significant, so we can use
0776      * config1 to pass two MSRs' config.
0777      */
0778     for (er = nhmex_uncore_mbox_extra_regs; er->msr; er++) {
0779         if (er->event != (event->hw.config & er->config_mask))
0780             continue;
0781         if (event->attr.config1 & ~er->valid_mask)
0782             return -EINVAL;
0783 
0784         msr = er->msr + type->msr_offset * box->pmu->pmu_idx;
0785         if (WARN_ON_ONCE(msr >= 0xffff || er->idx >= 0xff))
0786             return -EINVAL;
0787 
0788         /* always use the 32~63 bits to pass the PLD config */
0789         if (er->idx == EXTRA_REG_NHMEX_M_PLD)
0790             reg_idx = 1;
0791         else if (WARN_ON_ONCE(reg_idx > 0))
0792             return -EINVAL;
0793 
0794         reg1->idx &= ~(0xff << (reg_idx * 8));
0795         reg1->reg &= ~(0xffff << (reg_idx * 16));
0796         reg1->idx |= nhmex_mbox_extra_reg_idx(er) << (reg_idx * 8);
0797         reg1->reg |= msr << (reg_idx * 16);
0798         reg1->config = event->attr.config1;
0799         reg_idx++;
0800     }
0801     /*
0802      * The mbox only provides ability to perform address matching
0803      * for the PLD events.
0804      */
0805     if (reg_idx == 2) {
0806         reg2->idx = EXTRA_REG_NHMEX_M_FILTER;
0807         if (event->attr.config2 & NHMEX_M_PMON_MM_CFG_EN)
0808             reg2->config = event->attr.config2;
0809         else
0810             reg2->config = ~0ULL;
0811         if (box->pmu->pmu_idx == 0)
0812             reg2->reg = NHMEX_M0_MSR_PMU_MM_CFG;
0813         else
0814             reg2->reg = NHMEX_M1_MSR_PMU_MM_CFG;
0815     }
0816     return 0;
0817 }
0818 
0819 static u64 nhmex_mbox_shared_reg_config(struct intel_uncore_box *box, int idx)
0820 {
0821     struct intel_uncore_extra_reg *er;
0822     unsigned long flags;
0823     u64 config;
0824 
0825     if (idx < EXTRA_REG_NHMEX_M_ZDP_CTL_FVC)
0826         return box->shared_regs[idx].config;
0827 
0828     er = &box->shared_regs[EXTRA_REG_NHMEX_M_ZDP_CTL_FVC];
0829     raw_spin_lock_irqsave(&er->lock, flags);
0830     config = er->config;
0831     raw_spin_unlock_irqrestore(&er->lock, flags);
0832     return config;
0833 }
0834 
0835 static void nhmex_mbox_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
0836 {
0837     struct hw_perf_event *hwc = &event->hw;
0838     struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
0839     struct hw_perf_event_extra *reg2 = &hwc->branch_reg;
0840     int idx;
0841 
0842     idx = __BITS_VALUE(reg1->idx, 0, 8);
0843     if (idx != 0xff)
0844         wrmsrl(__BITS_VALUE(reg1->reg, 0, 16),
0845             nhmex_mbox_shared_reg_config(box, idx));
0846     idx = __BITS_VALUE(reg1->idx, 1, 8);
0847     if (idx != 0xff)
0848         wrmsrl(__BITS_VALUE(reg1->reg, 1, 16),
0849             nhmex_mbox_shared_reg_config(box, idx));
0850 
0851     if (reg2->idx != EXTRA_REG_NONE) {
0852         wrmsrl(reg2->reg, 0);
0853         if (reg2->config != ~0ULL) {
0854             wrmsrl(reg2->reg + 1,
0855                 reg2->config & NHMEX_M_PMON_ADDR_MATCH_MASK);
0856             wrmsrl(reg2->reg + 2, NHMEX_M_PMON_ADDR_MASK_MASK &
0857                 (reg2->config >> NHMEX_M_PMON_ADDR_MASK_SHIFT));
0858             wrmsrl(reg2->reg, NHMEX_M_PMON_MM_CFG_EN);
0859         }
0860     }
0861 
0862     wrmsrl(hwc->config_base, hwc->config | NHMEX_PMON_CTL_EN_BIT0);
0863 }
0864 
0865 DEFINE_UNCORE_FORMAT_ATTR(count_mode,       count_mode, "config:2-3");
0866 DEFINE_UNCORE_FORMAT_ATTR(storage_mode,     storage_mode,   "config:4-5");
0867 DEFINE_UNCORE_FORMAT_ATTR(wrap_mode,        wrap_mode,  "config:6");
0868 DEFINE_UNCORE_FORMAT_ATTR(flag_mode,        flag_mode,  "config:7");
0869 DEFINE_UNCORE_FORMAT_ATTR(inc_sel,      inc_sel,    "config:9-13");
0870 DEFINE_UNCORE_FORMAT_ATTR(set_flag_sel,     set_flag_sel,   "config:19-21");
0871 DEFINE_UNCORE_FORMAT_ATTR(filter_cfg_en,    filter_cfg_en,  "config2:63");
0872 DEFINE_UNCORE_FORMAT_ATTR(filter_match,     filter_match,   "config2:0-33");
0873 DEFINE_UNCORE_FORMAT_ATTR(filter_mask,      filter_mask,    "config2:34-61");
0874 DEFINE_UNCORE_FORMAT_ATTR(dsp,          dsp,        "config1:0-31");
0875 DEFINE_UNCORE_FORMAT_ATTR(thr,          thr,        "config1:0-31");
0876 DEFINE_UNCORE_FORMAT_ATTR(fvc,          fvc,        "config1:0-31");
0877 DEFINE_UNCORE_FORMAT_ATTR(pgt,          pgt,        "config1:0-31");
0878 DEFINE_UNCORE_FORMAT_ATTR(map,          map,        "config1:0-31");
0879 DEFINE_UNCORE_FORMAT_ATTR(iss,          iss,        "config1:0-31");
0880 DEFINE_UNCORE_FORMAT_ATTR(pld,          pld,        "config1:32-63");
0881 
0882 static struct attribute *nhmex_uncore_mbox_formats_attr[] = {
0883     &format_attr_count_mode.attr,
0884     &format_attr_storage_mode.attr,
0885     &format_attr_wrap_mode.attr,
0886     &format_attr_flag_mode.attr,
0887     &format_attr_inc_sel.attr,
0888     &format_attr_set_flag_sel.attr,
0889     &format_attr_filter_cfg_en.attr,
0890     &format_attr_filter_match.attr,
0891     &format_attr_filter_mask.attr,
0892     &format_attr_dsp.attr,
0893     &format_attr_thr.attr,
0894     &format_attr_fvc.attr,
0895     &format_attr_pgt.attr,
0896     &format_attr_map.attr,
0897     &format_attr_iss.attr,
0898     &format_attr_pld.attr,
0899     NULL,
0900 };
0901 
0902 static const struct attribute_group nhmex_uncore_mbox_format_group = {
0903     .name       = "format",
0904     .attrs      = nhmex_uncore_mbox_formats_attr,
0905 };
0906 
0907 static struct uncore_event_desc nhmex_uncore_mbox_events[] = {
0908     INTEL_UNCORE_EVENT_DESC(bbox_cmds_read, "inc_sel=0xd,fvc=0x2800"),
0909     INTEL_UNCORE_EVENT_DESC(bbox_cmds_write, "inc_sel=0xd,fvc=0x2820"),
0910     { /* end: all zeroes */ },
0911 };
0912 
0913 static struct uncore_event_desc wsmex_uncore_mbox_events[] = {
0914     INTEL_UNCORE_EVENT_DESC(bbox_cmds_read, "inc_sel=0xd,fvc=0x5000"),
0915     INTEL_UNCORE_EVENT_DESC(bbox_cmds_write, "inc_sel=0xd,fvc=0x5040"),
0916     { /* end: all zeroes */ },
0917 };
0918 
0919 static struct intel_uncore_ops nhmex_uncore_mbox_ops = {
0920     NHMEX_UNCORE_OPS_COMMON_INIT(),
0921     .enable_event   = nhmex_mbox_msr_enable_event,
0922     .hw_config  = nhmex_mbox_hw_config,
0923     .get_constraint = nhmex_mbox_get_constraint,
0924     .put_constraint = nhmex_mbox_put_constraint,
0925 };
0926 
0927 static struct intel_uncore_type nhmex_uncore_mbox = {
0928     .name           = "mbox",
0929     .num_counters       = 6,
0930     .num_boxes      = 2,
0931     .perf_ctr_bits      = 48,
0932     .event_ctl      = NHMEX_M0_MSR_PMU_CTL0,
0933     .perf_ctr       = NHMEX_M0_MSR_PMU_CNT0,
0934     .event_mask     = NHMEX_M_PMON_RAW_EVENT_MASK,
0935     .box_ctl        = NHMEX_M0_MSR_GLOBAL_CTL,
0936     .msr_offset     = NHMEX_M_MSR_OFFSET,
0937     .pair_ctr_ctl       = 1,
0938     .num_shared_regs    = 8,
0939     .event_descs        = nhmex_uncore_mbox_events,
0940     .ops            = &nhmex_uncore_mbox_ops,
0941     .format_group       = &nhmex_uncore_mbox_format_group,
0942 };
0943 
0944 static void nhmex_rbox_alter_er(struct intel_uncore_box *box, struct perf_event *event)
0945 {
0946     struct hw_perf_event *hwc = &event->hw;
0947     struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
0948 
0949     /* adjust the main event selector and extra register index */
0950     if (reg1->idx % 2) {
0951         reg1->idx--;
0952         hwc->config -= 1 << NHMEX_R_PMON_CTL_EV_SEL_SHIFT;
0953     } else {
0954         reg1->idx++;
0955         hwc->config += 1 << NHMEX_R_PMON_CTL_EV_SEL_SHIFT;
0956     }
0957 
0958     /* adjust extra register config */
0959     switch (reg1->idx % 6) {
0960     case 2:
0961         /* shift the 8~15 bits to the 0~7 bits */
0962         reg1->config >>= 8;
0963         break;
0964     case 3:
0965         /* shift the 0~7 bits to the 8~15 bits */
0966         reg1->config <<= 8;
0967         break;
0968     }
0969 }
0970 
0971 /*
0972  * Each rbox has 4 event set which monitor PQI port 0~3 or 4~7.
0973  * An event set consists of 6 events, the 3rd and 4th events in
0974  * an event set use the same extra register. So an event set uses
0975  * 5 extra registers.
0976  */
0977 static struct event_constraint *
0978 nhmex_rbox_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
0979 {
0980     struct hw_perf_event *hwc = &event->hw;
0981     struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
0982     struct hw_perf_event_extra *reg2 = &hwc->branch_reg;
0983     struct intel_uncore_extra_reg *er;
0984     unsigned long flags;
0985     int idx, er_idx;
0986     u64 config1;
0987     bool ok = false;
0988 
0989     if (!uncore_box_is_fake(box) && reg1->alloc)
0990         return NULL;
0991 
0992     idx = reg1->idx % 6;
0993     config1 = reg1->config;
0994 again:
0995     er_idx = idx;
0996     /* the 3rd and 4th events use the same extra register */
0997     if (er_idx > 2)
0998         er_idx--;
0999     er_idx += (reg1->idx / 6) * 5;
1000 
1001     er = &box->shared_regs[er_idx];
1002     raw_spin_lock_irqsave(&er->lock, flags);
1003     if (idx < 2) {
1004         if (!atomic_read(&er->ref) || er->config == reg1->config) {
1005             atomic_inc(&er->ref);
1006             er->config = reg1->config;
1007             ok = true;
1008         }
1009     } else if (idx == 2 || idx == 3) {
1010         /*
1011          * these two events use different fields in a extra register,
1012          * the 0~7 bits and the 8~15 bits respectively.
1013          */
1014         u64 mask = 0xff << ((idx - 2) * 8);
1015         if (!__BITS_VALUE(atomic_read(&er->ref), idx - 2, 8) ||
1016                 !((er->config ^ config1) & mask)) {
1017             atomic_add(1 << ((idx - 2) * 8), &er->ref);
1018             er->config &= ~mask;
1019             er->config |= config1 & mask;
1020             ok = true;
1021         }
1022     } else {
1023         if (!atomic_read(&er->ref) ||
1024                 (er->config == (hwc->config >> 32) &&
1025                  er->config1 == reg1->config &&
1026                  er->config2 == reg2->config)) {
1027             atomic_inc(&er->ref);
1028             er->config = (hwc->config >> 32);
1029             er->config1 = reg1->config;
1030             er->config2 = reg2->config;
1031             ok = true;
1032         }
1033     }
1034     raw_spin_unlock_irqrestore(&er->lock, flags);
1035 
1036     if (!ok) {
1037         /*
1038          * The Rbox events are always in pairs. The paired
1039          * events are functional identical, but use different
1040          * extra registers. If we failed to take an extra
1041          * register, try the alternative.
1042          */
1043         idx ^= 1;
1044         if (idx != reg1->idx % 6) {
1045             if (idx == 2)
1046                 config1 >>= 8;
1047             else if (idx == 3)
1048                 config1 <<= 8;
1049             goto again;
1050         }
1051     } else {
1052         if (!uncore_box_is_fake(box)) {
1053             if (idx != reg1->idx % 6)
1054                 nhmex_rbox_alter_er(box, event);
1055             reg1->alloc = 1;
1056         }
1057         return NULL;
1058     }
1059     return &uncore_constraint_empty;
1060 }
1061 
1062 static void nhmex_rbox_put_constraint(struct intel_uncore_box *box, struct perf_event *event)
1063 {
1064     struct intel_uncore_extra_reg *er;
1065     struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
1066     int idx, er_idx;
1067 
1068     if (uncore_box_is_fake(box) || !reg1->alloc)
1069         return;
1070 
1071     idx = reg1->idx % 6;
1072     er_idx = idx;
1073     if (er_idx > 2)
1074         er_idx--;
1075     er_idx += (reg1->idx / 6) * 5;
1076 
1077     er = &box->shared_regs[er_idx];
1078     if (idx == 2 || idx == 3)
1079         atomic_sub(1 << ((idx - 2) * 8), &er->ref);
1080     else
1081         atomic_dec(&er->ref);
1082 
1083     reg1->alloc = 0;
1084 }
1085 
1086 static int nhmex_rbox_hw_config(struct intel_uncore_box *box, struct perf_event *event)
1087 {
1088     struct hw_perf_event *hwc = &event->hw;
1089     struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
1090     struct hw_perf_event_extra *reg2 = &event->hw.branch_reg;
1091     int idx;
1092 
1093     idx = (event->hw.config & NHMEX_R_PMON_CTL_EV_SEL_MASK) >>
1094         NHMEX_R_PMON_CTL_EV_SEL_SHIFT;
1095     if (idx >= 0x18)
1096         return -EINVAL;
1097 
1098     reg1->idx = idx;
1099     reg1->config = event->attr.config1;
1100 
1101     switch (idx % 6) {
1102     case 4:
1103     case 5:
1104         hwc->config |= event->attr.config & (~0ULL << 32);
1105         reg2->config = event->attr.config2;
1106         break;
1107     }
1108     return 0;
1109 }
1110 
1111 static void nhmex_rbox_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
1112 {
1113     struct hw_perf_event *hwc = &event->hw;
1114     struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
1115     struct hw_perf_event_extra *reg2 = &hwc->branch_reg;
1116     int idx, port;
1117 
1118     idx = reg1->idx;
1119     port = idx / 6 + box->pmu->pmu_idx * 4;
1120 
1121     switch (idx % 6) {
1122     case 0:
1123         wrmsrl(NHMEX_R_MSR_PORTN_IPERF_CFG0(port), reg1->config);
1124         break;
1125     case 1:
1126         wrmsrl(NHMEX_R_MSR_PORTN_IPERF_CFG1(port), reg1->config);
1127         break;
1128     case 2:
1129     case 3:
1130         wrmsrl(NHMEX_R_MSR_PORTN_QLX_CFG(port),
1131             uncore_shared_reg_config(box, 2 + (idx / 6) * 5));
1132         break;
1133     case 4:
1134         wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET1_MM_CFG(port),
1135             hwc->config >> 32);
1136         wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET1_MATCH(port), reg1->config);
1137         wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET1_MASK(port), reg2->config);
1138         break;
1139     case 5:
1140         wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET2_MM_CFG(port),
1141             hwc->config >> 32);
1142         wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET2_MATCH(port), reg1->config);
1143         wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET2_MASK(port), reg2->config);
1144         break;
1145     }
1146 
1147     wrmsrl(hwc->config_base, NHMEX_PMON_CTL_EN_BIT0 |
1148         (hwc->config & NHMEX_R_PMON_CTL_EV_SEL_MASK));
1149 }
1150 
1151 DEFINE_UNCORE_FORMAT_ATTR(xbr_mm_cfg, xbr_mm_cfg, "config:32-63");
1152 DEFINE_UNCORE_FORMAT_ATTR(xbr_match, xbr_match, "config1:0-63");
1153 DEFINE_UNCORE_FORMAT_ATTR(xbr_mask, xbr_mask, "config2:0-63");
1154 DEFINE_UNCORE_FORMAT_ATTR(qlx_cfg, qlx_cfg, "config1:0-15");
1155 DEFINE_UNCORE_FORMAT_ATTR(iperf_cfg, iperf_cfg, "config1:0-31");
1156 
1157 static struct attribute *nhmex_uncore_rbox_formats_attr[] = {
1158     &format_attr_event5.attr,
1159     &format_attr_xbr_mm_cfg.attr,
1160     &format_attr_xbr_match.attr,
1161     &format_attr_xbr_mask.attr,
1162     &format_attr_qlx_cfg.attr,
1163     &format_attr_iperf_cfg.attr,
1164     NULL,
1165 };
1166 
1167 static const struct attribute_group nhmex_uncore_rbox_format_group = {
1168     .name = "format",
1169     .attrs = nhmex_uncore_rbox_formats_attr,
1170 };
1171 
1172 static struct uncore_event_desc nhmex_uncore_rbox_events[] = {
1173     INTEL_UNCORE_EVENT_DESC(qpi0_flit_send,     "event=0x0,iperf_cfg=0x80000000"),
1174     INTEL_UNCORE_EVENT_DESC(qpi1_filt_send,     "event=0x6,iperf_cfg=0x80000000"),
1175     INTEL_UNCORE_EVENT_DESC(qpi0_idle_filt,     "event=0x0,iperf_cfg=0x40000000"),
1176     INTEL_UNCORE_EVENT_DESC(qpi1_idle_filt,     "event=0x6,iperf_cfg=0x40000000"),
1177     INTEL_UNCORE_EVENT_DESC(qpi0_date_response, "event=0x0,iperf_cfg=0xc4"),
1178     INTEL_UNCORE_EVENT_DESC(qpi1_date_response, "event=0x6,iperf_cfg=0xc4"),
1179     { /* end: all zeroes */ },
1180 };
1181 
1182 static struct intel_uncore_ops nhmex_uncore_rbox_ops = {
1183     NHMEX_UNCORE_OPS_COMMON_INIT(),
1184     .enable_event       = nhmex_rbox_msr_enable_event,
1185     .hw_config      = nhmex_rbox_hw_config,
1186     .get_constraint     = nhmex_rbox_get_constraint,
1187     .put_constraint     = nhmex_rbox_put_constraint,
1188 };
1189 
1190 static struct intel_uncore_type nhmex_uncore_rbox = {
1191     .name           = "rbox",
1192     .num_counters       = 8,
1193     .num_boxes      = 2,
1194     .perf_ctr_bits      = 48,
1195     .event_ctl      = NHMEX_R_MSR_PMON_CTL0,
1196     .perf_ctr       = NHMEX_R_MSR_PMON_CNT0,
1197     .event_mask     = NHMEX_R_PMON_RAW_EVENT_MASK,
1198     .box_ctl        = NHMEX_R_MSR_GLOBAL_CTL,
1199     .msr_offset     = NHMEX_R_MSR_OFFSET,
1200     .pair_ctr_ctl       = 1,
1201     .num_shared_regs    = 20,
1202     .event_descs        = nhmex_uncore_rbox_events,
1203     .ops            = &nhmex_uncore_rbox_ops,
1204     .format_group       = &nhmex_uncore_rbox_format_group
1205 };
1206 
1207 static struct intel_uncore_type *nhmex_msr_uncores[] = {
1208     &nhmex_uncore_ubox,
1209     &nhmex_uncore_cbox,
1210     &nhmex_uncore_bbox,
1211     &nhmex_uncore_sbox,
1212     &nhmex_uncore_mbox,
1213     &nhmex_uncore_rbox,
1214     &nhmex_uncore_wbox,
1215     NULL,
1216 };
1217 
1218 void nhmex_uncore_cpu_init(void)
1219 {
1220     if (boot_cpu_data.x86_model == 46)
1221         uncore_nhmex = true;
1222     else
1223         nhmex_uncore_mbox.event_descs = wsmex_uncore_mbox_events;
1224     if (nhmex_uncore_cbox.num_boxes > boot_cpu_data.x86_max_cores)
1225         nhmex_uncore_cbox.num_boxes = boot_cpu_data.x86_max_cores;
1226     uncore_msr_uncores = nhmex_msr_uncores;
1227 }
1228 /* end of Nehalem-EX uncore support */
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!