Back to home page

LXR

 
 

    


0001 /**
0002  * @file op_model_p4.c
0003  * P4 model-specific MSR operations
0004  *
0005  * @remark Copyright 2002 OProfile authors
0006  * @remark Read the file COPYING
0007  *
0008  * @author Graydon Hoare
0009  */
0010 
0011 #include <linux/oprofile.h>
0012 #include <linux/smp.h>
0013 #include <linux/ptrace.h>
0014 #include <asm/nmi.h>
0015 #include <asm/msr.h>
0016 #include <asm/fixmap.h>
0017 #include <asm/apic.h>
0018 
0019 
0020 #include "op_x86_model.h"
0021 #include "op_counter.h"
0022 
0023 #define NUM_EVENTS 39
0024 
0025 #define NUM_COUNTERS_NON_HT 8
0026 #define NUM_ESCRS_NON_HT 45
0027 #define NUM_CCCRS_NON_HT 18
0028 #define NUM_CONTROLS_NON_HT (NUM_ESCRS_NON_HT + NUM_CCCRS_NON_HT)
0029 
0030 #define NUM_COUNTERS_HT2 4
0031 #define NUM_ESCRS_HT2 23
0032 #define NUM_CCCRS_HT2 9
0033 #define NUM_CONTROLS_HT2 (NUM_ESCRS_HT2 + NUM_CCCRS_HT2)
0034 
0035 #define OP_CTR_OVERFLOW         (1ULL<<31)
0036 
0037 static unsigned int num_counters = NUM_COUNTERS_NON_HT;
0038 static unsigned int num_controls = NUM_CONTROLS_NON_HT;
0039 
0040 /* this has to be checked dynamically since the
0041    hyper-threadedness of a chip is discovered at
0042    kernel boot-time. */
0043 static inline void setup_num_counters(void)
0044 {
0045 #ifdef CONFIG_SMP
0046     if (smp_num_siblings == 2) {
0047         num_counters = NUM_COUNTERS_HT2;
0048         num_controls = NUM_CONTROLS_HT2;
0049     }
0050 #endif
0051 }
0052 
0053 static inline int addr_increment(void)
0054 {
0055 #ifdef CONFIG_SMP
0056     return smp_num_siblings == 2 ? 2 : 1;
0057 #else
0058     return 1;
0059 #endif
0060 }
0061 
0062 
0063 /* tables to simulate simplified hardware view of p4 registers */
0064 struct p4_counter_binding {
0065     int virt_counter;
0066     int counter_address;
0067     int cccr_address;
0068 };
0069 
0070 struct p4_event_binding {
0071     int escr_select;  /* value to put in CCCR */
0072     int event_select; /* value to put in ESCR */
0073     struct {
0074         int virt_counter; /* for this counter... */
0075         int escr_address; /* use this ESCR       */
0076     } bindings[2];
0077 };
0078 
0079 /* nb: these CTR_* defines are a duplicate of defines in
0080    event/i386.p4*events. */
0081 
0082 
0083 #define CTR_BPU_0      (1 << 0)
0084 #define CTR_MS_0       (1 << 1)
0085 #define CTR_FLAME_0    (1 << 2)
0086 #define CTR_IQ_4       (1 << 3)
0087 #define CTR_BPU_2      (1 << 4)
0088 #define CTR_MS_2       (1 << 5)
0089 #define CTR_FLAME_2    (1 << 6)
0090 #define CTR_IQ_5       (1 << 7)
0091 
0092 static struct p4_counter_binding p4_counters[NUM_COUNTERS_NON_HT] = {
0093     { CTR_BPU_0,   MSR_P4_BPU_PERFCTR0,   MSR_P4_BPU_CCCR0 },
0094     { CTR_MS_0,    MSR_P4_MS_PERFCTR0,    MSR_P4_MS_CCCR0 },
0095     { CTR_FLAME_0, MSR_P4_FLAME_PERFCTR0, MSR_P4_FLAME_CCCR0 },
0096     { CTR_IQ_4,    MSR_P4_IQ_PERFCTR4,    MSR_P4_IQ_CCCR4 },
0097     { CTR_BPU_2,   MSR_P4_BPU_PERFCTR2,   MSR_P4_BPU_CCCR2 },
0098     { CTR_MS_2,    MSR_P4_MS_PERFCTR2,    MSR_P4_MS_CCCR2 },
0099     { CTR_FLAME_2, MSR_P4_FLAME_PERFCTR2, MSR_P4_FLAME_CCCR2 },
0100     { CTR_IQ_5,    MSR_P4_IQ_PERFCTR5,    MSR_P4_IQ_CCCR5 }
0101 };
0102 
0103 #define NUM_UNUSED_CCCRS (NUM_CCCRS_NON_HT - NUM_COUNTERS_NON_HT)
0104 
0105 /* p4 event codes in libop/op_event.h are indices into this table. */
0106 
0107 static struct p4_event_binding p4_events[NUM_EVENTS] = {
0108 
0109     { /* BRANCH_RETIRED */
0110         0x05, 0x06,
0111         { {CTR_IQ_4, MSR_P4_CRU_ESCR2},
0112           {CTR_IQ_5, MSR_P4_CRU_ESCR3} }
0113     },
0114 
0115     { /* MISPRED_BRANCH_RETIRED */
0116         0x04, 0x03,
0117         { { CTR_IQ_4, MSR_P4_CRU_ESCR0},
0118           { CTR_IQ_5, MSR_P4_CRU_ESCR1} }
0119     },
0120 
0121     { /* TC_DELIVER_MODE */
0122         0x01, 0x01,
0123         { { CTR_MS_0, MSR_P4_TC_ESCR0},
0124           { CTR_MS_2, MSR_P4_TC_ESCR1} }
0125     },
0126 
0127     { /* BPU_FETCH_REQUEST */
0128         0x00, 0x03,
0129         { { CTR_BPU_0, MSR_P4_BPU_ESCR0},
0130           { CTR_BPU_2, MSR_P4_BPU_ESCR1} }
0131     },
0132 
0133     { /* ITLB_REFERENCE */
0134         0x03, 0x18,
0135         { { CTR_BPU_0, MSR_P4_ITLB_ESCR0},
0136           { CTR_BPU_2, MSR_P4_ITLB_ESCR1} }
0137     },
0138 
0139     { /* MEMORY_CANCEL */
0140         0x05, 0x02,
0141         { { CTR_FLAME_0, MSR_P4_DAC_ESCR0},
0142           { CTR_FLAME_2, MSR_P4_DAC_ESCR1} }
0143     },
0144 
0145     { /* MEMORY_COMPLETE */
0146         0x02, 0x08,
0147         { { CTR_FLAME_0, MSR_P4_SAAT_ESCR0},
0148           { CTR_FLAME_2, MSR_P4_SAAT_ESCR1} }
0149     },
0150 
0151     { /* LOAD_PORT_REPLAY */
0152         0x02, 0x04,
0153         { { CTR_FLAME_0, MSR_P4_SAAT_ESCR0},
0154           { CTR_FLAME_2, MSR_P4_SAAT_ESCR1} }
0155     },
0156 
0157     { /* STORE_PORT_REPLAY */
0158         0x02, 0x05,
0159         { { CTR_FLAME_0, MSR_P4_SAAT_ESCR0},
0160           { CTR_FLAME_2, MSR_P4_SAAT_ESCR1} }
0161     },
0162 
0163     { /* MOB_LOAD_REPLAY */
0164         0x02, 0x03,
0165         { { CTR_BPU_0, MSR_P4_MOB_ESCR0},
0166           { CTR_BPU_2, MSR_P4_MOB_ESCR1} }
0167     },
0168 
0169     { /* PAGE_WALK_TYPE */
0170         0x04, 0x01,
0171         { { CTR_BPU_0, MSR_P4_PMH_ESCR0},
0172           { CTR_BPU_2, MSR_P4_PMH_ESCR1} }
0173     },
0174 
0175     { /* BSQ_CACHE_REFERENCE */
0176         0x07, 0x0c,
0177         { { CTR_BPU_0, MSR_P4_BSU_ESCR0},
0178           { CTR_BPU_2, MSR_P4_BSU_ESCR1} }
0179     },
0180 
0181     { /* IOQ_ALLOCATION */
0182         0x06, 0x03,
0183         { { CTR_BPU_0, MSR_P4_FSB_ESCR0},
0184           { 0, 0 } }
0185     },
0186 
0187     { /* IOQ_ACTIVE_ENTRIES */
0188         0x06, 0x1a,
0189         { { CTR_BPU_2, MSR_P4_FSB_ESCR1},
0190           { 0, 0 } }
0191     },
0192 
0193     { /* FSB_DATA_ACTIVITY */
0194         0x06, 0x17,
0195         { { CTR_BPU_0, MSR_P4_FSB_ESCR0},
0196           { CTR_BPU_2, MSR_P4_FSB_ESCR1} }
0197     },
0198 
0199     { /* BSQ_ALLOCATION */
0200         0x07, 0x05,
0201         { { CTR_BPU_0, MSR_P4_BSU_ESCR0},
0202           { 0, 0 } }
0203     },
0204 
0205     { /* BSQ_ACTIVE_ENTRIES */
0206         0x07, 0x06,
0207         { { CTR_BPU_2, MSR_P4_BSU_ESCR1 /* guess */},
0208           { 0, 0 } }
0209     },
0210 
0211     { /* X87_ASSIST */
0212         0x05, 0x03,
0213         { { CTR_IQ_4, MSR_P4_CRU_ESCR2},
0214           { CTR_IQ_5, MSR_P4_CRU_ESCR3} }
0215     },
0216 
0217     { /* SSE_INPUT_ASSIST */
0218         0x01, 0x34,
0219         { { CTR_FLAME_0, MSR_P4_FIRM_ESCR0},
0220           { CTR_FLAME_2, MSR_P4_FIRM_ESCR1} }
0221     },
0222 
0223     { /* PACKED_SP_UOP */
0224         0x01, 0x08,
0225         { { CTR_FLAME_0, MSR_P4_FIRM_ESCR0},
0226           { CTR_FLAME_2, MSR_P4_FIRM_ESCR1} }
0227     },
0228 
0229     { /* PACKED_DP_UOP */
0230         0x01, 0x0c,
0231         { { CTR_FLAME_0, MSR_P4_FIRM_ESCR0},
0232           { CTR_FLAME_2, MSR_P4_FIRM_ESCR1} }
0233     },
0234 
0235     { /* SCALAR_SP_UOP */
0236         0x01, 0x0a,
0237         { { CTR_FLAME_0, MSR_P4_FIRM_ESCR0},
0238           { CTR_FLAME_2, MSR_P4_FIRM_ESCR1} }
0239     },
0240 
0241     { /* SCALAR_DP_UOP */
0242         0x01, 0x0e,
0243         { { CTR_FLAME_0, MSR_P4_FIRM_ESCR0},
0244           { CTR_FLAME_2, MSR_P4_FIRM_ESCR1} }
0245     },
0246 
0247     { /* 64BIT_MMX_UOP */
0248         0x01, 0x02,
0249         { { CTR_FLAME_0, MSR_P4_FIRM_ESCR0},
0250           { CTR_FLAME_2, MSR_P4_FIRM_ESCR1} }
0251     },
0252 
0253     { /* 128BIT_MMX_UOP */
0254         0x01, 0x1a,
0255         { { CTR_FLAME_0, MSR_P4_FIRM_ESCR0},
0256           { CTR_FLAME_2, MSR_P4_FIRM_ESCR1} }
0257     },
0258 
0259     { /* X87_FP_UOP */
0260         0x01, 0x04,
0261         { { CTR_FLAME_0, MSR_P4_FIRM_ESCR0},
0262           { CTR_FLAME_2, MSR_P4_FIRM_ESCR1} }
0263     },
0264 
0265     { /* X87_SIMD_MOVES_UOP */
0266         0x01, 0x2e,
0267         { { CTR_FLAME_0, MSR_P4_FIRM_ESCR0},
0268           { CTR_FLAME_2, MSR_P4_FIRM_ESCR1} }
0269     },
0270 
0271     { /* MACHINE_CLEAR */
0272         0x05, 0x02,
0273         { { CTR_IQ_4, MSR_P4_CRU_ESCR2},
0274           { CTR_IQ_5, MSR_P4_CRU_ESCR3} }
0275     },
0276 
0277     { /* GLOBAL_POWER_EVENTS */
0278         0x06, 0x13 /* older manual says 0x05, newer 0x13 */,
0279         { { CTR_BPU_0, MSR_P4_FSB_ESCR0},
0280           { CTR_BPU_2, MSR_P4_FSB_ESCR1} }
0281     },
0282 
0283     { /* TC_MS_XFER */
0284         0x00, 0x05,
0285         { { CTR_MS_0, MSR_P4_MS_ESCR0},
0286           { CTR_MS_2, MSR_P4_MS_ESCR1} }
0287     },
0288 
0289     { /* UOP_QUEUE_WRITES */
0290         0x00, 0x09,
0291         { { CTR_MS_0, MSR_P4_MS_ESCR0},
0292           { CTR_MS_2, MSR_P4_MS_ESCR1} }
0293     },
0294 
0295     { /* FRONT_END_EVENT */
0296         0x05, 0x08,
0297         { { CTR_IQ_4, MSR_P4_CRU_ESCR2},
0298           { CTR_IQ_5, MSR_P4_CRU_ESCR3} }
0299     },
0300 
0301     { /* EXECUTION_EVENT */
0302         0x05, 0x0c,
0303         { { CTR_IQ_4, MSR_P4_CRU_ESCR2},
0304           { CTR_IQ_5, MSR_P4_CRU_ESCR3} }
0305     },
0306 
0307     { /* REPLAY_EVENT */
0308         0x05, 0x09,
0309         { { CTR_IQ_4, MSR_P4_CRU_ESCR2},
0310           { CTR_IQ_5, MSR_P4_CRU_ESCR3} }
0311     },
0312 
0313     { /* INSTR_RETIRED */
0314         0x04, 0x02,
0315         { { CTR_IQ_4, MSR_P4_CRU_ESCR0},
0316           { CTR_IQ_5, MSR_P4_CRU_ESCR1} }
0317     },
0318 
0319     { /* UOPS_RETIRED */
0320         0x04, 0x01,
0321         { { CTR_IQ_4, MSR_P4_CRU_ESCR0},
0322           { CTR_IQ_5, MSR_P4_CRU_ESCR1} }
0323     },
0324 
0325     { /* UOP_TYPE */
0326         0x02, 0x02,
0327         { { CTR_IQ_4, MSR_P4_RAT_ESCR0},
0328           { CTR_IQ_5, MSR_P4_RAT_ESCR1} }
0329     },
0330 
0331     { /* RETIRED_MISPRED_BRANCH_TYPE */
0332         0x02, 0x05,
0333         { { CTR_MS_0, MSR_P4_TBPU_ESCR0},
0334           { CTR_MS_2, MSR_P4_TBPU_ESCR1} }
0335     },
0336 
0337     { /* RETIRED_BRANCH_TYPE */
0338         0x02, 0x04,
0339         { { CTR_MS_0, MSR_P4_TBPU_ESCR0},
0340           { CTR_MS_2, MSR_P4_TBPU_ESCR1} }
0341     }
0342 };
0343 
0344 
0345 #define MISC_PMC_ENABLED_P(x) ((x) & 1 << 7)
0346 
0347 #define ESCR_RESERVED_BITS 0x80000003
0348 #define ESCR_CLEAR(escr) ((escr) &= ESCR_RESERVED_BITS)
0349 #define ESCR_SET_USR_0(escr, usr) ((escr) |= (((usr) & 1) << 2))
0350 #define ESCR_SET_OS_0(escr, os) ((escr) |= (((os) & 1) << 3))
0351 #define ESCR_SET_USR_1(escr, usr) ((escr) |= (((usr) & 1)))
0352 #define ESCR_SET_OS_1(escr, os) ((escr) |= (((os) & 1) << 1))
0353 #define ESCR_SET_EVENT_SELECT(escr, sel) ((escr) |= (((sel) & 0x3f) << 25))
0354 #define ESCR_SET_EVENT_MASK(escr, mask) ((escr) |= (((mask) & 0xffff) << 9))
0355 
0356 #define CCCR_RESERVED_BITS 0x38030FFF
0357 #define CCCR_CLEAR(cccr) ((cccr) &= CCCR_RESERVED_BITS)
0358 #define CCCR_SET_REQUIRED_BITS(cccr) ((cccr) |= 0x00030000)
0359 #define CCCR_SET_ESCR_SELECT(cccr, sel) ((cccr) |= (((sel) & 0x07) << 13))
0360 #define CCCR_SET_PMI_OVF_0(cccr) ((cccr) |= (1<<26))
0361 #define CCCR_SET_PMI_OVF_1(cccr) ((cccr) |= (1<<27))
0362 #define CCCR_SET_ENABLE(cccr) ((cccr) |= (1<<12))
0363 #define CCCR_SET_DISABLE(cccr) ((cccr) &= ~(1<<12))
0364 #define CCCR_OVF_P(cccr) ((cccr) & (1U<<31))
0365 #define CCCR_CLEAR_OVF(cccr) ((cccr) &= (~(1U<<31)))
0366 
0367 
0368 /* this assigns a "stagger" to the current CPU, which is used throughout
0369    the code in this module as an extra array offset, to select the "even"
0370    or "odd" part of all the divided resources. */
0371 static unsigned int get_stagger(void)
0372 {
0373 #ifdef CONFIG_SMP
0374     int cpu = smp_processor_id();
0375     return cpu != cpumask_first(this_cpu_cpumask_var_ptr(cpu_sibling_map));
0376 #endif
0377     return 0;
0378 }
0379 
0380 
0381 /* finally, mediate access to a real hardware counter
0382    by passing a "virtual" counter numer to this macro,
0383    along with your stagger setting. */
0384 #define VIRT_CTR(stagger, i) ((i) + ((num_counters) * (stagger)))
0385 
0386 static unsigned long reset_value[NUM_COUNTERS_NON_HT];
0387 
0388 static void p4_shutdown(struct op_msrs const * const msrs)
0389 {
0390     int i;
0391 
0392     for (i = 0; i < num_counters; ++i) {
0393         if (msrs->counters[i].addr)
0394             release_perfctr_nmi(msrs->counters[i].addr);
0395     }
0396     /*
0397      * some of the control registers are specially reserved in
0398      * conjunction with the counter registers (hence the starting offset).
0399      * This saves a few bits.
0400      */
0401     for (i = num_counters; i < num_controls; ++i) {
0402         if (msrs->controls[i].addr)
0403             release_evntsel_nmi(msrs->controls[i].addr);
0404     }
0405 }
0406 
0407 static int p4_fill_in_addresses(struct op_msrs * const msrs)
0408 {
0409     unsigned int i;
0410     unsigned int addr, cccraddr, stag;
0411 
0412     setup_num_counters();
0413     stag = get_stagger();
0414 
0415     /* the counter & cccr registers we pay attention to */
0416     for (i = 0; i < num_counters; ++i) {
0417         addr = p4_counters[VIRT_CTR(stag, i)].counter_address;
0418         cccraddr = p4_counters[VIRT_CTR(stag, i)].cccr_address;
0419         if (reserve_perfctr_nmi(addr)) {
0420             msrs->counters[i].addr = addr;
0421             msrs->controls[i].addr = cccraddr;
0422         }
0423     }
0424 
0425     /* 43 ESCR registers in three or four discontiguous group */
0426     for (addr = MSR_P4_BSU_ESCR0 + stag;
0427          addr < MSR_P4_IQ_ESCR0; ++i, addr += addr_increment()) {
0428         if (reserve_evntsel_nmi(addr))
0429             msrs->controls[i].addr = addr;
0430     }
0431 
0432     /* no IQ_ESCR0/1 on some models, we save a seconde time BSU_ESCR0/1
0433      * to avoid special case in nmi_{save|restore}_registers() */
0434     if (boot_cpu_data.x86_model >= 0x3) {
0435         for (addr = MSR_P4_BSU_ESCR0 + stag;
0436              addr <= MSR_P4_BSU_ESCR1; ++i, addr += addr_increment()) {
0437             if (reserve_evntsel_nmi(addr))
0438                 msrs->controls[i].addr = addr;
0439         }
0440     } else {
0441         for (addr = MSR_P4_IQ_ESCR0 + stag;
0442              addr <= MSR_P4_IQ_ESCR1; ++i, addr += addr_increment()) {
0443             if (reserve_evntsel_nmi(addr))
0444                 msrs->controls[i].addr = addr;
0445         }
0446     }
0447 
0448     for (addr = MSR_P4_RAT_ESCR0 + stag;
0449          addr <= MSR_P4_SSU_ESCR0; ++i, addr += addr_increment()) {
0450         if (reserve_evntsel_nmi(addr))
0451             msrs->controls[i].addr = addr;
0452     }
0453 
0454     for (addr = MSR_P4_MS_ESCR0 + stag;
0455          addr <= MSR_P4_TC_ESCR1; ++i, addr += addr_increment()) {
0456         if (reserve_evntsel_nmi(addr))
0457             msrs->controls[i].addr = addr;
0458     }
0459 
0460     for (addr = MSR_P4_IX_ESCR0 + stag;
0461          addr <= MSR_P4_CRU_ESCR3; ++i, addr += addr_increment()) {
0462         if (reserve_evntsel_nmi(addr))
0463             msrs->controls[i].addr = addr;
0464     }
0465 
0466     /* there are 2 remaining non-contiguously located ESCRs */
0467 
0468     if (num_counters == NUM_COUNTERS_NON_HT) {
0469         /* standard non-HT CPUs handle both remaining ESCRs*/
0470         if (reserve_evntsel_nmi(MSR_P4_CRU_ESCR5))
0471             msrs->controls[i++].addr = MSR_P4_CRU_ESCR5;
0472         if (reserve_evntsel_nmi(MSR_P4_CRU_ESCR4))
0473             msrs->controls[i++].addr = MSR_P4_CRU_ESCR4;
0474 
0475     } else if (stag == 0) {
0476         /* HT CPUs give the first remainder to the even thread, as
0477            the 32nd control register */
0478         if (reserve_evntsel_nmi(MSR_P4_CRU_ESCR4))
0479             msrs->controls[i++].addr = MSR_P4_CRU_ESCR4;
0480 
0481     } else {
0482         /* and two copies of the second to the odd thread,
0483            for the 22st and 23nd control registers */
0484         if (reserve_evntsel_nmi(MSR_P4_CRU_ESCR5)) {
0485             msrs->controls[i++].addr = MSR_P4_CRU_ESCR5;
0486             msrs->controls[i++].addr = MSR_P4_CRU_ESCR5;
0487         }
0488     }
0489 
0490     for (i = 0; i < num_counters; ++i) {
0491         if (!counter_config[i].enabled)
0492             continue;
0493         if (msrs->controls[i].addr)
0494             continue;
0495         op_x86_warn_reserved(i);
0496         p4_shutdown(msrs);
0497         return -EBUSY;
0498     }
0499 
0500     return 0;
0501 }
0502 
0503 
0504 static void pmc_setup_one_p4_counter(unsigned int ctr)
0505 {
0506     int i;
0507     int const maxbind = 2;
0508     unsigned int cccr = 0;
0509     unsigned int escr = 0;
0510     unsigned int high = 0;
0511     unsigned int counter_bit;
0512     struct p4_event_binding *ev = NULL;
0513     unsigned int stag;
0514 
0515     stag = get_stagger();
0516 
0517     /* convert from counter *number* to counter *bit* */
0518     counter_bit = 1 << VIRT_CTR(stag, ctr);
0519 
0520     /* find our event binding structure. */
0521     if (counter_config[ctr].event <= 0 || counter_config[ctr].event > NUM_EVENTS) {
0522         printk(KERN_ERR
0523                "oprofile: P4 event code 0x%lx out of range\n",
0524                counter_config[ctr].event);
0525         return;
0526     }
0527 
0528     ev = &(p4_events[counter_config[ctr].event - 1]);
0529 
0530     for (i = 0; i < maxbind; i++) {
0531         if (ev->bindings[i].virt_counter & counter_bit) {
0532 
0533             /* modify ESCR */
0534             rdmsr(ev->bindings[i].escr_address, escr, high);
0535             ESCR_CLEAR(escr);
0536             if (stag == 0) {
0537                 ESCR_SET_USR_0(escr, counter_config[ctr].user);
0538                 ESCR_SET_OS_0(escr, counter_config[ctr].kernel);
0539             } else {
0540                 ESCR_SET_USR_1(escr, counter_config[ctr].user);
0541                 ESCR_SET_OS_1(escr, counter_config[ctr].kernel);
0542             }
0543             ESCR_SET_EVENT_SELECT(escr, ev->event_select);
0544             ESCR_SET_EVENT_MASK(escr, counter_config[ctr].unit_mask);
0545             wrmsr(ev->bindings[i].escr_address, escr, high);
0546 
0547             /* modify CCCR */
0548             rdmsr(p4_counters[VIRT_CTR(stag, ctr)].cccr_address,
0549                   cccr, high);
0550             CCCR_CLEAR(cccr);
0551             CCCR_SET_REQUIRED_BITS(cccr);
0552             CCCR_SET_ESCR_SELECT(cccr, ev->escr_select);
0553             if (stag == 0)
0554                 CCCR_SET_PMI_OVF_0(cccr);
0555             else
0556                 CCCR_SET_PMI_OVF_1(cccr);
0557             wrmsr(p4_counters[VIRT_CTR(stag, ctr)].cccr_address,
0558                   cccr, high);
0559             return;
0560         }
0561     }
0562 
0563     printk(KERN_ERR
0564            "oprofile: P4 event code 0x%lx no binding, stag %d ctr %d\n",
0565            counter_config[ctr].event, stag, ctr);
0566 }
0567 
0568 
0569 static void p4_setup_ctrs(struct op_x86_model_spec const *model,
0570               struct op_msrs const * const msrs)
0571 {
0572     unsigned int i;
0573     unsigned int low, high;
0574     unsigned int stag;
0575 
0576     stag = get_stagger();
0577 
0578     rdmsr(MSR_IA32_MISC_ENABLE, low, high);
0579     if (!MISC_PMC_ENABLED_P(low)) {
0580         printk(KERN_ERR "oprofile: P4 PMC not available\n");
0581         return;
0582     }
0583 
0584     /* clear the cccrs we will use */
0585     for (i = 0; i < num_counters; i++) {
0586         if (unlikely(!msrs->controls[i].addr))
0587             continue;
0588         rdmsr(p4_counters[VIRT_CTR(stag, i)].cccr_address, low, high);
0589         CCCR_CLEAR(low);
0590         CCCR_SET_REQUIRED_BITS(low);
0591         wrmsr(p4_counters[VIRT_CTR(stag, i)].cccr_address, low, high);
0592     }
0593 
0594     /* clear all escrs (including those outside our concern) */
0595     for (i = num_counters; i < num_controls; i++) {
0596         if (unlikely(!msrs->controls[i].addr))
0597             continue;
0598         wrmsr(msrs->controls[i].addr, 0, 0);
0599     }
0600 
0601     /* setup all counters */
0602     for (i = 0; i < num_counters; ++i) {
0603         if (counter_config[i].enabled && msrs->controls[i].addr) {
0604             reset_value[i] = counter_config[i].count;
0605             pmc_setup_one_p4_counter(i);
0606             wrmsrl(p4_counters[VIRT_CTR(stag, i)].counter_address,
0607                    -(u64)counter_config[i].count);
0608         } else {
0609             reset_value[i] = 0;
0610         }
0611     }
0612 }
0613 
0614 
0615 static int p4_check_ctrs(struct pt_regs * const regs,
0616              struct op_msrs const * const msrs)
0617 {
0618     unsigned long ctr, low, high, stag, real;
0619     int i;
0620 
0621     stag = get_stagger();
0622 
0623     for (i = 0; i < num_counters; ++i) {
0624 
0625         if (!reset_value[i])
0626             continue;
0627 
0628         /*
0629          * there is some eccentricity in the hardware which
0630          * requires that we perform 2 extra corrections:
0631          *
0632          * - check both the CCCR:OVF flag for overflow and the
0633          *   counter high bit for un-flagged overflows.
0634          *
0635          * - write the counter back twice to ensure it gets
0636          *   updated properly.
0637          *
0638          * the former seems to be related to extra NMIs happening
0639          * during the current NMI; the latter is reported as errata
0640          * N15 in intel doc 249199-029, pentium 4 specification
0641          * update, though their suggested work-around does not
0642          * appear to solve the problem.
0643          */
0644 
0645         real = VIRT_CTR(stag, i);
0646 
0647         rdmsr(p4_counters[real].cccr_address, low, high);
0648         rdmsr(p4_counters[real].counter_address, ctr, high);
0649         if (CCCR_OVF_P(low) || !(ctr & OP_CTR_OVERFLOW)) {
0650             oprofile_add_sample(regs, i);
0651             wrmsrl(p4_counters[real].counter_address,
0652                    -(u64)reset_value[i]);
0653             CCCR_CLEAR_OVF(low);
0654             wrmsr(p4_counters[real].cccr_address, low, high);
0655             wrmsrl(p4_counters[real].counter_address,
0656                    -(u64)reset_value[i]);
0657         }
0658     }
0659 
0660     /* P4 quirk: you have to re-unmask the apic vector */
0661     apic_write(APIC_LVTPC, apic_read(APIC_LVTPC) & ~APIC_LVT_MASKED);
0662 
0663     /* See op_model_ppro.c */
0664     return 1;
0665 }
0666 
0667 
0668 static void p4_start(struct op_msrs const * const msrs)
0669 {
0670     unsigned int low, high, stag;
0671     int i;
0672 
0673     stag = get_stagger();
0674 
0675     for (i = 0; i < num_counters; ++i) {
0676         if (!reset_value[i])
0677             continue;
0678         rdmsr(p4_counters[VIRT_CTR(stag, i)].cccr_address, low, high);
0679         CCCR_SET_ENABLE(low);
0680         wrmsr(p4_counters[VIRT_CTR(stag, i)].cccr_address, low, high);
0681     }
0682 }
0683 
0684 
0685 static void p4_stop(struct op_msrs const * const msrs)
0686 {
0687     unsigned int low, high, stag;
0688     int i;
0689 
0690     stag = get_stagger();
0691 
0692     for (i = 0; i < num_counters; ++i) {
0693         if (!reset_value[i])
0694             continue;
0695         rdmsr(p4_counters[VIRT_CTR(stag, i)].cccr_address, low, high);
0696         CCCR_SET_DISABLE(low);
0697         wrmsr(p4_counters[VIRT_CTR(stag, i)].cccr_address, low, high);
0698     }
0699 }
0700 
0701 #ifdef CONFIG_SMP
0702 struct op_x86_model_spec op_p4_ht2_spec = {
0703     .num_counters       = NUM_COUNTERS_HT2,
0704     .num_controls       = NUM_CONTROLS_HT2,
0705     .fill_in_addresses  = &p4_fill_in_addresses,
0706     .setup_ctrs     = &p4_setup_ctrs,
0707     .check_ctrs     = &p4_check_ctrs,
0708     .start          = &p4_start,
0709     .stop           = &p4_stop,
0710     .shutdown       = &p4_shutdown
0711 };
0712 #endif
0713 
0714 struct op_x86_model_spec op_p4_spec = {
0715     .num_counters       = NUM_COUNTERS_NON_HT,
0716     .num_controls       = NUM_CONTROLS_NON_HT,
0717     .fill_in_addresses  = &p4_fill_in_addresses,
0718     .setup_ctrs     = &p4_setup_ctrs,
0719     .check_ctrs     = &p4_check_ctrs,
0720     .start          = &p4_start,
0721     .stop           = &p4_stop,
0722     .shutdown       = &p4_shutdown
0723 };