Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * This only handles 32bit MTRR on 32bit hosts. This is strictly wrong
0004  * because MTRRs can span up to 40 bits (36bits on most modern x86)
0005  */
0006 
0007 #include <linux/export.h>
0008 #include <linux/init.h>
0009 #include <linux/io.h>
0010 #include <linux/mm.h>
0011 
0012 #include <asm/processor-flags.h>
0013 #include <asm/cpufeature.h>
0014 #include <asm/tlbflush.h>
0015 #include <asm/mtrr.h>
0016 #include <asm/msr.h>
0017 #include <asm/memtype.h>
0018 
0019 #include "mtrr.h"
0020 
0021 struct fixed_range_block {
0022     int base_msr;       /* start address of an MTRR block */
0023     int ranges;     /* number of MTRRs in this block  */
0024 };
0025 
0026 static struct fixed_range_block fixed_range_blocks[] = {
0027     { MSR_MTRRfix64K_00000, 1 }, /* one   64k MTRR  */
0028     { MSR_MTRRfix16K_80000, 2 }, /* two   16k MTRRs */
0029     { MSR_MTRRfix4K_C0000,  8 }, /* eight  4k MTRRs */
0030     {}
0031 };
0032 
0033 static unsigned long smp_changes_mask;
0034 static int mtrr_state_set;
0035 u64 mtrr_tom2;
0036 
0037 struct mtrr_state_type mtrr_state;
0038 EXPORT_SYMBOL_GPL(mtrr_state);
0039 
0040 /*
0041  * BIOS is expected to clear MtrrFixDramModEn bit, see for example
0042  * "BIOS and Kernel Developer's Guide for the AMD Athlon 64 and AMD
0043  * Opteron Processors" (26094 Rev. 3.30 February 2006), section
0044  * "13.2.1.2 SYSCFG Register": "The MtrrFixDramModEn bit should be set
0045  * to 1 during BIOS initialization of the fixed MTRRs, then cleared to
0046  * 0 for operation."
0047  */
0048 static inline void k8_check_syscfg_dram_mod_en(void)
0049 {
0050     u32 lo, hi;
0051 
0052     if (!((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) &&
0053           (boot_cpu_data.x86 >= 0x0f)))
0054         return;
0055 
0056     rdmsr(MSR_AMD64_SYSCFG, lo, hi);
0057     if (lo & K8_MTRRFIXRANGE_DRAM_MODIFY) {
0058         pr_err(FW_WARN "MTRR: CPU %u: SYSCFG[MtrrFixDramModEn]"
0059                " not cleared by BIOS, clearing this bit\n",
0060                smp_processor_id());
0061         lo &= ~K8_MTRRFIXRANGE_DRAM_MODIFY;
0062         mtrr_wrmsr(MSR_AMD64_SYSCFG, lo, hi);
0063     }
0064 }
0065 
0066 /* Get the size of contiguous MTRR range */
0067 static u64 get_mtrr_size(u64 mask)
0068 {
0069     u64 size;
0070 
0071     mask >>= PAGE_SHIFT;
0072     mask |= size_or_mask;
0073     size = -mask;
0074     size <<= PAGE_SHIFT;
0075     return size;
0076 }
0077 
0078 /*
0079  * Check and return the effective type for MTRR-MTRR type overlap.
0080  * Returns 1 if the effective type is UNCACHEABLE, else returns 0
0081  */
0082 static int check_type_overlap(u8 *prev, u8 *curr)
0083 {
0084     if (*prev == MTRR_TYPE_UNCACHABLE || *curr == MTRR_TYPE_UNCACHABLE) {
0085         *prev = MTRR_TYPE_UNCACHABLE;
0086         *curr = MTRR_TYPE_UNCACHABLE;
0087         return 1;
0088     }
0089 
0090     if ((*prev == MTRR_TYPE_WRBACK && *curr == MTRR_TYPE_WRTHROUGH) ||
0091         (*prev == MTRR_TYPE_WRTHROUGH && *curr == MTRR_TYPE_WRBACK)) {
0092         *prev = MTRR_TYPE_WRTHROUGH;
0093         *curr = MTRR_TYPE_WRTHROUGH;
0094     }
0095 
0096     if (*prev != *curr) {
0097         *prev = MTRR_TYPE_UNCACHABLE;
0098         *curr = MTRR_TYPE_UNCACHABLE;
0099         return 1;
0100     }
0101 
0102     return 0;
0103 }
0104 
0105 /**
0106  * mtrr_type_lookup_fixed - look up memory type in MTRR fixed entries
0107  *
0108  * Return the MTRR fixed memory type of 'start'.
0109  *
0110  * MTRR fixed entries are divided into the following ways:
0111  *  0x00000 - 0x7FFFF : This range is divided into eight 64KB sub-ranges
0112  *  0x80000 - 0xBFFFF : This range is divided into sixteen 16KB sub-ranges
0113  *  0xC0000 - 0xFFFFF : This range is divided into sixty-four 4KB sub-ranges
0114  *
0115  * Return Values:
0116  * MTRR_TYPE_(type)  - Matched memory type
0117  * MTRR_TYPE_INVALID - Unmatched
0118  */
0119 static u8 mtrr_type_lookup_fixed(u64 start, u64 end)
0120 {
0121     int idx;
0122 
0123     if (start >= 0x100000)
0124         return MTRR_TYPE_INVALID;
0125 
0126     /* 0x0 - 0x7FFFF */
0127     if (start < 0x80000) {
0128         idx = 0;
0129         idx += (start >> 16);
0130         return mtrr_state.fixed_ranges[idx];
0131     /* 0x80000 - 0xBFFFF */
0132     } else if (start < 0xC0000) {
0133         idx = 1 * 8;
0134         idx += ((start - 0x80000) >> 14);
0135         return mtrr_state.fixed_ranges[idx];
0136     }
0137 
0138     /* 0xC0000 - 0xFFFFF */
0139     idx = 3 * 8;
0140     idx += ((start - 0xC0000) >> 12);
0141     return mtrr_state.fixed_ranges[idx];
0142 }
0143 
0144 /**
0145  * mtrr_type_lookup_variable - look up memory type in MTRR variable entries
0146  *
0147  * Return Value:
0148  * MTRR_TYPE_(type) - Matched memory type or default memory type (unmatched)
0149  *
0150  * Output Arguments:
0151  * repeat - Set to 1 when [start:end] spanned across MTRR range and type
0152  *      returned corresponds only to [start:*partial_end].  Caller has
0153  *      to lookup again for [*partial_end:end].
0154  *
0155  * uniform - Set to 1 when an MTRR covers the region uniformly, i.e. the
0156  *       region is fully covered by a single MTRR entry or the default
0157  *       type.
0158  */
0159 static u8 mtrr_type_lookup_variable(u64 start, u64 end, u64 *partial_end,
0160                     int *repeat, u8 *uniform)
0161 {
0162     int i;
0163     u64 base, mask;
0164     u8 prev_match, curr_match;
0165 
0166     *repeat = 0;
0167     *uniform = 1;
0168 
0169     prev_match = MTRR_TYPE_INVALID;
0170     for (i = 0; i < num_var_ranges; ++i) {
0171         unsigned short start_state, end_state, inclusive;
0172 
0173         if (!(mtrr_state.var_ranges[i].mask_lo & (1 << 11)))
0174             continue;
0175 
0176         base = (((u64)mtrr_state.var_ranges[i].base_hi) << 32) +
0177                (mtrr_state.var_ranges[i].base_lo & PAGE_MASK);
0178         mask = (((u64)mtrr_state.var_ranges[i].mask_hi) << 32) +
0179                (mtrr_state.var_ranges[i].mask_lo & PAGE_MASK);
0180 
0181         start_state = ((start & mask) == (base & mask));
0182         end_state = ((end & mask) == (base & mask));
0183         inclusive = ((start < base) && (end > base));
0184 
0185         if ((start_state != end_state) || inclusive) {
0186             /*
0187              * We have start:end spanning across an MTRR.
0188              * We split the region into either
0189              *
0190              * - start_state:1
0191              * (start:mtrr_end)(mtrr_end:end)
0192              * - end_state:1
0193              * (start:mtrr_start)(mtrr_start:end)
0194              * - inclusive:1
0195              * (start:mtrr_start)(mtrr_start:mtrr_end)(mtrr_end:end)
0196              *
0197              * depending on kind of overlap.
0198              *
0199              * Return the type of the first region and a pointer
0200              * to the start of next region so that caller will be
0201              * advised to lookup again after having adjusted start
0202              * and end.
0203              *
0204              * Note: This way we handle overlaps with multiple
0205              * entries and the default type properly.
0206              */
0207             if (start_state)
0208                 *partial_end = base + get_mtrr_size(mask);
0209             else
0210                 *partial_end = base;
0211 
0212             if (unlikely(*partial_end <= start)) {
0213                 WARN_ON(1);
0214                 *partial_end = start + PAGE_SIZE;
0215             }
0216 
0217             end = *partial_end - 1; /* end is inclusive */
0218             *repeat = 1;
0219             *uniform = 0;
0220         }
0221 
0222         if ((start & mask) != (base & mask))
0223             continue;
0224 
0225         curr_match = mtrr_state.var_ranges[i].base_lo & 0xff;
0226         if (prev_match == MTRR_TYPE_INVALID) {
0227             prev_match = curr_match;
0228             continue;
0229         }
0230 
0231         *uniform = 0;
0232         if (check_type_overlap(&prev_match, &curr_match))
0233             return curr_match;
0234     }
0235 
0236     if (prev_match != MTRR_TYPE_INVALID)
0237         return prev_match;
0238 
0239     return mtrr_state.def_type;
0240 }
0241 
0242 /**
0243  * mtrr_type_lookup - look up memory type in MTRR
0244  *
0245  * Return Values:
0246  * MTRR_TYPE_(type)  - The effective MTRR type for the region
0247  * MTRR_TYPE_INVALID - MTRR is disabled
0248  *
0249  * Output Argument:
0250  * uniform - Set to 1 when an MTRR covers the region uniformly, i.e. the
0251  *       region is fully covered by a single MTRR entry or the default
0252  *       type.
0253  */
0254 u8 mtrr_type_lookup(u64 start, u64 end, u8 *uniform)
0255 {
0256     u8 type, prev_type, is_uniform = 1, dummy;
0257     int repeat;
0258     u64 partial_end;
0259 
0260     /* Make end inclusive instead of exclusive */
0261     end--;
0262 
0263     if (!mtrr_state_set)
0264         return MTRR_TYPE_INVALID;
0265 
0266     if (!(mtrr_state.enabled & MTRR_STATE_MTRR_ENABLED))
0267         return MTRR_TYPE_INVALID;
0268 
0269     /*
0270      * Look up the fixed ranges first, which take priority over
0271      * the variable ranges.
0272      */
0273     if ((start < 0x100000) &&
0274         (mtrr_state.have_fixed) &&
0275         (mtrr_state.enabled & MTRR_STATE_MTRR_FIXED_ENABLED)) {
0276         is_uniform = 0;
0277         type = mtrr_type_lookup_fixed(start, end);
0278         goto out;
0279     }
0280 
0281     /*
0282      * Look up the variable ranges.  Look of multiple ranges matching
0283      * this address and pick type as per MTRR precedence.
0284      */
0285     type = mtrr_type_lookup_variable(start, end, &partial_end,
0286                      &repeat, &is_uniform);
0287 
0288     /*
0289      * Common path is with repeat = 0.
0290      * However, we can have cases where [start:end] spans across some
0291      * MTRR ranges and/or the default type.  Do repeated lookups for
0292      * that case here.
0293      */
0294     while (repeat) {
0295         prev_type = type;
0296         start = partial_end;
0297         is_uniform = 0;
0298         type = mtrr_type_lookup_variable(start, end, &partial_end,
0299                          &repeat, &dummy);
0300 
0301         if (check_type_overlap(&prev_type, &type))
0302             goto out;
0303     }
0304 
0305     if (mtrr_tom2 && (start >= (1ULL<<32)) && (end < mtrr_tom2))
0306         type = MTRR_TYPE_WRBACK;
0307 
0308 out:
0309     *uniform = is_uniform;
0310     return type;
0311 }
0312 
0313 /* Get the MSR pair relating to a var range */
0314 static void
0315 get_mtrr_var_range(unsigned int index, struct mtrr_var_range *vr)
0316 {
0317     rdmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi);
0318     rdmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi);
0319 }
0320 
0321 /* Fill the MSR pair relating to a var range */
0322 void fill_mtrr_var_range(unsigned int index,
0323         u32 base_lo, u32 base_hi, u32 mask_lo, u32 mask_hi)
0324 {
0325     struct mtrr_var_range *vr;
0326 
0327     vr = mtrr_state.var_ranges;
0328 
0329     vr[index].base_lo = base_lo;
0330     vr[index].base_hi = base_hi;
0331     vr[index].mask_lo = mask_lo;
0332     vr[index].mask_hi = mask_hi;
0333 }
0334 
0335 static void get_fixed_ranges(mtrr_type *frs)
0336 {
0337     unsigned int *p = (unsigned int *)frs;
0338     int i;
0339 
0340     k8_check_syscfg_dram_mod_en();
0341 
0342     rdmsr(MSR_MTRRfix64K_00000, p[0], p[1]);
0343 
0344     for (i = 0; i < 2; i++)
0345         rdmsr(MSR_MTRRfix16K_80000 + i, p[2 + i * 2], p[3 + i * 2]);
0346     for (i = 0; i < 8; i++)
0347         rdmsr(MSR_MTRRfix4K_C0000 + i, p[6 + i * 2], p[7 + i * 2]);
0348 }
0349 
0350 void mtrr_save_fixed_ranges(void *info)
0351 {
0352     if (boot_cpu_has(X86_FEATURE_MTRR))
0353         get_fixed_ranges(mtrr_state.fixed_ranges);
0354 }
0355 
0356 static unsigned __initdata last_fixed_start;
0357 static unsigned __initdata last_fixed_end;
0358 static mtrr_type __initdata last_fixed_type;
0359 
0360 static void __init print_fixed_last(void)
0361 {
0362     if (!last_fixed_end)
0363         return;
0364 
0365     pr_debug("  %05X-%05X %s\n", last_fixed_start,
0366          last_fixed_end - 1, mtrr_attrib_to_str(last_fixed_type));
0367 
0368     last_fixed_end = 0;
0369 }
0370 
0371 static void __init update_fixed_last(unsigned base, unsigned end,
0372                      mtrr_type type)
0373 {
0374     last_fixed_start = base;
0375     last_fixed_end = end;
0376     last_fixed_type = type;
0377 }
0378 
0379 static void __init
0380 print_fixed(unsigned base, unsigned step, const mtrr_type *types)
0381 {
0382     unsigned i;
0383 
0384     for (i = 0; i < 8; ++i, ++types, base += step) {
0385         if (last_fixed_end == 0) {
0386             update_fixed_last(base, base + step, *types);
0387             continue;
0388         }
0389         if (last_fixed_end == base && last_fixed_type == *types) {
0390             last_fixed_end = base + step;
0391             continue;
0392         }
0393         /* new segments: gap or different type */
0394         print_fixed_last();
0395         update_fixed_last(base, base + step, *types);
0396     }
0397 }
0398 
0399 static void prepare_set(void);
0400 static void post_set(void);
0401 
0402 static void __init print_mtrr_state(void)
0403 {
0404     unsigned int i;
0405     int high_width;
0406 
0407     pr_debug("MTRR default type: %s\n",
0408          mtrr_attrib_to_str(mtrr_state.def_type));
0409     if (mtrr_state.have_fixed) {
0410         pr_debug("MTRR fixed ranges %sabled:\n",
0411             ((mtrr_state.enabled & MTRR_STATE_MTRR_ENABLED) &&
0412              (mtrr_state.enabled & MTRR_STATE_MTRR_FIXED_ENABLED)) ?
0413              "en" : "dis");
0414         print_fixed(0x00000, 0x10000, mtrr_state.fixed_ranges + 0);
0415         for (i = 0; i < 2; ++i)
0416             print_fixed(0x80000 + i * 0x20000, 0x04000,
0417                     mtrr_state.fixed_ranges + (i + 1) * 8);
0418         for (i = 0; i < 8; ++i)
0419             print_fixed(0xC0000 + i * 0x08000, 0x01000,
0420                     mtrr_state.fixed_ranges + (i + 3) * 8);
0421 
0422         /* tail */
0423         print_fixed_last();
0424     }
0425     pr_debug("MTRR variable ranges %sabled:\n",
0426          mtrr_state.enabled & MTRR_STATE_MTRR_ENABLED ? "en" : "dis");
0427     high_width = (__ffs64(size_or_mask) - (32 - PAGE_SHIFT) + 3) / 4;
0428 
0429     for (i = 0; i < num_var_ranges; ++i) {
0430         if (mtrr_state.var_ranges[i].mask_lo & (1 << 11))
0431             pr_debug("  %u base %0*X%05X000 mask %0*X%05X000 %s\n",
0432                  i,
0433                  high_width,
0434                  mtrr_state.var_ranges[i].base_hi,
0435                  mtrr_state.var_ranges[i].base_lo >> 12,
0436                  high_width,
0437                  mtrr_state.var_ranges[i].mask_hi,
0438                  mtrr_state.var_ranges[i].mask_lo >> 12,
0439                  mtrr_attrib_to_str(mtrr_state.var_ranges[i].base_lo & 0xff));
0440         else
0441             pr_debug("  %u disabled\n", i);
0442     }
0443     if (mtrr_tom2)
0444         pr_debug("TOM2: %016llx aka %lldM\n", mtrr_tom2, mtrr_tom2>>20);
0445 }
0446 
0447 /* PAT setup for BP. We need to go through sync steps here */
0448 void __init mtrr_bp_pat_init(void)
0449 {
0450     unsigned long flags;
0451 
0452     local_irq_save(flags);
0453     prepare_set();
0454 
0455     pat_init();
0456 
0457     post_set();
0458     local_irq_restore(flags);
0459 }
0460 
0461 /* Grab all of the MTRR state for this CPU into *state */
0462 bool __init get_mtrr_state(void)
0463 {
0464     struct mtrr_var_range *vrs;
0465     unsigned lo, dummy;
0466     unsigned int i;
0467 
0468     vrs = mtrr_state.var_ranges;
0469 
0470     rdmsr(MSR_MTRRcap, lo, dummy);
0471     mtrr_state.have_fixed = (lo >> 8) & 1;
0472 
0473     for (i = 0; i < num_var_ranges; i++)
0474         get_mtrr_var_range(i, &vrs[i]);
0475     if (mtrr_state.have_fixed)
0476         get_fixed_ranges(mtrr_state.fixed_ranges);
0477 
0478     rdmsr(MSR_MTRRdefType, lo, dummy);
0479     mtrr_state.def_type = (lo & 0xff);
0480     mtrr_state.enabled = (lo & 0xc00) >> 10;
0481 
0482     if (amd_special_default_mtrr()) {
0483         unsigned low, high;
0484 
0485         /* TOP_MEM2 */
0486         rdmsr(MSR_K8_TOP_MEM2, low, high);
0487         mtrr_tom2 = high;
0488         mtrr_tom2 <<= 32;
0489         mtrr_tom2 |= low;
0490         mtrr_tom2 &= 0xffffff800000ULL;
0491     }
0492 
0493     print_mtrr_state();
0494 
0495     mtrr_state_set = 1;
0496 
0497     return !!(mtrr_state.enabled & MTRR_STATE_MTRR_ENABLED);
0498 }
0499 
0500 /* Some BIOS's are messed up and don't set all MTRRs the same! */
0501 void __init mtrr_state_warn(void)
0502 {
0503     unsigned long mask = smp_changes_mask;
0504 
0505     if (!mask)
0506         return;
0507     if (mask & MTRR_CHANGE_MASK_FIXED)
0508         pr_warn("mtrr: your CPUs had inconsistent fixed MTRR settings\n");
0509     if (mask & MTRR_CHANGE_MASK_VARIABLE)
0510         pr_warn("mtrr: your CPUs had inconsistent variable MTRR settings\n");
0511     if (mask & MTRR_CHANGE_MASK_DEFTYPE)
0512         pr_warn("mtrr: your CPUs had inconsistent MTRRdefType settings\n");
0513 
0514     pr_info("mtrr: probably your BIOS does not setup all CPUs.\n");
0515     pr_info("mtrr: corrected configuration.\n");
0516 }
0517 
0518 /*
0519  * Doesn't attempt to pass an error out to MTRR users
0520  * because it's quite complicated in some cases and probably not
0521  * worth it because the best error handling is to ignore it.
0522  */
0523 void mtrr_wrmsr(unsigned msr, unsigned a, unsigned b)
0524 {
0525     if (wrmsr_safe(msr, a, b) < 0) {
0526         pr_err("MTRR: CPU %u: Writing MSR %x to %x:%x failed\n",
0527             smp_processor_id(), msr, a, b);
0528     }
0529 }
0530 
0531 /**
0532  * set_fixed_range - checks & updates a fixed-range MTRR if it
0533  *           differs from the value it should have
0534  * @msr: MSR address of the MTTR which should be checked and updated
0535  * @changed: pointer which indicates whether the MTRR needed to be changed
0536  * @msrwords: pointer to the MSR values which the MSR should have
0537  */
0538 static void set_fixed_range(int msr, bool *changed, unsigned int *msrwords)
0539 {
0540     unsigned lo, hi;
0541 
0542     rdmsr(msr, lo, hi);
0543 
0544     if (lo != msrwords[0] || hi != msrwords[1]) {
0545         mtrr_wrmsr(msr, msrwords[0], msrwords[1]);
0546         *changed = true;
0547     }
0548 }
0549 
0550 /**
0551  * generic_get_free_region - Get a free MTRR.
0552  * @base: The starting (base) address of the region.
0553  * @size: The size (in bytes) of the region.
0554  * @replace_reg: mtrr index to be replaced; set to invalid value if none.
0555  *
0556  * Returns: The index of the region on success, else negative on error.
0557  */
0558 int
0559 generic_get_free_region(unsigned long base, unsigned long size, int replace_reg)
0560 {
0561     unsigned long lbase, lsize;
0562     mtrr_type ltype;
0563     int i, max;
0564 
0565     max = num_var_ranges;
0566     if (replace_reg >= 0 && replace_reg < max)
0567         return replace_reg;
0568 
0569     for (i = 0; i < max; ++i) {
0570         mtrr_if->get(i, &lbase, &lsize, &ltype);
0571         if (lsize == 0)
0572             return i;
0573     }
0574 
0575     return -ENOSPC;
0576 }
0577 
0578 static void generic_get_mtrr(unsigned int reg, unsigned long *base,
0579                  unsigned long *size, mtrr_type *type)
0580 {
0581     u32 mask_lo, mask_hi, base_lo, base_hi;
0582     unsigned int hi;
0583     u64 tmp, mask;
0584 
0585     /*
0586      * get_mtrr doesn't need to update mtrr_state, also it could be called
0587      * from any cpu, so try to print it out directly.
0588      */
0589     get_cpu();
0590 
0591     rdmsr(MTRRphysMask_MSR(reg), mask_lo, mask_hi);
0592 
0593     if ((mask_lo & 0x800) == 0) {
0594         /*  Invalid (i.e. free) range */
0595         *base = 0;
0596         *size = 0;
0597         *type = 0;
0598         goto out_put_cpu;
0599     }
0600 
0601     rdmsr(MTRRphysBase_MSR(reg), base_lo, base_hi);
0602 
0603     /* Work out the shifted address mask: */
0604     tmp = (u64)mask_hi << (32 - PAGE_SHIFT) | mask_lo >> PAGE_SHIFT;
0605     mask = size_or_mask | tmp;
0606 
0607     /* Expand tmp with high bits to all 1s: */
0608     hi = fls64(tmp);
0609     if (hi > 0) {
0610         tmp |= ~((1ULL<<(hi - 1)) - 1);
0611 
0612         if (tmp != mask) {
0613             pr_warn("mtrr: your BIOS has configured an incorrect mask, fixing it.\n");
0614             add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
0615             mask = tmp;
0616         }
0617     }
0618 
0619     /*
0620      * This works correctly if size is a power of two, i.e. a
0621      * contiguous range:
0622      */
0623     *size = -mask;
0624     *base = (u64)base_hi << (32 - PAGE_SHIFT) | base_lo >> PAGE_SHIFT;
0625     *type = base_lo & 0xff;
0626 
0627 out_put_cpu:
0628     put_cpu();
0629 }
0630 
0631 /**
0632  * set_fixed_ranges - checks & updates the fixed-range MTRRs if they
0633  *            differ from the saved set
0634  * @frs: pointer to fixed-range MTRR values, saved by get_fixed_ranges()
0635  */
0636 static int set_fixed_ranges(mtrr_type *frs)
0637 {
0638     unsigned long long *saved = (unsigned long long *)frs;
0639     bool changed = false;
0640     int block = -1, range;
0641 
0642     k8_check_syscfg_dram_mod_en();
0643 
0644     while (fixed_range_blocks[++block].ranges) {
0645         for (range = 0; range < fixed_range_blocks[block].ranges; range++)
0646             set_fixed_range(fixed_range_blocks[block].base_msr + range,
0647                     &changed, (unsigned int *)saved++);
0648     }
0649 
0650     return changed;
0651 }
0652 
0653 /*
0654  * Set the MSR pair relating to a var range.
0655  * Returns true if changes are made.
0656  */
0657 static bool set_mtrr_var_ranges(unsigned int index, struct mtrr_var_range *vr)
0658 {
0659     unsigned int lo, hi;
0660     bool changed = false;
0661 
0662     rdmsr(MTRRphysBase_MSR(index), lo, hi);
0663     if ((vr->base_lo & 0xfffff0ffUL) != (lo & 0xfffff0ffUL)
0664         || (vr->base_hi & (size_and_mask >> (32 - PAGE_SHIFT))) !=
0665         (hi & (size_and_mask >> (32 - PAGE_SHIFT)))) {
0666 
0667         mtrr_wrmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi);
0668         changed = true;
0669     }
0670 
0671     rdmsr(MTRRphysMask_MSR(index), lo, hi);
0672 
0673     if ((vr->mask_lo & 0xfffff800UL) != (lo & 0xfffff800UL)
0674         || (vr->mask_hi & (size_and_mask >> (32 - PAGE_SHIFT))) !=
0675         (hi & (size_and_mask >> (32 - PAGE_SHIFT)))) {
0676         mtrr_wrmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi);
0677         changed = true;
0678     }
0679     return changed;
0680 }
0681 
0682 static u32 deftype_lo, deftype_hi;
0683 
0684 /**
0685  * set_mtrr_state - Set the MTRR state for this CPU.
0686  *
0687  * NOTE: The CPU must already be in a safe state for MTRR changes.
0688  * RETURNS: 0 if no changes made, else a mask indicating what was changed.
0689  */
0690 static unsigned long set_mtrr_state(void)
0691 {
0692     unsigned long change_mask = 0;
0693     unsigned int i;
0694 
0695     for (i = 0; i < num_var_ranges; i++) {
0696         if (set_mtrr_var_ranges(i, &mtrr_state.var_ranges[i]))
0697             change_mask |= MTRR_CHANGE_MASK_VARIABLE;
0698     }
0699 
0700     if (mtrr_state.have_fixed && set_fixed_ranges(mtrr_state.fixed_ranges))
0701         change_mask |= MTRR_CHANGE_MASK_FIXED;
0702 
0703     /*
0704      * Set_mtrr_restore restores the old value of MTRRdefType,
0705      * so to set it we fiddle with the saved value:
0706      */
0707     if ((deftype_lo & 0xff) != mtrr_state.def_type
0708         || ((deftype_lo & 0xc00) >> 10) != mtrr_state.enabled) {
0709 
0710         deftype_lo = (deftype_lo & ~0xcff) | mtrr_state.def_type |
0711                  (mtrr_state.enabled << 10);
0712         change_mask |= MTRR_CHANGE_MASK_DEFTYPE;
0713     }
0714 
0715     return change_mask;
0716 }
0717 
0718 
0719 static unsigned long cr4;
0720 static DEFINE_RAW_SPINLOCK(set_atomicity_lock);
0721 
0722 /*
0723  * Since we are disabling the cache don't allow any interrupts,
0724  * they would run extremely slow and would only increase the pain.
0725  *
0726  * The caller must ensure that local interrupts are disabled and
0727  * are reenabled after post_set() has been called.
0728  */
0729 static void prepare_set(void) __acquires(set_atomicity_lock)
0730 {
0731     unsigned long cr0;
0732 
0733     /*
0734      * Note that this is not ideal
0735      * since the cache is only flushed/disabled for this CPU while the
0736      * MTRRs are changed, but changing this requires more invasive
0737      * changes to the way the kernel boots
0738      */
0739 
0740     raw_spin_lock(&set_atomicity_lock);
0741 
0742     /* Enter the no-fill (CD=1, NW=0) cache mode and flush caches. */
0743     cr0 = read_cr0() | X86_CR0_CD;
0744     write_cr0(cr0);
0745 
0746     /*
0747      * Cache flushing is the most time-consuming step when programming
0748      * the MTRRs. Fortunately, as per the Intel Software Development
0749      * Manual, we can skip it if the processor supports cache self-
0750      * snooping.
0751      */
0752     if (!static_cpu_has(X86_FEATURE_SELFSNOOP))
0753         wbinvd();
0754 
0755     /* Save value of CR4 and clear Page Global Enable (bit 7) */
0756     if (boot_cpu_has(X86_FEATURE_PGE)) {
0757         cr4 = __read_cr4();
0758         __write_cr4(cr4 & ~X86_CR4_PGE);
0759     }
0760 
0761     /* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */
0762     count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
0763     flush_tlb_local();
0764 
0765     /* Save MTRR state */
0766     rdmsr(MSR_MTRRdefType, deftype_lo, deftype_hi);
0767 
0768     /* Disable MTRRs, and set the default type to uncached */
0769     mtrr_wrmsr(MSR_MTRRdefType, deftype_lo & ~0xcff, deftype_hi);
0770 
0771     /* Again, only flush caches if we have to. */
0772     if (!static_cpu_has(X86_FEATURE_SELFSNOOP))
0773         wbinvd();
0774 }
0775 
0776 static void post_set(void) __releases(set_atomicity_lock)
0777 {
0778     /* Flush TLBs (no need to flush caches - they are disabled) */
0779     count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
0780     flush_tlb_local();
0781 
0782     /* Intel (P6) standard MTRRs */
0783     mtrr_wrmsr(MSR_MTRRdefType, deftype_lo, deftype_hi);
0784 
0785     /* Enable caches */
0786     write_cr0(read_cr0() & ~X86_CR0_CD);
0787 
0788     /* Restore value of CR4 */
0789     if (boot_cpu_has(X86_FEATURE_PGE))
0790         __write_cr4(cr4);
0791     raw_spin_unlock(&set_atomicity_lock);
0792 }
0793 
0794 static void generic_set_all(void)
0795 {
0796     unsigned long mask, count;
0797     unsigned long flags;
0798 
0799     local_irq_save(flags);
0800     prepare_set();
0801 
0802     /* Actually set the state */
0803     mask = set_mtrr_state();
0804 
0805     /* also set PAT */
0806     pat_init();
0807 
0808     post_set();
0809     local_irq_restore(flags);
0810 
0811     /* Use the atomic bitops to update the global mask */
0812     for (count = 0; count < sizeof(mask) * 8; ++count) {
0813         if (mask & 0x01)
0814             set_bit(count, &smp_changes_mask);
0815         mask >>= 1;
0816     }
0817 
0818 }
0819 
0820 /**
0821  * generic_set_mtrr - set variable MTRR register on the local CPU.
0822  *
0823  * @reg: The register to set.
0824  * @base: The base address of the region.
0825  * @size: The size of the region. If this is 0 the region is disabled.
0826  * @type: The type of the region.
0827  *
0828  * Returns nothing.
0829  */
0830 static void generic_set_mtrr(unsigned int reg, unsigned long base,
0831                  unsigned long size, mtrr_type type)
0832 {
0833     unsigned long flags;
0834     struct mtrr_var_range *vr;
0835 
0836     vr = &mtrr_state.var_ranges[reg];
0837 
0838     local_irq_save(flags);
0839     prepare_set();
0840 
0841     if (size == 0) {
0842         /*
0843          * The invalid bit is kept in the mask, so we simply
0844          * clear the relevant mask register to disable a range.
0845          */
0846         mtrr_wrmsr(MTRRphysMask_MSR(reg), 0, 0);
0847         memset(vr, 0, sizeof(struct mtrr_var_range));
0848     } else {
0849         vr->base_lo = base << PAGE_SHIFT | type;
0850         vr->base_hi = (base & size_and_mask) >> (32 - PAGE_SHIFT);
0851         vr->mask_lo = -size << PAGE_SHIFT | 0x800;
0852         vr->mask_hi = (-size & size_and_mask) >> (32 - PAGE_SHIFT);
0853 
0854         mtrr_wrmsr(MTRRphysBase_MSR(reg), vr->base_lo, vr->base_hi);
0855         mtrr_wrmsr(MTRRphysMask_MSR(reg), vr->mask_lo, vr->mask_hi);
0856     }
0857 
0858     post_set();
0859     local_irq_restore(flags);
0860 }
0861 
0862 int generic_validate_add_page(unsigned long base, unsigned long size,
0863                   unsigned int type)
0864 {
0865     unsigned long lbase, last;
0866 
0867     /*
0868      * For Intel PPro stepping <= 7
0869      * must be 4 MiB aligned and not touch 0x70000000 -> 0x7003FFFF
0870      */
0871     if (is_cpu(INTEL) && boot_cpu_data.x86 == 6 &&
0872         boot_cpu_data.x86_model == 1 &&
0873         boot_cpu_data.x86_stepping <= 7) {
0874         if (base & ((1 << (22 - PAGE_SHIFT)) - 1)) {
0875             pr_warn("mtrr: base(0x%lx000) is not 4 MiB aligned\n", base);
0876             return -EINVAL;
0877         }
0878         if (!(base + size < 0x70000 || base > 0x7003F) &&
0879             (type == MTRR_TYPE_WRCOMB
0880              || type == MTRR_TYPE_WRBACK)) {
0881             pr_warn("mtrr: writable mtrr between 0x70000000 and 0x7003FFFF may hang the CPU.\n");
0882             return -EINVAL;
0883         }
0884     }
0885 
0886     /*
0887      * Check upper bits of base and last are equal and lower bits are 0
0888      * for base and 1 for last
0889      */
0890     last = base + size - 1;
0891     for (lbase = base; !(lbase & 1) && (last & 1);
0892          lbase = lbase >> 1, last = last >> 1)
0893         ;
0894     if (lbase != last) {
0895         pr_warn("mtrr: base(0x%lx000) is not aligned on a size(0x%lx000) boundary\n", base, size);
0896         return -EINVAL;
0897     }
0898     return 0;
0899 }
0900 
0901 static int generic_have_wrcomb(void)
0902 {
0903     unsigned long config, dummy;
0904     rdmsr(MSR_MTRRcap, config, dummy);
0905     return config & (1 << 10);
0906 }
0907 
0908 int positive_have_wrcomb(void)
0909 {
0910     return 1;
0911 }
0912 
0913 /*
0914  * Generic structure...
0915  */
0916 const struct mtrr_ops generic_mtrr_ops = {
0917     .use_intel_if       = 1,
0918     .set_all        = generic_set_all,
0919     .get            = generic_get_mtrr,
0920     .get_free_region    = generic_get_free_region,
0921     .set            = generic_set_mtrr,
0922     .validate_add_page  = generic_validate_add_page,
0923     .have_wrcomb        = generic_have_wrcomb,
0924 };
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!