Back to home page

OSCL-LXR
 
 

     

0001 /*
0002  * MTRR (Memory Type Range Register) cleanup
0003  *
0004  *  Copyright (C) 2009 Yinghai Lu
0005  *
0006  * This library is free software; you can redistribute it and/or
0007  * modify it under the terms of the GNU Library General Public
0008  * License as published by the Free Software Foundation; either
0009  * version 2 of the License, or (at your option) any later version.
0010  *
0011  * This library is distributed in the hope that it will be useful,
0012  * but WITHOUT ANY WARRANTY; without even the implied warranty of
0013  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
0014  * Library General Public License for more details.
0015  *
0016  * You should have received a copy of the GNU Library General Public
0017  * License along with this library; if not, write to the Free
0018  * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
0019  */
0020 #include <linux/init.h>
0021 #include <linux/pci.h>
0022 #include <linux/smp.h>
0023 #include <linux/cpu.h>
0024 #include <linux/mutex.h>
0025 #include <linux/uaccess.h>
0026 #include <linux/kvm_para.h>
0027 #include <linux/range.h>
0028 
0029 #include <asm/processor.h>
0030 #include <asm/e820/api.h>
0031 #include <asm/mtrr.h>
0032 #include <asm/msr.h>
0033 
0034 #include "mtrr.h"
0035 
0036 struct var_mtrr_range_state {
0037     unsigned long   base_pfn;
0038     unsigned long   size_pfn;
0039     mtrr_type   type;
0040 };
0041 
0042 struct var_mtrr_state {
0043     unsigned long   range_startk;
0044     unsigned long   range_sizek;
0045     unsigned long   chunk_sizek;
0046     unsigned long   gran_sizek;
0047     unsigned int    reg;
0048 };
0049 
0050 /* Should be related to MTRR_VAR_RANGES nums */
0051 #define RANGE_NUM               256
0052 
0053 static struct range __initdata      range[RANGE_NUM];
0054 static int __initdata               nr_range;
0055 
0056 static struct var_mtrr_range_state __initdata   range_state[RANGE_NUM];
0057 
0058 static int __initdata debug_print;
0059 #define Dprintk(x...) do { if (debug_print) pr_debug(x); } while (0)
0060 
0061 #define BIOS_BUG_MSG \
0062     "WARNING: BIOS bug: VAR MTRR %d contains strange UC entry under 1M, check with your system vendor!\n"
0063 
0064 static int __init
0065 x86_get_mtrr_mem_range(struct range *range, int nr_range,
0066                unsigned long extra_remove_base,
0067                unsigned long extra_remove_size)
0068 {
0069     unsigned long base, size;
0070     mtrr_type type;
0071     int i;
0072 
0073     for (i = 0; i < num_var_ranges; i++) {
0074         type = range_state[i].type;
0075         if (type != MTRR_TYPE_WRBACK)
0076             continue;
0077         base = range_state[i].base_pfn;
0078         size = range_state[i].size_pfn;
0079         nr_range = add_range_with_merge(range, RANGE_NUM, nr_range,
0080                         base, base + size);
0081     }
0082     if (debug_print) {
0083         pr_debug("After WB checking\n");
0084         for (i = 0; i < nr_range; i++)
0085             pr_debug("MTRR MAP PFN: %016llx - %016llx\n",
0086                  range[i].start, range[i].end);
0087     }
0088 
0089     /* Take out UC ranges: */
0090     for (i = 0; i < num_var_ranges; i++) {
0091         type = range_state[i].type;
0092         if (type != MTRR_TYPE_UNCACHABLE &&
0093             type != MTRR_TYPE_WRPROT)
0094             continue;
0095         size = range_state[i].size_pfn;
0096         if (!size)
0097             continue;
0098         base = range_state[i].base_pfn;
0099         if (base < (1<<(20-PAGE_SHIFT)) && mtrr_state.have_fixed &&
0100             (mtrr_state.enabled & MTRR_STATE_MTRR_ENABLED) &&
0101             (mtrr_state.enabled & MTRR_STATE_MTRR_FIXED_ENABLED)) {
0102             /* Var MTRR contains UC entry below 1M? Skip it: */
0103             pr_warn(BIOS_BUG_MSG, i);
0104             if (base + size <= (1<<(20-PAGE_SHIFT)))
0105                 continue;
0106             size -= (1<<(20-PAGE_SHIFT)) - base;
0107             base = 1<<(20-PAGE_SHIFT);
0108         }
0109         subtract_range(range, RANGE_NUM, base, base + size);
0110     }
0111     if (extra_remove_size)
0112         subtract_range(range, RANGE_NUM, extra_remove_base,
0113                  extra_remove_base + extra_remove_size);
0114 
0115     if  (debug_print) {
0116         pr_debug("After UC checking\n");
0117         for (i = 0; i < RANGE_NUM; i++) {
0118             if (!range[i].end)
0119                 continue;
0120             pr_debug("MTRR MAP PFN: %016llx - %016llx\n",
0121                  range[i].start, range[i].end);
0122         }
0123     }
0124 
0125     /* sort the ranges */
0126     nr_range = clean_sort_range(range, RANGE_NUM);
0127     if  (debug_print) {
0128         pr_debug("After sorting\n");
0129         for (i = 0; i < nr_range; i++)
0130             pr_debug("MTRR MAP PFN: %016llx - %016llx\n",
0131                  range[i].start, range[i].end);
0132     }
0133 
0134     return nr_range;
0135 }
0136 
0137 #ifdef CONFIG_MTRR_SANITIZER
0138 
0139 static unsigned long __init sum_ranges(struct range *range, int nr_range)
0140 {
0141     unsigned long sum = 0;
0142     int i;
0143 
0144     for (i = 0; i < nr_range; i++)
0145         sum += range[i].end - range[i].start;
0146 
0147     return sum;
0148 }
0149 
0150 static int enable_mtrr_cleanup __initdata =
0151     CONFIG_MTRR_SANITIZER_ENABLE_DEFAULT;
0152 
0153 static int __init disable_mtrr_cleanup_setup(char *str)
0154 {
0155     enable_mtrr_cleanup = 0;
0156     return 0;
0157 }
0158 early_param("disable_mtrr_cleanup", disable_mtrr_cleanup_setup);
0159 
0160 static int __init enable_mtrr_cleanup_setup(char *str)
0161 {
0162     enable_mtrr_cleanup = 1;
0163     return 0;
0164 }
0165 early_param("enable_mtrr_cleanup", enable_mtrr_cleanup_setup);
0166 
0167 static int __init mtrr_cleanup_debug_setup(char *str)
0168 {
0169     debug_print = 1;
0170     return 0;
0171 }
0172 early_param("mtrr_cleanup_debug", mtrr_cleanup_debug_setup);
0173 
0174 static void __init
0175 set_var_mtrr(unsigned int reg, unsigned long basek, unsigned long sizek,
0176          unsigned char type, unsigned int address_bits)
0177 {
0178     u32 base_lo, base_hi, mask_lo, mask_hi;
0179     u64 base, mask;
0180 
0181     if (!sizek) {
0182         fill_mtrr_var_range(reg, 0, 0, 0, 0);
0183         return;
0184     }
0185 
0186     mask = (1ULL << address_bits) - 1;
0187     mask &= ~((((u64)sizek) << 10) - 1);
0188 
0189     base = ((u64)basek) << 10;
0190 
0191     base |= type;
0192     mask |= 0x800;
0193 
0194     base_lo = base & ((1ULL<<32) - 1);
0195     base_hi = base >> 32;
0196 
0197     mask_lo = mask & ((1ULL<<32) - 1);
0198     mask_hi = mask >> 32;
0199 
0200     fill_mtrr_var_range(reg, base_lo, base_hi, mask_lo, mask_hi);
0201 }
0202 
0203 static void __init
0204 save_var_mtrr(unsigned int reg, unsigned long basek, unsigned long sizek,
0205           unsigned char type)
0206 {
0207     range_state[reg].base_pfn = basek >> (PAGE_SHIFT - 10);
0208     range_state[reg].size_pfn = sizek >> (PAGE_SHIFT - 10);
0209     range_state[reg].type = type;
0210 }
0211 
0212 static void __init set_var_mtrr_all(unsigned int address_bits)
0213 {
0214     unsigned long basek, sizek;
0215     unsigned char type;
0216     unsigned int reg;
0217 
0218     for (reg = 0; reg < num_var_ranges; reg++) {
0219         basek = range_state[reg].base_pfn << (PAGE_SHIFT - 10);
0220         sizek = range_state[reg].size_pfn << (PAGE_SHIFT - 10);
0221         type = range_state[reg].type;
0222 
0223         set_var_mtrr(reg, basek, sizek, type, address_bits);
0224     }
0225 }
0226 
0227 static unsigned long to_size_factor(unsigned long sizek, char *factorp)
0228 {
0229     unsigned long base = sizek;
0230     char factor;
0231 
0232     if (base & ((1<<10) - 1)) {
0233         /* Not MB-aligned: */
0234         factor = 'K';
0235     } else if (base & ((1<<20) - 1)) {
0236         factor = 'M';
0237         base >>= 10;
0238     } else {
0239         factor = 'G';
0240         base >>= 20;
0241     }
0242 
0243     *factorp = factor;
0244 
0245     return base;
0246 }
0247 
0248 static unsigned int __init
0249 range_to_mtrr(unsigned int reg, unsigned long range_startk,
0250           unsigned long range_sizek, unsigned char type)
0251 {
0252     if (!range_sizek || (reg >= num_var_ranges))
0253         return reg;
0254 
0255     while (range_sizek) {
0256         unsigned long max_align, align;
0257         unsigned long sizek;
0258 
0259         /* Compute the maximum size with which we can make a range: */
0260         if (range_startk)
0261             max_align = __ffs(range_startk);
0262         else
0263             max_align = BITS_PER_LONG - 1;
0264 
0265         align = __fls(range_sizek);
0266         if (align > max_align)
0267             align = max_align;
0268 
0269         sizek = 1UL << align;
0270         if (debug_print) {
0271             char start_factor = 'K', size_factor = 'K';
0272             unsigned long start_base, size_base;
0273 
0274             start_base = to_size_factor(range_startk, &start_factor);
0275             size_base = to_size_factor(sizek, &size_factor);
0276 
0277             Dprintk("Setting variable MTRR %d, "
0278                 "base: %ld%cB, range: %ld%cB, type %s\n",
0279                 reg, start_base, start_factor,
0280                 size_base, size_factor,
0281                 (type == MTRR_TYPE_UNCACHABLE) ? "UC" :
0282                    ((type == MTRR_TYPE_WRBACK) ? "WB" : "Other")
0283                 );
0284         }
0285         save_var_mtrr(reg++, range_startk, sizek, type);
0286         range_startk += sizek;
0287         range_sizek -= sizek;
0288         if (reg >= num_var_ranges)
0289             break;
0290     }
0291     return reg;
0292 }
0293 
0294 static unsigned __init
0295 range_to_mtrr_with_hole(struct var_mtrr_state *state, unsigned long basek,
0296             unsigned long sizek)
0297 {
0298     unsigned long hole_basek, hole_sizek;
0299     unsigned long second_sizek;
0300     unsigned long range0_basek, range0_sizek;
0301     unsigned long range_basek, range_sizek;
0302     unsigned long chunk_sizek;
0303     unsigned long gran_sizek;
0304 
0305     hole_basek = 0;
0306     hole_sizek = 0;
0307     second_sizek = 0;
0308     chunk_sizek = state->chunk_sizek;
0309     gran_sizek = state->gran_sizek;
0310 
0311     /* Align with gran size, prevent small block used up MTRRs: */
0312     range_basek = ALIGN(state->range_startk, gran_sizek);
0313     if ((range_basek > basek) && basek)
0314         return second_sizek;
0315 
0316     state->range_sizek -= (range_basek - state->range_startk);
0317     range_sizek = ALIGN(state->range_sizek, gran_sizek);
0318 
0319     while (range_sizek > state->range_sizek) {
0320         range_sizek -= gran_sizek;
0321         if (!range_sizek)
0322             return 0;
0323     }
0324     state->range_sizek = range_sizek;
0325 
0326     /* Try to append some small hole: */
0327     range0_basek = state->range_startk;
0328     range0_sizek = ALIGN(state->range_sizek, chunk_sizek);
0329 
0330     /* No increase: */
0331     if (range0_sizek == state->range_sizek) {
0332         Dprintk("rangeX: %016lx - %016lx\n",
0333             range0_basek<<10,
0334             (range0_basek + state->range_sizek)<<10);
0335         state->reg = range_to_mtrr(state->reg, range0_basek,
0336                 state->range_sizek, MTRR_TYPE_WRBACK);
0337         return 0;
0338     }
0339 
0340     /* Only cut back when it is not the last: */
0341     if (sizek) {
0342         while (range0_basek + range0_sizek > (basek + sizek)) {
0343             if (range0_sizek >= chunk_sizek)
0344                 range0_sizek -= chunk_sizek;
0345             else
0346                 range0_sizek = 0;
0347 
0348             if (!range0_sizek)
0349                 break;
0350         }
0351     }
0352 
0353 second_try:
0354     range_basek = range0_basek + range0_sizek;
0355 
0356     /* One hole in the middle: */
0357     if (range_basek > basek && range_basek <= (basek + sizek))
0358         second_sizek = range_basek - basek;
0359 
0360     if (range0_sizek > state->range_sizek) {
0361 
0362         /* One hole in middle or at the end: */
0363         hole_sizek = range0_sizek - state->range_sizek - second_sizek;
0364 
0365         /* Hole size should be less than half of range0 size: */
0366         if (hole_sizek >= (range0_sizek >> 1) &&
0367             range0_sizek >= chunk_sizek) {
0368             range0_sizek -= chunk_sizek;
0369             second_sizek = 0;
0370             hole_sizek = 0;
0371 
0372             goto second_try;
0373         }
0374     }
0375 
0376     if (range0_sizek) {
0377         Dprintk("range0: %016lx - %016lx\n",
0378             range0_basek<<10,
0379             (range0_basek + range0_sizek)<<10);
0380         state->reg = range_to_mtrr(state->reg, range0_basek,
0381                 range0_sizek, MTRR_TYPE_WRBACK);
0382     }
0383 
0384     if (range0_sizek < state->range_sizek) {
0385         /* Need to handle left over range: */
0386         range_sizek = state->range_sizek - range0_sizek;
0387 
0388         Dprintk("range: %016lx - %016lx\n",
0389              range_basek<<10,
0390              (range_basek + range_sizek)<<10);
0391 
0392         state->reg = range_to_mtrr(state->reg, range_basek,
0393                  range_sizek, MTRR_TYPE_WRBACK);
0394     }
0395 
0396     if (hole_sizek) {
0397         hole_basek = range_basek - hole_sizek - second_sizek;
0398         Dprintk("hole: %016lx - %016lx\n",
0399              hole_basek<<10,
0400              (hole_basek + hole_sizek)<<10);
0401         state->reg = range_to_mtrr(state->reg, hole_basek,
0402                  hole_sizek, MTRR_TYPE_UNCACHABLE);
0403     }
0404 
0405     return second_sizek;
0406 }
0407 
0408 static void __init
0409 set_var_mtrr_range(struct var_mtrr_state *state, unsigned long base_pfn,
0410            unsigned long size_pfn)
0411 {
0412     unsigned long basek, sizek;
0413     unsigned long second_sizek = 0;
0414 
0415     if (state->reg >= num_var_ranges)
0416         return;
0417 
0418     basek = base_pfn << (PAGE_SHIFT - 10);
0419     sizek = size_pfn << (PAGE_SHIFT - 10);
0420 
0421     /* See if I can merge with the last range: */
0422     if ((basek <= 1024) ||
0423         (state->range_startk + state->range_sizek == basek)) {
0424         unsigned long endk = basek + sizek;
0425         state->range_sizek = endk - state->range_startk;
0426         return;
0427     }
0428     /* Write the range mtrrs: */
0429     if (state->range_sizek != 0)
0430         second_sizek = range_to_mtrr_with_hole(state, basek, sizek);
0431 
0432     /* Allocate an msr: */
0433     state->range_startk = basek + second_sizek;
0434     state->range_sizek  = sizek - second_sizek;
0435 }
0436 
0437 /* Minimum size of mtrr block that can take hole: */
0438 static u64 mtrr_chunk_size __initdata = (256ULL<<20);
0439 
0440 static int __init parse_mtrr_chunk_size_opt(char *p)
0441 {
0442     if (!p)
0443         return -EINVAL;
0444     mtrr_chunk_size = memparse(p, &p);
0445     return 0;
0446 }
0447 early_param("mtrr_chunk_size", parse_mtrr_chunk_size_opt);
0448 
0449 /* Granularity of mtrr of block: */
0450 static u64 mtrr_gran_size __initdata;
0451 
0452 static int __init parse_mtrr_gran_size_opt(char *p)
0453 {
0454     if (!p)
0455         return -EINVAL;
0456     mtrr_gran_size = memparse(p, &p);
0457     return 0;
0458 }
0459 early_param("mtrr_gran_size", parse_mtrr_gran_size_opt);
0460 
0461 static unsigned long nr_mtrr_spare_reg __initdata =
0462                  CONFIG_MTRR_SANITIZER_SPARE_REG_NR_DEFAULT;
0463 
0464 static int __init parse_mtrr_spare_reg(char *arg)
0465 {
0466     if (arg)
0467         nr_mtrr_spare_reg = simple_strtoul(arg, NULL, 0);
0468     return 0;
0469 }
0470 early_param("mtrr_spare_reg_nr", parse_mtrr_spare_reg);
0471 
0472 static int __init
0473 x86_setup_var_mtrrs(struct range *range, int nr_range,
0474             u64 chunk_size, u64 gran_size)
0475 {
0476     struct var_mtrr_state var_state;
0477     int num_reg;
0478     int i;
0479 
0480     var_state.range_startk  = 0;
0481     var_state.range_sizek   = 0;
0482     var_state.reg       = 0;
0483     var_state.chunk_sizek   = chunk_size >> 10;
0484     var_state.gran_sizek    = gran_size >> 10;
0485 
0486     memset(range_state, 0, sizeof(range_state));
0487 
0488     /* Write the range: */
0489     for (i = 0; i < nr_range; i++) {
0490         set_var_mtrr_range(&var_state, range[i].start,
0491                    range[i].end - range[i].start);
0492     }
0493 
0494     /* Write the last range: */
0495     if (var_state.range_sizek != 0)
0496         range_to_mtrr_with_hole(&var_state, 0, 0);
0497 
0498     num_reg = var_state.reg;
0499     /* Clear out the extra MTRR's: */
0500     while (var_state.reg < num_var_ranges) {
0501         save_var_mtrr(var_state.reg, 0, 0, 0);
0502         var_state.reg++;
0503     }
0504 
0505     return num_reg;
0506 }
0507 
0508 struct mtrr_cleanup_result {
0509     unsigned long   gran_sizek;
0510     unsigned long   chunk_sizek;
0511     unsigned long   lose_cover_sizek;
0512     unsigned int    num_reg;
0513     int     bad;
0514 };
0515 
0516 /*
0517  * gran_size: 64K, 128K, 256K, 512K, 1M, 2M, ..., 2G
0518  * chunk size: gran_size, ..., 2G
0519  * so we need (1+16)*8
0520  */
0521 #define NUM_RESULT  136
0522 #define PSHIFT      (PAGE_SHIFT - 10)
0523 
0524 static struct mtrr_cleanup_result __initdata result[NUM_RESULT];
0525 static unsigned long __initdata min_loss_pfn[RANGE_NUM];
0526 
0527 static void __init print_out_mtrr_range_state(void)
0528 {
0529     char start_factor = 'K', size_factor = 'K';
0530     unsigned long start_base, size_base;
0531     mtrr_type type;
0532     int i;
0533 
0534     for (i = 0; i < num_var_ranges; i++) {
0535 
0536         size_base = range_state[i].size_pfn << (PAGE_SHIFT - 10);
0537         if (!size_base)
0538             continue;
0539 
0540         size_base = to_size_factor(size_base, &size_factor);
0541         start_base = range_state[i].base_pfn << (PAGE_SHIFT - 10);
0542         start_base = to_size_factor(start_base, &start_factor);
0543         type = range_state[i].type;
0544 
0545         pr_debug("reg %d, base: %ld%cB, range: %ld%cB, type %s\n",
0546             i, start_base, start_factor,
0547             size_base, size_factor,
0548             (type == MTRR_TYPE_UNCACHABLE) ? "UC" :
0549                 ((type == MTRR_TYPE_WRPROT) ? "WP" :
0550                  ((type == MTRR_TYPE_WRBACK) ? "WB" : "Other"))
0551             );
0552     }
0553 }
0554 
0555 static int __init mtrr_need_cleanup(void)
0556 {
0557     int i;
0558     mtrr_type type;
0559     unsigned long size;
0560     /* Extra one for all 0: */
0561     int num[MTRR_NUM_TYPES + 1];
0562 
0563     /* Check entries number: */
0564     memset(num, 0, sizeof(num));
0565     for (i = 0; i < num_var_ranges; i++) {
0566         type = range_state[i].type;
0567         size = range_state[i].size_pfn;
0568         if (type >= MTRR_NUM_TYPES)
0569             continue;
0570         if (!size)
0571             type = MTRR_NUM_TYPES;
0572         num[type]++;
0573     }
0574 
0575     /* Check if we got UC entries: */
0576     if (!num[MTRR_TYPE_UNCACHABLE])
0577         return 0;
0578 
0579     /* Check if we only had WB and UC */
0580     if (num[MTRR_TYPE_WRBACK] + num[MTRR_TYPE_UNCACHABLE] !=
0581         num_var_ranges - num[MTRR_NUM_TYPES])
0582         return 0;
0583 
0584     return 1;
0585 }
0586 
0587 static unsigned long __initdata range_sums;
0588 
0589 static void __init
0590 mtrr_calc_range_state(u64 chunk_size, u64 gran_size,
0591               unsigned long x_remove_base,
0592               unsigned long x_remove_size, int i)
0593 {
0594     /*
0595      * range_new should really be an automatic variable, but
0596      * putting 4096 bytes on the stack is frowned upon, to put it
0597      * mildly. It is safe to make it a static __initdata variable,
0598      * since mtrr_calc_range_state is only called during init and
0599      * there's no way it will call itself recursively.
0600      */
0601     static struct range range_new[RANGE_NUM] __initdata;
0602     unsigned long range_sums_new;
0603     int nr_range_new;
0604     int num_reg;
0605 
0606     /* Convert ranges to var ranges state: */
0607     num_reg = x86_setup_var_mtrrs(range, nr_range, chunk_size, gran_size);
0608 
0609     /* We got new setting in range_state, check it: */
0610     memset(range_new, 0, sizeof(range_new));
0611     nr_range_new = x86_get_mtrr_mem_range(range_new, 0,
0612                 x_remove_base, x_remove_size);
0613     range_sums_new = sum_ranges(range_new, nr_range_new);
0614 
0615     result[i].chunk_sizek = chunk_size >> 10;
0616     result[i].gran_sizek = gran_size >> 10;
0617     result[i].num_reg = num_reg;
0618 
0619     if (range_sums < range_sums_new) {
0620         result[i].lose_cover_sizek = (range_sums_new - range_sums) << PSHIFT;
0621         result[i].bad = 1;
0622     } else {
0623         result[i].lose_cover_sizek = (range_sums - range_sums_new) << PSHIFT;
0624     }
0625 
0626     /* Double check it: */
0627     if (!result[i].bad && !result[i].lose_cover_sizek) {
0628         if (nr_range_new != nr_range || memcmp(range, range_new, sizeof(range)))
0629             result[i].bad = 1;
0630     }
0631 
0632     if (!result[i].bad && (range_sums - range_sums_new < min_loss_pfn[num_reg]))
0633         min_loss_pfn[num_reg] = range_sums - range_sums_new;
0634 }
0635 
0636 static void __init mtrr_print_out_one_result(int i)
0637 {
0638     unsigned long gran_base, chunk_base, lose_base;
0639     char gran_factor, chunk_factor, lose_factor;
0640 
0641     gran_base = to_size_factor(result[i].gran_sizek, &gran_factor);
0642     chunk_base = to_size_factor(result[i].chunk_sizek, &chunk_factor);
0643     lose_base = to_size_factor(result[i].lose_cover_sizek, &lose_factor);
0644 
0645     pr_info("%sgran_size: %ld%c \tchunk_size: %ld%c \t",
0646         result[i].bad ? "*BAD*" : " ",
0647         gran_base, gran_factor, chunk_base, chunk_factor);
0648     pr_cont("num_reg: %d  \tlose cover RAM: %s%ld%c\n",
0649         result[i].num_reg, result[i].bad ? "-" : "",
0650         lose_base, lose_factor);
0651 }
0652 
0653 static int __init mtrr_search_optimal_index(void)
0654 {
0655     int num_reg_good;
0656     int index_good;
0657     int i;
0658 
0659     if (nr_mtrr_spare_reg >= num_var_ranges)
0660         nr_mtrr_spare_reg = num_var_ranges - 1;
0661 
0662     num_reg_good = -1;
0663     for (i = num_var_ranges - nr_mtrr_spare_reg; i > 0; i--) {
0664         if (!min_loss_pfn[i])
0665             num_reg_good = i;
0666     }
0667 
0668     index_good = -1;
0669     if (num_reg_good != -1) {
0670         for (i = 0; i < NUM_RESULT; i++) {
0671             if (!result[i].bad &&
0672                 result[i].num_reg == num_reg_good &&
0673                 !result[i].lose_cover_sizek) {
0674                 index_good = i;
0675                 break;
0676             }
0677         }
0678     }
0679 
0680     return index_good;
0681 }
0682 
0683 int __init mtrr_cleanup(unsigned address_bits)
0684 {
0685     unsigned long x_remove_base, x_remove_size;
0686     unsigned long base, size, def, dummy;
0687     u64 chunk_size, gran_size;
0688     mtrr_type type;
0689     int index_good;
0690     int i;
0691 
0692     if (!is_cpu(INTEL) || enable_mtrr_cleanup < 1)
0693         return 0;
0694 
0695     rdmsr(MSR_MTRRdefType, def, dummy);
0696     def &= 0xff;
0697     if (def != MTRR_TYPE_UNCACHABLE)
0698         return 0;
0699 
0700     /* Get it and store it aside: */
0701     memset(range_state, 0, sizeof(range_state));
0702     for (i = 0; i < num_var_ranges; i++) {
0703         mtrr_if->get(i, &base, &size, &type);
0704         range_state[i].base_pfn = base;
0705         range_state[i].size_pfn = size;
0706         range_state[i].type = type;
0707     }
0708 
0709     /* Check if we need handle it and can handle it: */
0710     if (!mtrr_need_cleanup())
0711         return 0;
0712 
0713     /* Print original var MTRRs at first, for debugging: */
0714     pr_debug("original variable MTRRs\n");
0715     print_out_mtrr_range_state();
0716 
0717     memset(range, 0, sizeof(range));
0718     x_remove_size = 0;
0719     x_remove_base = 1 << (32 - PAGE_SHIFT);
0720     if (mtrr_tom2)
0721         x_remove_size = (mtrr_tom2 >> PAGE_SHIFT) - x_remove_base;
0722 
0723     /*
0724      * [0, 1M) should always be covered by var mtrr with WB
0725      * and fixed mtrrs should take effect before var mtrr for it:
0726      */
0727     nr_range = add_range_with_merge(range, RANGE_NUM, 0, 0,
0728                     1ULL<<(20 - PAGE_SHIFT));
0729     /* add from var mtrr at last */
0730     nr_range = x86_get_mtrr_mem_range(range, nr_range,
0731                       x_remove_base, x_remove_size);
0732 
0733     range_sums = sum_ranges(range, nr_range);
0734     pr_info("total RAM covered: %ldM\n",
0735            range_sums >> (20 - PAGE_SHIFT));
0736 
0737     if (mtrr_chunk_size && mtrr_gran_size) {
0738         i = 0;
0739         mtrr_calc_range_state(mtrr_chunk_size, mtrr_gran_size,
0740                       x_remove_base, x_remove_size, i);
0741 
0742         mtrr_print_out_one_result(i);
0743 
0744         if (!result[i].bad) {
0745             set_var_mtrr_all(address_bits);
0746             pr_debug("New variable MTRRs\n");
0747             print_out_mtrr_range_state();
0748             return 1;
0749         }
0750         pr_info("invalid mtrr_gran_size or mtrr_chunk_size, will find optimal one\n");
0751     }
0752 
0753     i = 0;
0754     memset(min_loss_pfn, 0xff, sizeof(min_loss_pfn));
0755     memset(result, 0, sizeof(result));
0756     for (gran_size = (1ULL<<16); gran_size < (1ULL<<32); gran_size <<= 1) {
0757 
0758         for (chunk_size = gran_size; chunk_size < (1ULL<<32);
0759              chunk_size <<= 1) {
0760 
0761             if (i >= NUM_RESULT)
0762                 continue;
0763 
0764             mtrr_calc_range_state(chunk_size, gran_size,
0765                       x_remove_base, x_remove_size, i);
0766             if (debug_print) {
0767                 mtrr_print_out_one_result(i);
0768                 pr_info("\n");
0769             }
0770 
0771             i++;
0772         }
0773     }
0774 
0775     /* Try to find the optimal index: */
0776     index_good = mtrr_search_optimal_index();
0777 
0778     if (index_good != -1) {
0779         pr_info("Found optimal setting for mtrr clean up\n");
0780         i = index_good;
0781         mtrr_print_out_one_result(i);
0782 
0783         /* Convert ranges to var ranges state: */
0784         chunk_size = result[i].chunk_sizek;
0785         chunk_size <<= 10;
0786         gran_size = result[i].gran_sizek;
0787         gran_size <<= 10;
0788         x86_setup_var_mtrrs(range, nr_range, chunk_size, gran_size);
0789         set_var_mtrr_all(address_bits);
0790         pr_debug("New variable MTRRs\n");
0791         print_out_mtrr_range_state();
0792         return 1;
0793     } else {
0794         /* print out all */
0795         for (i = 0; i < NUM_RESULT; i++)
0796             mtrr_print_out_one_result(i);
0797     }
0798 
0799     pr_info("mtrr_cleanup: can not find optimal value\n");
0800     pr_info("please specify mtrr_gran_size/mtrr_chunk_size\n");
0801 
0802     return 0;
0803 }
0804 #else
0805 int __init mtrr_cleanup(unsigned address_bits)
0806 {
0807     return 0;
0808 }
0809 #endif
0810 
0811 static int disable_mtrr_trim;
0812 
0813 static int __init disable_mtrr_trim_setup(char *str)
0814 {
0815     disable_mtrr_trim = 1;
0816     return 0;
0817 }
0818 early_param("disable_mtrr_trim", disable_mtrr_trim_setup);
0819 
0820 /*
0821  * Newer AMD K8s and later CPUs have a special magic MSR way to force WB
0822  * for memory >4GB. Check for that here.
0823  * Note this won't check if the MTRRs < 4GB where the magic bit doesn't
0824  * apply to are wrong, but so far we don't know of any such case in the wild.
0825  */
0826 #define Tom2Enabled     (1U << 21)
0827 #define Tom2ForceMemTypeWB  (1U << 22)
0828 
0829 int __init amd_special_default_mtrr(void)
0830 {
0831     u32 l, h;
0832 
0833     if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
0834         boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
0835         return 0;
0836     if (boot_cpu_data.x86 < 0xf)
0837         return 0;
0838     /* In case some hypervisor doesn't pass SYSCFG through: */
0839     if (rdmsr_safe(MSR_AMD64_SYSCFG, &l, &h) < 0)
0840         return 0;
0841     /*
0842      * Memory between 4GB and top of mem is forced WB by this magic bit.
0843      * Reserved before K8RevF, but should be zero there.
0844      */
0845     if ((l & (Tom2Enabled | Tom2ForceMemTypeWB)) ==
0846          (Tom2Enabled | Tom2ForceMemTypeWB))
0847         return 1;
0848     return 0;
0849 }
0850 
0851 static u64 __init
0852 real_trim_memory(unsigned long start_pfn, unsigned long limit_pfn)
0853 {
0854     u64 trim_start, trim_size;
0855 
0856     trim_start = start_pfn;
0857     trim_start <<= PAGE_SHIFT;
0858 
0859     trim_size = limit_pfn;
0860     trim_size <<= PAGE_SHIFT;
0861     trim_size -= trim_start;
0862 
0863     return e820__range_update(trim_start, trim_size, E820_TYPE_RAM, E820_TYPE_RESERVED);
0864 }
0865 
0866 /**
0867  * mtrr_trim_uncached_memory - trim RAM not covered by MTRRs
0868  * @end_pfn: ending page frame number
0869  *
0870  * Some buggy BIOSes don't setup the MTRRs properly for systems with certain
0871  * memory configurations.  This routine checks that the highest MTRR matches
0872  * the end of memory, to make sure the MTRRs having a write back type cover
0873  * all of the memory the kernel is intending to use.  If not, it'll trim any
0874  * memory off the end by adjusting end_pfn, removing it from the kernel's
0875  * allocation pools, warning the user with an obnoxious message.
0876  */
0877 int __init mtrr_trim_uncached_memory(unsigned long end_pfn)
0878 {
0879     unsigned long i, base, size, highest_pfn = 0, def, dummy;
0880     mtrr_type type;
0881     u64 total_trim_size;
0882     /* extra one for all 0 */
0883     int num[MTRR_NUM_TYPES + 1];
0884 
0885     /*
0886      * Make sure we only trim uncachable memory on machines that
0887      * support the Intel MTRR architecture:
0888      */
0889     if (!is_cpu(INTEL) || disable_mtrr_trim)
0890         return 0;
0891 
0892     rdmsr(MSR_MTRRdefType, def, dummy);
0893     def &= 0xff;
0894     if (def != MTRR_TYPE_UNCACHABLE)
0895         return 0;
0896 
0897     /* Get it and store it aside: */
0898     memset(range_state, 0, sizeof(range_state));
0899     for (i = 0; i < num_var_ranges; i++) {
0900         mtrr_if->get(i, &base, &size, &type);
0901         range_state[i].base_pfn = base;
0902         range_state[i].size_pfn = size;
0903         range_state[i].type = type;
0904     }
0905 
0906     /* Find highest cached pfn: */
0907     for (i = 0; i < num_var_ranges; i++) {
0908         type = range_state[i].type;
0909         if (type != MTRR_TYPE_WRBACK)
0910             continue;
0911         base = range_state[i].base_pfn;
0912         size = range_state[i].size_pfn;
0913         if (highest_pfn < base + size)
0914             highest_pfn = base + size;
0915     }
0916 
0917     /* kvm/qemu doesn't have mtrr set right, don't trim them all: */
0918     if (!highest_pfn) {
0919         pr_info("CPU MTRRs all blank - virtualized system.\n");
0920         return 0;
0921     }
0922 
0923     /* Check entries number: */
0924     memset(num, 0, sizeof(num));
0925     for (i = 0; i < num_var_ranges; i++) {
0926         type = range_state[i].type;
0927         if (type >= MTRR_NUM_TYPES)
0928             continue;
0929         size = range_state[i].size_pfn;
0930         if (!size)
0931             type = MTRR_NUM_TYPES;
0932         num[type]++;
0933     }
0934 
0935     /* No entry for WB? */
0936     if (!num[MTRR_TYPE_WRBACK])
0937         return 0;
0938 
0939     /* Check if we only had WB and UC: */
0940     if (num[MTRR_TYPE_WRBACK] + num[MTRR_TYPE_UNCACHABLE] !=
0941         num_var_ranges - num[MTRR_NUM_TYPES])
0942         return 0;
0943 
0944     memset(range, 0, sizeof(range));
0945     nr_range = 0;
0946     if (mtrr_tom2) {
0947         range[nr_range].start = (1ULL<<(32 - PAGE_SHIFT));
0948         range[nr_range].end = mtrr_tom2 >> PAGE_SHIFT;
0949         if (highest_pfn < range[nr_range].end)
0950             highest_pfn = range[nr_range].end;
0951         nr_range++;
0952     }
0953     nr_range = x86_get_mtrr_mem_range(range, nr_range, 0, 0);
0954 
0955     /* Check the head: */
0956     total_trim_size = 0;
0957     if (range[0].start)
0958         total_trim_size += real_trim_memory(0, range[0].start);
0959 
0960     /* Check the holes: */
0961     for (i = 0; i < nr_range - 1; i++) {
0962         if (range[i].end < range[i+1].start)
0963             total_trim_size += real_trim_memory(range[i].end,
0964                                 range[i+1].start);
0965     }
0966 
0967     /* Check the top: */
0968     i = nr_range - 1;
0969     if (range[i].end < end_pfn)
0970         total_trim_size += real_trim_memory(range[i].end,
0971                              end_pfn);
0972 
0973     if (total_trim_size) {
0974         pr_warn("WARNING: BIOS bug: CPU MTRRs don't cover all of memory, losing %lluMB of RAM.\n",
0975             total_trim_size >> 20);
0976 
0977         if (!changed_by_mtrr_cleanup)
0978             WARN_ON(1);
0979 
0980         pr_info("update e820 for mtrr\n");
0981         e820__update_table_print();
0982 
0983         return 1;
0984     }
0985 
0986     return 0;
0987 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!