Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Copyright (c) 2019, Intel Corporation.
0004  *
0005  * Heterogeneous Memory Attributes Table (HMAT) representation
0006  *
0007  * This program parses and reports the platform's HMAT tables, and registers
0008  * the applicable attributes with the node's interfaces.
0009  */
0010 
0011 #define pr_fmt(fmt) "acpi/hmat: " fmt
0012 #define dev_fmt(fmt) "acpi/hmat: " fmt
0013 
0014 #include <linux/acpi.h>
0015 #include <linux/bitops.h>
0016 #include <linux/device.h>
0017 #include <linux/init.h>
0018 #include <linux/list.h>
0019 #include <linux/mm.h>
0020 #include <linux/platform_device.h>
0021 #include <linux/list_sort.h>
0022 #include <linux/memregion.h>
0023 #include <linux/memory.h>
0024 #include <linux/mutex.h>
0025 #include <linux/node.h>
0026 #include <linux/sysfs.h>
0027 #include <linux/dax.h>
0028 
0029 static u8 hmat_revision;
0030 static int hmat_disable __initdata;
0031 
0032 void __init disable_hmat(void)
0033 {
0034     hmat_disable = 1;
0035 }
0036 
0037 static LIST_HEAD(targets);
0038 static LIST_HEAD(initiators);
0039 static LIST_HEAD(localities);
0040 
0041 static DEFINE_MUTEX(target_lock);
0042 
0043 /*
0044  * The defined enum order is used to prioritize attributes to break ties when
0045  * selecting the best performing node.
0046  */
0047 enum locality_types {
0048     WRITE_LATENCY,
0049     READ_LATENCY,
0050     WRITE_BANDWIDTH,
0051     READ_BANDWIDTH,
0052 };
0053 
0054 static struct memory_locality *localities_types[4];
0055 
0056 struct target_cache {
0057     struct list_head node;
0058     struct node_cache_attrs cache_attrs;
0059 };
0060 
0061 struct memory_target {
0062     struct list_head node;
0063     unsigned int memory_pxm;
0064     unsigned int processor_pxm;
0065     struct resource memregions;
0066     struct node_hmem_attrs hmem_attrs[2];
0067     struct list_head caches;
0068     struct node_cache_attrs cache_attrs;
0069     bool registered;
0070 };
0071 
0072 struct memory_initiator {
0073     struct list_head node;
0074     unsigned int processor_pxm;
0075     bool has_cpu;
0076 };
0077 
0078 struct memory_locality {
0079     struct list_head node;
0080     struct acpi_hmat_locality *hmat_loc;
0081 };
0082 
0083 static struct memory_initiator *find_mem_initiator(unsigned int cpu_pxm)
0084 {
0085     struct memory_initiator *initiator;
0086 
0087     list_for_each_entry(initiator, &initiators, node)
0088         if (initiator->processor_pxm == cpu_pxm)
0089             return initiator;
0090     return NULL;
0091 }
0092 
0093 static struct memory_target *find_mem_target(unsigned int mem_pxm)
0094 {
0095     struct memory_target *target;
0096 
0097     list_for_each_entry(target, &targets, node)
0098         if (target->memory_pxm == mem_pxm)
0099             return target;
0100     return NULL;
0101 }
0102 
0103 static __init void alloc_memory_initiator(unsigned int cpu_pxm)
0104 {
0105     struct memory_initiator *initiator;
0106 
0107     if (pxm_to_node(cpu_pxm) == NUMA_NO_NODE)
0108         return;
0109 
0110     initiator = find_mem_initiator(cpu_pxm);
0111     if (initiator)
0112         return;
0113 
0114     initiator = kzalloc(sizeof(*initiator), GFP_KERNEL);
0115     if (!initiator)
0116         return;
0117 
0118     initiator->processor_pxm = cpu_pxm;
0119     initiator->has_cpu = node_state(pxm_to_node(cpu_pxm), N_CPU);
0120     list_add_tail(&initiator->node, &initiators);
0121 }
0122 
0123 static __init void alloc_memory_target(unsigned int mem_pxm,
0124         resource_size_t start, resource_size_t len)
0125 {
0126     struct memory_target *target;
0127 
0128     target = find_mem_target(mem_pxm);
0129     if (!target) {
0130         target = kzalloc(sizeof(*target), GFP_KERNEL);
0131         if (!target)
0132             return;
0133         target->memory_pxm = mem_pxm;
0134         target->processor_pxm = PXM_INVAL;
0135         target->memregions = (struct resource) {
0136             .name   = "ACPI mem",
0137             .start  = 0,
0138             .end    = -1,
0139             .flags  = IORESOURCE_MEM,
0140         };
0141         list_add_tail(&target->node, &targets);
0142         INIT_LIST_HEAD(&target->caches);
0143     }
0144 
0145     /*
0146      * There are potentially multiple ranges per PXM, so record each
0147      * in the per-target memregions resource tree.
0148      */
0149     if (!__request_region(&target->memregions, start, len, "memory target",
0150                 IORESOURCE_MEM))
0151         pr_warn("failed to reserve %#llx - %#llx in pxm: %d\n",
0152                 start, start + len, mem_pxm);
0153 }
0154 
0155 static __init const char *hmat_data_type(u8 type)
0156 {
0157     switch (type) {
0158     case ACPI_HMAT_ACCESS_LATENCY:
0159         return "Access Latency";
0160     case ACPI_HMAT_READ_LATENCY:
0161         return "Read Latency";
0162     case ACPI_HMAT_WRITE_LATENCY:
0163         return "Write Latency";
0164     case ACPI_HMAT_ACCESS_BANDWIDTH:
0165         return "Access Bandwidth";
0166     case ACPI_HMAT_READ_BANDWIDTH:
0167         return "Read Bandwidth";
0168     case ACPI_HMAT_WRITE_BANDWIDTH:
0169         return "Write Bandwidth";
0170     default:
0171         return "Reserved";
0172     }
0173 }
0174 
0175 static __init const char *hmat_data_type_suffix(u8 type)
0176 {
0177     switch (type) {
0178     case ACPI_HMAT_ACCESS_LATENCY:
0179     case ACPI_HMAT_READ_LATENCY:
0180     case ACPI_HMAT_WRITE_LATENCY:
0181         return " nsec";
0182     case ACPI_HMAT_ACCESS_BANDWIDTH:
0183     case ACPI_HMAT_READ_BANDWIDTH:
0184     case ACPI_HMAT_WRITE_BANDWIDTH:
0185         return " MB/s";
0186     default:
0187         return "";
0188     }
0189 }
0190 
0191 static u32 hmat_normalize(u16 entry, u64 base, u8 type)
0192 {
0193     u32 value;
0194 
0195     /*
0196      * Check for invalid and overflow values
0197      */
0198     if (entry == 0xffff || !entry)
0199         return 0;
0200     else if (base > (UINT_MAX / (entry)))
0201         return 0;
0202 
0203     /*
0204      * Divide by the base unit for version 1, convert latency from
0205      * picosenonds to nanoseconds if revision 2.
0206      */
0207     value = entry * base;
0208     if (hmat_revision == 1) {
0209         if (value < 10)
0210             return 0;
0211         value = DIV_ROUND_UP(value, 10);
0212     } else if (hmat_revision == 2) {
0213         switch (type) {
0214         case ACPI_HMAT_ACCESS_LATENCY:
0215         case ACPI_HMAT_READ_LATENCY:
0216         case ACPI_HMAT_WRITE_LATENCY:
0217             value = DIV_ROUND_UP(value, 1000);
0218             break;
0219         default:
0220             break;
0221         }
0222     }
0223     return value;
0224 }
0225 
0226 static void hmat_update_target_access(struct memory_target *target,
0227                       u8 type, u32 value, int access)
0228 {
0229     switch (type) {
0230     case ACPI_HMAT_ACCESS_LATENCY:
0231         target->hmem_attrs[access].read_latency = value;
0232         target->hmem_attrs[access].write_latency = value;
0233         break;
0234     case ACPI_HMAT_READ_LATENCY:
0235         target->hmem_attrs[access].read_latency = value;
0236         break;
0237     case ACPI_HMAT_WRITE_LATENCY:
0238         target->hmem_attrs[access].write_latency = value;
0239         break;
0240     case ACPI_HMAT_ACCESS_BANDWIDTH:
0241         target->hmem_attrs[access].read_bandwidth = value;
0242         target->hmem_attrs[access].write_bandwidth = value;
0243         break;
0244     case ACPI_HMAT_READ_BANDWIDTH:
0245         target->hmem_attrs[access].read_bandwidth = value;
0246         break;
0247     case ACPI_HMAT_WRITE_BANDWIDTH:
0248         target->hmem_attrs[access].write_bandwidth = value;
0249         break;
0250     default:
0251         break;
0252     }
0253 }
0254 
0255 static __init void hmat_add_locality(struct acpi_hmat_locality *hmat_loc)
0256 {
0257     struct memory_locality *loc;
0258 
0259     loc = kzalloc(sizeof(*loc), GFP_KERNEL);
0260     if (!loc) {
0261         pr_notice_once("Failed to allocate HMAT locality\n");
0262         return;
0263     }
0264 
0265     loc->hmat_loc = hmat_loc;
0266     list_add_tail(&loc->node, &localities);
0267 
0268     switch (hmat_loc->data_type) {
0269     case ACPI_HMAT_ACCESS_LATENCY:
0270         localities_types[READ_LATENCY] = loc;
0271         localities_types[WRITE_LATENCY] = loc;
0272         break;
0273     case ACPI_HMAT_READ_LATENCY:
0274         localities_types[READ_LATENCY] = loc;
0275         break;
0276     case ACPI_HMAT_WRITE_LATENCY:
0277         localities_types[WRITE_LATENCY] = loc;
0278         break;
0279     case ACPI_HMAT_ACCESS_BANDWIDTH:
0280         localities_types[READ_BANDWIDTH] = loc;
0281         localities_types[WRITE_BANDWIDTH] = loc;
0282         break;
0283     case ACPI_HMAT_READ_BANDWIDTH:
0284         localities_types[READ_BANDWIDTH] = loc;
0285         break;
0286     case ACPI_HMAT_WRITE_BANDWIDTH:
0287         localities_types[WRITE_BANDWIDTH] = loc;
0288         break;
0289     default:
0290         break;
0291     }
0292 }
0293 
0294 static __init int hmat_parse_locality(union acpi_subtable_headers *header,
0295                       const unsigned long end)
0296 {
0297     struct acpi_hmat_locality *hmat_loc = (void *)header;
0298     struct memory_target *target;
0299     unsigned int init, targ, total_size, ipds, tpds;
0300     u32 *inits, *targs, value;
0301     u16 *entries;
0302     u8 type, mem_hier;
0303 
0304     if (hmat_loc->header.length < sizeof(*hmat_loc)) {
0305         pr_notice("HMAT: Unexpected locality header length: %u\n",
0306              hmat_loc->header.length);
0307         return -EINVAL;
0308     }
0309 
0310     type = hmat_loc->data_type;
0311     mem_hier = hmat_loc->flags & ACPI_HMAT_MEMORY_HIERARCHY;
0312     ipds = hmat_loc->number_of_initiator_Pds;
0313     tpds = hmat_loc->number_of_target_Pds;
0314     total_size = sizeof(*hmat_loc) + sizeof(*entries) * ipds * tpds +
0315              sizeof(*inits) * ipds + sizeof(*targs) * tpds;
0316     if (hmat_loc->header.length < total_size) {
0317         pr_notice("HMAT: Unexpected locality header length:%u, minimum required:%u\n",
0318              hmat_loc->header.length, total_size);
0319         return -EINVAL;
0320     }
0321 
0322     pr_info("HMAT: Locality: Flags:%02x Type:%s Initiator Domains:%u Target Domains:%u Base:%lld\n",
0323         hmat_loc->flags, hmat_data_type(type), ipds, tpds,
0324         hmat_loc->entry_base_unit);
0325 
0326     inits = (u32 *)(hmat_loc + 1);
0327     targs = inits + ipds;
0328     entries = (u16 *)(targs + tpds);
0329     for (init = 0; init < ipds; init++) {
0330         alloc_memory_initiator(inits[init]);
0331         for (targ = 0; targ < tpds; targ++) {
0332             value = hmat_normalize(entries[init * tpds + targ],
0333                            hmat_loc->entry_base_unit,
0334                            type);
0335             pr_info("  Initiator-Target[%u-%u]:%u%s\n",
0336                 inits[init], targs[targ], value,
0337                 hmat_data_type_suffix(type));
0338 
0339             if (mem_hier == ACPI_HMAT_MEMORY) {
0340                 target = find_mem_target(targs[targ]);
0341                 if (target && target->processor_pxm == inits[init]) {
0342                     hmat_update_target_access(target, type, value, 0);
0343                     /* If the node has a CPU, update access 1 */
0344                     if (node_state(pxm_to_node(inits[init]), N_CPU))
0345                         hmat_update_target_access(target, type, value, 1);
0346                 }
0347             }
0348         }
0349     }
0350 
0351     if (mem_hier == ACPI_HMAT_MEMORY)
0352         hmat_add_locality(hmat_loc);
0353 
0354     return 0;
0355 }
0356 
0357 static __init int hmat_parse_cache(union acpi_subtable_headers *header,
0358                    const unsigned long end)
0359 {
0360     struct acpi_hmat_cache *cache = (void *)header;
0361     struct memory_target *target;
0362     struct target_cache *tcache;
0363     u32 attrs;
0364 
0365     if (cache->header.length < sizeof(*cache)) {
0366         pr_notice("HMAT: Unexpected cache header length: %u\n",
0367              cache->header.length);
0368         return -EINVAL;
0369     }
0370 
0371     attrs = cache->cache_attributes;
0372     pr_info("HMAT: Cache: Domain:%u Size:%llu Attrs:%08x SMBIOS Handles:%d\n",
0373         cache->memory_PD, cache->cache_size, attrs,
0374         cache->number_of_SMBIOShandles);
0375 
0376     target = find_mem_target(cache->memory_PD);
0377     if (!target)
0378         return 0;
0379 
0380     tcache = kzalloc(sizeof(*tcache), GFP_KERNEL);
0381     if (!tcache) {
0382         pr_notice_once("Failed to allocate HMAT cache info\n");
0383         return 0;
0384     }
0385 
0386     tcache->cache_attrs.size = cache->cache_size;
0387     tcache->cache_attrs.level = (attrs & ACPI_HMAT_CACHE_LEVEL) >> 4;
0388     tcache->cache_attrs.line_size = (attrs & ACPI_HMAT_CACHE_LINE_SIZE) >> 16;
0389 
0390     switch ((attrs & ACPI_HMAT_CACHE_ASSOCIATIVITY) >> 8) {
0391     case ACPI_HMAT_CA_DIRECT_MAPPED:
0392         tcache->cache_attrs.indexing = NODE_CACHE_DIRECT_MAP;
0393         break;
0394     case ACPI_HMAT_CA_COMPLEX_CACHE_INDEXING:
0395         tcache->cache_attrs.indexing = NODE_CACHE_INDEXED;
0396         break;
0397     case ACPI_HMAT_CA_NONE:
0398     default:
0399         tcache->cache_attrs.indexing = NODE_CACHE_OTHER;
0400         break;
0401     }
0402 
0403     switch ((attrs & ACPI_HMAT_WRITE_POLICY) >> 12) {
0404     case ACPI_HMAT_CP_WB:
0405         tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_BACK;
0406         break;
0407     case ACPI_HMAT_CP_WT:
0408         tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_THROUGH;
0409         break;
0410     case ACPI_HMAT_CP_NONE:
0411     default:
0412         tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_OTHER;
0413         break;
0414     }
0415     list_add_tail(&tcache->node, &target->caches);
0416 
0417     return 0;
0418 }
0419 
0420 static int __init hmat_parse_proximity_domain(union acpi_subtable_headers *header,
0421                           const unsigned long end)
0422 {
0423     struct acpi_hmat_proximity_domain *p = (void *)header;
0424     struct memory_target *target = NULL;
0425 
0426     if (p->header.length != sizeof(*p)) {
0427         pr_notice("HMAT: Unexpected address range header length: %u\n",
0428              p->header.length);
0429         return -EINVAL;
0430     }
0431 
0432     if (hmat_revision == 1)
0433         pr_info("HMAT: Memory (%#llx length %#llx) Flags:%04x Processor Domain:%u Memory Domain:%u\n",
0434             p->reserved3, p->reserved4, p->flags, p->processor_PD,
0435             p->memory_PD);
0436     else
0437         pr_info("HMAT: Memory Flags:%04x Processor Domain:%u Memory Domain:%u\n",
0438             p->flags, p->processor_PD, p->memory_PD);
0439 
0440     if ((hmat_revision == 1 && p->flags & ACPI_HMAT_MEMORY_PD_VALID) ||
0441         hmat_revision > 1) {
0442         target = find_mem_target(p->memory_PD);
0443         if (!target) {
0444             pr_debug("HMAT: Memory Domain missing from SRAT\n");
0445             return -EINVAL;
0446         }
0447     }
0448     if (target && p->flags & ACPI_HMAT_PROCESSOR_PD_VALID) {
0449         int p_node = pxm_to_node(p->processor_PD);
0450 
0451         if (p_node == NUMA_NO_NODE) {
0452             pr_debug("HMAT: Invalid Processor Domain\n");
0453             return -EINVAL;
0454         }
0455         target->processor_pxm = p->processor_PD;
0456     }
0457 
0458     return 0;
0459 }
0460 
0461 static int __init hmat_parse_subtable(union acpi_subtable_headers *header,
0462                       const unsigned long end)
0463 {
0464     struct acpi_hmat_structure *hdr = (void *)header;
0465 
0466     if (!hdr)
0467         return -EINVAL;
0468 
0469     switch (hdr->type) {
0470     case ACPI_HMAT_TYPE_PROXIMITY:
0471         return hmat_parse_proximity_domain(header, end);
0472     case ACPI_HMAT_TYPE_LOCALITY:
0473         return hmat_parse_locality(header, end);
0474     case ACPI_HMAT_TYPE_CACHE:
0475         return hmat_parse_cache(header, end);
0476     default:
0477         return -EINVAL;
0478     }
0479 }
0480 
0481 static __init int srat_parse_mem_affinity(union acpi_subtable_headers *header,
0482                       const unsigned long end)
0483 {
0484     struct acpi_srat_mem_affinity *ma = (void *)header;
0485 
0486     if (!ma)
0487         return -EINVAL;
0488     if (!(ma->flags & ACPI_SRAT_MEM_ENABLED))
0489         return 0;
0490     alloc_memory_target(ma->proximity_domain, ma->base_address, ma->length);
0491     return 0;
0492 }
0493 
0494 static u32 hmat_initiator_perf(struct memory_target *target,
0495                    struct memory_initiator *initiator,
0496                    struct acpi_hmat_locality *hmat_loc)
0497 {
0498     unsigned int ipds, tpds, i, idx = 0, tdx = 0;
0499     u32 *inits, *targs;
0500     u16 *entries;
0501 
0502     ipds = hmat_loc->number_of_initiator_Pds;
0503     tpds = hmat_loc->number_of_target_Pds;
0504     inits = (u32 *)(hmat_loc + 1);
0505     targs = inits + ipds;
0506     entries = (u16 *)(targs + tpds);
0507 
0508     for (i = 0; i < ipds; i++) {
0509         if (inits[i] == initiator->processor_pxm) {
0510             idx = i;
0511             break;
0512         }
0513     }
0514 
0515     if (i == ipds)
0516         return 0;
0517 
0518     for (i = 0; i < tpds; i++) {
0519         if (targs[i] == target->memory_pxm) {
0520             tdx = i;
0521             break;
0522         }
0523     }
0524     if (i == tpds)
0525         return 0;
0526 
0527     return hmat_normalize(entries[idx * tpds + tdx],
0528                   hmat_loc->entry_base_unit,
0529                   hmat_loc->data_type);
0530 }
0531 
0532 static bool hmat_update_best(u8 type, u32 value, u32 *best)
0533 {
0534     bool updated = false;
0535 
0536     if (!value)
0537         return false;
0538 
0539     switch (type) {
0540     case ACPI_HMAT_ACCESS_LATENCY:
0541     case ACPI_HMAT_READ_LATENCY:
0542     case ACPI_HMAT_WRITE_LATENCY:
0543         if (!*best || *best > value) {
0544             *best = value;
0545             updated = true;
0546         }
0547         break;
0548     case ACPI_HMAT_ACCESS_BANDWIDTH:
0549     case ACPI_HMAT_READ_BANDWIDTH:
0550     case ACPI_HMAT_WRITE_BANDWIDTH:
0551         if (!*best || *best < value) {
0552             *best = value;
0553             updated = true;
0554         }
0555         break;
0556     }
0557 
0558     return updated;
0559 }
0560 
0561 static int initiator_cmp(void *priv, const struct list_head *a,
0562              const struct list_head *b)
0563 {
0564     struct memory_initiator *ia;
0565     struct memory_initiator *ib;
0566     unsigned long *p_nodes = priv;
0567 
0568     ia = list_entry(a, struct memory_initiator, node);
0569     ib = list_entry(b, struct memory_initiator, node);
0570 
0571     set_bit(ia->processor_pxm, p_nodes);
0572     set_bit(ib->processor_pxm, p_nodes);
0573 
0574     return ia->processor_pxm - ib->processor_pxm;
0575 }
0576 
0577 static void hmat_register_target_initiators(struct memory_target *target)
0578 {
0579     static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);
0580     struct memory_initiator *initiator;
0581     unsigned int mem_nid, cpu_nid;
0582     struct memory_locality *loc = NULL;
0583     u32 best = 0;
0584     bool access0done = false;
0585     int i;
0586 
0587     mem_nid = pxm_to_node(target->memory_pxm);
0588     /*
0589      * If the Address Range Structure provides a local processor pxm, link
0590      * only that one. Otherwise, find the best performance attributes and
0591      * register all initiators that match.
0592      */
0593     if (target->processor_pxm != PXM_INVAL) {
0594         cpu_nid = pxm_to_node(target->processor_pxm);
0595         register_memory_node_under_compute_node(mem_nid, cpu_nid, 0);
0596         access0done = true;
0597         if (node_state(cpu_nid, N_CPU)) {
0598             register_memory_node_under_compute_node(mem_nid, cpu_nid, 1);
0599             return;
0600         }
0601     }
0602 
0603     if (list_empty(&localities))
0604         return;
0605 
0606     /*
0607      * We need the initiator list sorted so we can use bitmap_clear for
0608      * previously set initiators when we find a better memory accessor.
0609      * We'll also use the sorting to prime the candidate nodes with known
0610      * initiators.
0611      */
0612     bitmap_zero(p_nodes, MAX_NUMNODES);
0613     list_sort(p_nodes, &initiators, initiator_cmp);
0614     if (!access0done) {
0615         for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) {
0616             loc = localities_types[i];
0617             if (!loc)
0618                 continue;
0619 
0620             best = 0;
0621             list_for_each_entry(initiator, &initiators, node) {
0622                 u32 value;
0623 
0624                 if (!test_bit(initiator->processor_pxm, p_nodes))
0625                     continue;
0626 
0627                 value = hmat_initiator_perf(target, initiator,
0628                                 loc->hmat_loc);
0629                 if (hmat_update_best(loc->hmat_loc->data_type, value, &best))
0630                     bitmap_clear(p_nodes, 0, initiator->processor_pxm);
0631                 if (value != best)
0632                     clear_bit(initiator->processor_pxm, p_nodes);
0633             }
0634             if (best)
0635                 hmat_update_target_access(target, loc->hmat_loc->data_type,
0636                               best, 0);
0637         }
0638 
0639         for_each_set_bit(i, p_nodes, MAX_NUMNODES) {
0640             cpu_nid = pxm_to_node(i);
0641             register_memory_node_under_compute_node(mem_nid, cpu_nid, 0);
0642         }
0643     }
0644 
0645     /* Access 1 ignores Generic Initiators */
0646     bitmap_zero(p_nodes, MAX_NUMNODES);
0647     list_sort(p_nodes, &initiators, initiator_cmp);
0648     best = 0;
0649     for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) {
0650         loc = localities_types[i];
0651         if (!loc)
0652             continue;
0653 
0654         best = 0;
0655         list_for_each_entry(initiator, &initiators, node) {
0656             u32 value;
0657 
0658             if (!initiator->has_cpu) {
0659                 clear_bit(initiator->processor_pxm, p_nodes);
0660                 continue;
0661             }
0662             if (!test_bit(initiator->processor_pxm, p_nodes))
0663                 continue;
0664 
0665             value = hmat_initiator_perf(target, initiator, loc->hmat_loc);
0666             if (hmat_update_best(loc->hmat_loc->data_type, value, &best))
0667                 bitmap_clear(p_nodes, 0, initiator->processor_pxm);
0668             if (value != best)
0669                 clear_bit(initiator->processor_pxm, p_nodes);
0670         }
0671         if (best)
0672             hmat_update_target_access(target, loc->hmat_loc->data_type, best, 1);
0673     }
0674     for_each_set_bit(i, p_nodes, MAX_NUMNODES) {
0675         cpu_nid = pxm_to_node(i);
0676         register_memory_node_under_compute_node(mem_nid, cpu_nid, 1);
0677     }
0678 }
0679 
0680 static void hmat_register_target_cache(struct memory_target *target)
0681 {
0682     unsigned mem_nid = pxm_to_node(target->memory_pxm);
0683     struct target_cache *tcache;
0684 
0685     list_for_each_entry(tcache, &target->caches, node)
0686         node_add_cache(mem_nid, &tcache->cache_attrs);
0687 }
0688 
0689 static void hmat_register_target_perf(struct memory_target *target, int access)
0690 {
0691     unsigned mem_nid = pxm_to_node(target->memory_pxm);
0692     node_set_perf_attrs(mem_nid, &target->hmem_attrs[access], access);
0693 }
0694 
0695 static void hmat_register_target_devices(struct memory_target *target)
0696 {
0697     struct resource *res;
0698 
0699     /*
0700      * Do not bother creating devices if no driver is available to
0701      * consume them.
0702      */
0703     if (!IS_ENABLED(CONFIG_DEV_DAX_HMEM))
0704         return;
0705 
0706     for (res = target->memregions.child; res; res = res->sibling) {
0707         int target_nid = pxm_to_node(target->memory_pxm);
0708 
0709         hmem_register_device(target_nid, res);
0710     }
0711 }
0712 
0713 static void hmat_register_target(struct memory_target *target)
0714 {
0715     int nid = pxm_to_node(target->memory_pxm);
0716 
0717     /*
0718      * Devices may belong to either an offline or online
0719      * node, so unconditionally add them.
0720      */
0721     hmat_register_target_devices(target);
0722 
0723     /*
0724      * Skip offline nodes. This can happen when memory
0725      * marked EFI_MEMORY_SP, "specific purpose", is applied
0726      * to all the memory in a proximity domain leading to
0727      * the node being marked offline / unplugged, or if
0728      * memory-only "hotplug" node is offline.
0729      */
0730     if (nid == NUMA_NO_NODE || !node_online(nid))
0731         return;
0732 
0733     mutex_lock(&target_lock);
0734     if (!target->registered) {
0735         hmat_register_target_initiators(target);
0736         hmat_register_target_cache(target);
0737         hmat_register_target_perf(target, 0);
0738         hmat_register_target_perf(target, 1);
0739         target->registered = true;
0740     }
0741     mutex_unlock(&target_lock);
0742 }
0743 
0744 static void hmat_register_targets(void)
0745 {
0746     struct memory_target *target;
0747 
0748     list_for_each_entry(target, &targets, node)
0749         hmat_register_target(target);
0750 }
0751 
0752 static int hmat_callback(struct notifier_block *self,
0753              unsigned long action, void *arg)
0754 {
0755     struct memory_target *target;
0756     struct memory_notify *mnb = arg;
0757     int pxm, nid = mnb->status_change_nid;
0758 
0759     if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
0760         return NOTIFY_OK;
0761 
0762     pxm = node_to_pxm(nid);
0763     target = find_mem_target(pxm);
0764     if (!target)
0765         return NOTIFY_OK;
0766 
0767     hmat_register_target(target);
0768     return NOTIFY_OK;
0769 }
0770 
0771 static struct notifier_block hmat_callback_nb = {
0772     .notifier_call = hmat_callback,
0773     .priority = 2,
0774 };
0775 
0776 static __init void hmat_free_structures(void)
0777 {
0778     struct memory_target *target, *tnext;
0779     struct memory_locality *loc, *lnext;
0780     struct memory_initiator *initiator, *inext;
0781     struct target_cache *tcache, *cnext;
0782 
0783     list_for_each_entry_safe(target, tnext, &targets, node) {
0784         struct resource *res, *res_next;
0785 
0786         list_for_each_entry_safe(tcache, cnext, &target->caches, node) {
0787             list_del(&tcache->node);
0788             kfree(tcache);
0789         }
0790 
0791         list_del(&target->node);
0792         res = target->memregions.child;
0793         while (res) {
0794             res_next = res->sibling;
0795             __release_region(&target->memregions, res->start,
0796                     resource_size(res));
0797             res = res_next;
0798         }
0799         kfree(target);
0800     }
0801 
0802     list_for_each_entry_safe(initiator, inext, &initiators, node) {
0803         list_del(&initiator->node);
0804         kfree(initiator);
0805     }
0806 
0807     list_for_each_entry_safe(loc, lnext, &localities, node) {
0808         list_del(&loc->node);
0809         kfree(loc);
0810     }
0811 }
0812 
0813 static __init int hmat_init(void)
0814 {
0815     struct acpi_table_header *tbl;
0816     enum acpi_hmat_type i;
0817     acpi_status status;
0818 
0819     if (srat_disabled() || hmat_disable)
0820         return 0;
0821 
0822     status = acpi_get_table(ACPI_SIG_SRAT, 0, &tbl);
0823     if (ACPI_FAILURE(status))
0824         return 0;
0825 
0826     if (acpi_table_parse_entries(ACPI_SIG_SRAT,
0827                 sizeof(struct acpi_table_srat),
0828                 ACPI_SRAT_TYPE_MEMORY_AFFINITY,
0829                 srat_parse_mem_affinity, 0) < 0)
0830         goto out_put;
0831     acpi_put_table(tbl);
0832 
0833     status = acpi_get_table(ACPI_SIG_HMAT, 0, &tbl);
0834     if (ACPI_FAILURE(status))
0835         goto out_put;
0836 
0837     hmat_revision = tbl->revision;
0838     switch (hmat_revision) {
0839     case 1:
0840     case 2:
0841         break;
0842     default:
0843         pr_notice("Ignoring HMAT: Unknown revision:%d\n", hmat_revision);
0844         goto out_put;
0845     }
0846 
0847     for (i = ACPI_HMAT_TYPE_PROXIMITY; i < ACPI_HMAT_TYPE_RESERVED; i++) {
0848         if (acpi_table_parse_entries(ACPI_SIG_HMAT,
0849                          sizeof(struct acpi_table_hmat), i,
0850                          hmat_parse_subtable, 0) < 0) {
0851             pr_notice("Ignoring HMAT: Invalid table");
0852             goto out_put;
0853         }
0854     }
0855     hmat_register_targets();
0856 
0857     /* Keep the table and structures if the notifier may use them */
0858     if (!register_hotmemory_notifier(&hmat_callback_nb))
0859         return 0;
0860 out_put:
0861     hmat_free_structures();
0862     acpi_put_table(tbl);
0863     return 0;
0864 }
0865 device_initcall(hmat_init);
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!