Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * SN Platform GRU Driver
0004  *
0005  *              FILE OPERATIONS & DRIVER INITIALIZATION
0006  *
0007  * This file supports the user system call for file open, close, mmap, etc.
0008  * This also incudes the driver initialization code.
0009  *
0010  *  (C) Copyright 2020 Hewlett Packard Enterprise Development LP
0011  *  Copyright (c) 2008-2014 Silicon Graphics, Inc.  All Rights Reserved.
0012  */
0013 
0014 #include <linux/module.h>
0015 #include <linux/kernel.h>
0016 #include <linux/errno.h>
0017 #include <linux/slab.h>
0018 #include <linux/mm.h>
0019 #include <linux/io.h>
0020 #include <linux/spinlock.h>
0021 #include <linux/device.h>
0022 #include <linux/miscdevice.h>
0023 #include <linux/interrupt.h>
0024 #include <linux/proc_fs.h>
0025 #include <linux/uaccess.h>
0026 #ifdef CONFIG_X86_64
0027 #include <asm/uv/uv_irq.h>
0028 #endif
0029 #include <asm/uv/uv.h>
0030 #include "gru.h"
0031 #include "grulib.h"
0032 #include "grutables.h"
0033 
0034 #include <asm/uv/uv_hub.h>
0035 #include <asm/uv/uv_mmrs.h>
0036 
0037 struct gru_blade_state *gru_base[GRU_MAX_BLADES] __read_mostly;
0038 unsigned long gru_start_paddr __read_mostly;
0039 void *gru_start_vaddr __read_mostly;
0040 unsigned long gru_end_paddr __read_mostly;
0041 unsigned int gru_max_gids __read_mostly;
0042 struct gru_stats_s gru_stats;
0043 
0044 /* Guaranteed user available resources on each node */
0045 static int max_user_cbrs, max_user_dsr_bytes;
0046 
0047 static struct miscdevice gru_miscdev;
0048 
0049 static int gru_supported(void)
0050 {
0051     return is_uv_system() &&
0052         (uv_hub_info->hub_revision < UV3_HUB_REVISION_BASE);
0053 }
0054 
0055 /*
0056  * gru_vma_close
0057  *
0058  * Called when unmapping a device mapping. Frees all gru resources
0059  * and tables belonging to the vma.
0060  */
0061 static void gru_vma_close(struct vm_area_struct *vma)
0062 {
0063     struct gru_vma_data *vdata;
0064     struct gru_thread_state *gts;
0065     struct list_head *entry, *next;
0066 
0067     if (!vma->vm_private_data)
0068         return;
0069 
0070     vdata = vma->vm_private_data;
0071     vma->vm_private_data = NULL;
0072     gru_dbg(grudev, "vma %p, file %p, vdata %p\n", vma, vma->vm_file,
0073                 vdata);
0074     list_for_each_safe(entry, next, &vdata->vd_head) {
0075         gts =
0076             list_entry(entry, struct gru_thread_state, ts_next);
0077         list_del(&gts->ts_next);
0078         mutex_lock(&gts->ts_ctxlock);
0079         if (gts->ts_gru)
0080             gru_unload_context(gts, 0);
0081         mutex_unlock(&gts->ts_ctxlock);
0082         gts_drop(gts);
0083     }
0084     kfree(vdata);
0085     STAT(vdata_free);
0086 }
0087 
0088 /*
0089  * gru_file_mmap
0090  *
0091  * Called when mmapping the device.  Initializes the vma with a fault handler
0092  * and private data structure necessary to allocate, track, and free the
0093  * underlying pages.
0094  */
0095 static int gru_file_mmap(struct file *file, struct vm_area_struct *vma)
0096 {
0097     if ((vma->vm_flags & (VM_SHARED | VM_WRITE)) != (VM_SHARED | VM_WRITE))
0098         return -EPERM;
0099 
0100     if (vma->vm_start & (GRU_GSEG_PAGESIZE - 1) ||
0101                 vma->vm_end & (GRU_GSEG_PAGESIZE - 1))
0102         return -EINVAL;
0103 
0104     vma->vm_flags |= VM_IO | VM_PFNMAP | VM_LOCKED |
0105              VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP;
0106     vma->vm_page_prot = PAGE_SHARED;
0107     vma->vm_ops = &gru_vm_ops;
0108 
0109     vma->vm_private_data = gru_alloc_vma_data(vma, 0);
0110     if (!vma->vm_private_data)
0111         return -ENOMEM;
0112 
0113     gru_dbg(grudev, "file %p, vaddr 0x%lx, vma %p, vdata %p\n",
0114         file, vma->vm_start, vma, vma->vm_private_data);
0115     return 0;
0116 }
0117 
0118 /*
0119  * Create a new GRU context
0120  */
0121 static int gru_create_new_context(unsigned long arg)
0122 {
0123     struct gru_create_context_req req;
0124     struct vm_area_struct *vma;
0125     struct gru_vma_data *vdata;
0126     int ret = -EINVAL;
0127 
0128     if (copy_from_user(&req, (void __user *)arg, sizeof(req)))
0129         return -EFAULT;
0130 
0131     if (req.data_segment_bytes > max_user_dsr_bytes)
0132         return -EINVAL;
0133     if (req.control_blocks > max_user_cbrs || !req.maximum_thread_count)
0134         return -EINVAL;
0135 
0136     if (!(req.options & GRU_OPT_MISS_MASK))
0137         req.options |= GRU_OPT_MISS_FMM_INTR;
0138 
0139     mmap_write_lock(current->mm);
0140     vma = gru_find_vma(req.gseg);
0141     if (vma) {
0142         vdata = vma->vm_private_data;
0143         vdata->vd_user_options = req.options;
0144         vdata->vd_dsr_au_count =
0145             GRU_DS_BYTES_TO_AU(req.data_segment_bytes);
0146         vdata->vd_cbr_au_count = GRU_CB_COUNT_TO_AU(req.control_blocks);
0147         vdata->vd_tlb_preload_count = req.tlb_preload_count;
0148         ret = 0;
0149     }
0150     mmap_write_unlock(current->mm);
0151 
0152     return ret;
0153 }
0154 
0155 /*
0156  * Get GRU configuration info (temp - for emulator testing)
0157  */
0158 static long gru_get_config_info(unsigned long arg)
0159 {
0160     struct gru_config_info info;
0161     int nodesperblade;
0162 
0163     if (num_online_nodes() > 1 &&
0164             (uv_node_to_blade_id(1) == uv_node_to_blade_id(0)))
0165         nodesperblade = 2;
0166     else
0167         nodesperblade = 1;
0168     memset(&info, 0, sizeof(info));
0169     info.cpus = num_online_cpus();
0170     info.nodes = num_online_nodes();
0171     info.blades = info.nodes / nodesperblade;
0172     info.chiplets = GRU_CHIPLETS_PER_BLADE * info.blades;
0173 
0174     if (copy_to_user((void __user *)arg, &info, sizeof(info)))
0175         return -EFAULT;
0176     return 0;
0177 }
0178 
0179 /*
0180  * gru_file_unlocked_ioctl
0181  *
0182  * Called to update file attributes via IOCTL calls.
0183  */
0184 static long gru_file_unlocked_ioctl(struct file *file, unsigned int req,
0185                     unsigned long arg)
0186 {
0187     int err = -EBADRQC;
0188 
0189     gru_dbg(grudev, "file %p, req 0x%x, 0x%lx\n", file, req, arg);
0190 
0191     switch (req) {
0192     case GRU_CREATE_CONTEXT:
0193         err = gru_create_new_context(arg);
0194         break;
0195     case GRU_SET_CONTEXT_OPTION:
0196         err = gru_set_context_option(arg);
0197         break;
0198     case GRU_USER_GET_EXCEPTION_DETAIL:
0199         err = gru_get_exception_detail(arg);
0200         break;
0201     case GRU_USER_UNLOAD_CONTEXT:
0202         err = gru_user_unload_context(arg);
0203         break;
0204     case GRU_USER_FLUSH_TLB:
0205         err = gru_user_flush_tlb(arg);
0206         break;
0207     case GRU_USER_CALL_OS:
0208         err = gru_handle_user_call_os(arg);
0209         break;
0210     case GRU_GET_GSEG_STATISTICS:
0211         err = gru_get_gseg_statistics(arg);
0212         break;
0213     case GRU_KTEST:
0214         err = gru_ktest(arg);
0215         break;
0216     case GRU_GET_CONFIG_INFO:
0217         err = gru_get_config_info(arg);
0218         break;
0219     case GRU_DUMP_CHIPLET_STATE:
0220         err = gru_dump_chiplet_request(arg);
0221         break;
0222     }
0223     return err;
0224 }
0225 
0226 /*
0227  * Called at init time to build tables for all GRUs that are present in the
0228  * system.
0229  */
0230 static void gru_init_chiplet(struct gru_state *gru, unsigned long paddr,
0231                  void *vaddr, int blade_id, int chiplet_id)
0232 {
0233     spin_lock_init(&gru->gs_lock);
0234     spin_lock_init(&gru->gs_asid_lock);
0235     gru->gs_gru_base_paddr = paddr;
0236     gru->gs_gru_base_vaddr = vaddr;
0237     gru->gs_gid = blade_id * GRU_CHIPLETS_PER_BLADE + chiplet_id;
0238     gru->gs_blade = gru_base[blade_id];
0239     gru->gs_blade_id = blade_id;
0240     gru->gs_chiplet_id = chiplet_id;
0241     gru->gs_cbr_map = (GRU_CBR_AU == 64) ? ~0 : (1UL << GRU_CBR_AU) - 1;
0242     gru->gs_dsr_map = (1UL << GRU_DSR_AU) - 1;
0243     gru->gs_asid_limit = MAX_ASID;
0244     gru_tgh_flush_init(gru);
0245     if (gru->gs_gid >= gru_max_gids)
0246         gru_max_gids = gru->gs_gid + 1;
0247     gru_dbg(grudev, "bid %d, gid %d, vaddr %p (0x%lx)\n",
0248         blade_id, gru->gs_gid, gru->gs_gru_base_vaddr,
0249         gru->gs_gru_base_paddr);
0250 }
0251 
0252 static int gru_init_tables(unsigned long gru_base_paddr, void *gru_base_vaddr)
0253 {
0254     int pnode, nid, bid, chip;
0255     int cbrs, dsrbytes, n;
0256     int order = get_order(sizeof(struct gru_blade_state));
0257     struct page *page;
0258     struct gru_state *gru;
0259     unsigned long paddr;
0260     void *vaddr;
0261 
0262     max_user_cbrs = GRU_NUM_CB;
0263     max_user_dsr_bytes = GRU_NUM_DSR_BYTES;
0264     for_each_possible_blade(bid) {
0265         pnode = uv_blade_to_pnode(bid);
0266         nid = uv_blade_to_memory_nid(bid);/* -1 if no memory on blade */
0267         page = alloc_pages_node(nid, GFP_KERNEL, order);
0268         if (!page)
0269             goto fail;
0270         gru_base[bid] = page_address(page);
0271         memset(gru_base[bid], 0, sizeof(struct gru_blade_state));
0272         gru_base[bid]->bs_lru_gru = &gru_base[bid]->bs_grus[0];
0273         spin_lock_init(&gru_base[bid]->bs_lock);
0274         init_rwsem(&gru_base[bid]->bs_kgts_sema);
0275 
0276         dsrbytes = 0;
0277         cbrs = 0;
0278         for (gru = gru_base[bid]->bs_grus, chip = 0;
0279                 chip < GRU_CHIPLETS_PER_BLADE;
0280                 chip++, gru++) {
0281             paddr = gru_chiplet_paddr(gru_base_paddr, pnode, chip);
0282             vaddr = gru_chiplet_vaddr(gru_base_vaddr, pnode, chip);
0283             gru_init_chiplet(gru, paddr, vaddr, bid, chip);
0284             n = hweight64(gru->gs_cbr_map) * GRU_CBR_AU_SIZE;
0285             cbrs = max(cbrs, n);
0286             n = hweight64(gru->gs_dsr_map) * GRU_DSR_AU_BYTES;
0287             dsrbytes = max(dsrbytes, n);
0288         }
0289         max_user_cbrs = min(max_user_cbrs, cbrs);
0290         max_user_dsr_bytes = min(max_user_dsr_bytes, dsrbytes);
0291     }
0292 
0293     return 0;
0294 
0295 fail:
0296     for (bid--; bid >= 0; bid--)
0297         free_pages((unsigned long)gru_base[bid], order);
0298     return -ENOMEM;
0299 }
0300 
0301 static void gru_free_tables(void)
0302 {
0303     int bid;
0304     int order = get_order(sizeof(struct gru_state) *
0305                   GRU_CHIPLETS_PER_BLADE);
0306 
0307     for (bid = 0; bid < GRU_MAX_BLADES; bid++)
0308         free_pages((unsigned long)gru_base[bid], order);
0309 }
0310 
0311 static unsigned long gru_chiplet_cpu_to_mmr(int chiplet, int cpu, int *corep)
0312 {
0313     unsigned long mmr = 0;
0314     int core;
0315 
0316     /*
0317      * We target the cores of a blade and not the hyperthreads themselves.
0318      * There is a max of 8 cores per socket and 2 sockets per blade,
0319      * making for a max total of 16 cores (i.e., 16 CPUs without
0320      * hyperthreading and 32 CPUs with hyperthreading).
0321      */
0322     core = uv_cpu_core_number(cpu) + UV_MAX_INT_CORES * uv_cpu_socket_number(cpu);
0323     if (core >= GRU_NUM_TFM || uv_cpu_ht_number(cpu))
0324         return 0;
0325 
0326     if (chiplet == 0) {
0327         mmr = UVH_GR0_TLB_INT0_CONFIG +
0328             core * (UVH_GR0_TLB_INT1_CONFIG - UVH_GR0_TLB_INT0_CONFIG);
0329     } else if (chiplet == 1) {
0330         mmr = UVH_GR1_TLB_INT0_CONFIG +
0331             core * (UVH_GR1_TLB_INT1_CONFIG - UVH_GR1_TLB_INT0_CONFIG);
0332     } else {
0333         BUG();
0334     }
0335 
0336     *corep = core;
0337     return mmr;
0338 }
0339 
0340 #ifdef CONFIG_IA64
0341 
0342 static int gru_irq_count[GRU_CHIPLETS_PER_BLADE];
0343 
0344 static void gru_noop(struct irq_data *d)
0345 {
0346 }
0347 
0348 static struct irq_chip gru_chip[GRU_CHIPLETS_PER_BLADE] = {
0349     [0 ... GRU_CHIPLETS_PER_BLADE - 1] {
0350         .irq_mask   = gru_noop,
0351         .irq_unmask = gru_noop,
0352         .irq_ack    = gru_noop
0353     }
0354 };
0355 
0356 static int gru_chiplet_setup_tlb_irq(int chiplet, char *irq_name,
0357             irq_handler_t irq_handler, int cpu, int blade)
0358 {
0359     unsigned long mmr;
0360     int irq = IRQ_GRU + chiplet;
0361     int ret, core;
0362 
0363     mmr = gru_chiplet_cpu_to_mmr(chiplet, cpu, &core);
0364     if (mmr == 0)
0365         return 0;
0366 
0367     if (gru_irq_count[chiplet] == 0) {
0368         gru_chip[chiplet].name = irq_name;
0369         ret = irq_set_chip(irq, &gru_chip[chiplet]);
0370         if (ret) {
0371             printk(KERN_ERR "%s: set_irq_chip failed, errno=%d\n",
0372                    GRU_DRIVER_ID_STR, -ret);
0373             return ret;
0374         }
0375 
0376         ret = request_irq(irq, irq_handler, 0, irq_name, NULL);
0377         if (ret) {
0378             printk(KERN_ERR "%s: request_irq failed, errno=%d\n",
0379                    GRU_DRIVER_ID_STR, -ret);
0380             return ret;
0381         }
0382     }
0383     gru_irq_count[chiplet]++;
0384 
0385     return 0;
0386 }
0387 
0388 static void gru_chiplet_teardown_tlb_irq(int chiplet, int cpu, int blade)
0389 {
0390     unsigned long mmr;
0391     int core, irq = IRQ_GRU + chiplet;
0392 
0393     if (gru_irq_count[chiplet] == 0)
0394         return;
0395 
0396     mmr = gru_chiplet_cpu_to_mmr(chiplet, cpu, &core);
0397     if (mmr == 0)
0398         return;
0399 
0400     if (--gru_irq_count[chiplet] == 0)
0401         free_irq(irq, NULL);
0402 }
0403 
0404 #elif defined CONFIG_X86_64
0405 
0406 static int gru_chiplet_setup_tlb_irq(int chiplet, char *irq_name,
0407             irq_handler_t irq_handler, int cpu, int blade)
0408 {
0409     unsigned long mmr;
0410     int irq, core;
0411     int ret;
0412 
0413     mmr = gru_chiplet_cpu_to_mmr(chiplet, cpu, &core);
0414     if (mmr == 0)
0415         return 0;
0416 
0417     irq = uv_setup_irq(irq_name, cpu, blade, mmr, UV_AFFINITY_CPU);
0418     if (irq < 0) {
0419         printk(KERN_ERR "%s: uv_setup_irq failed, errno=%d\n",
0420                GRU_DRIVER_ID_STR, -irq);
0421         return irq;
0422     }
0423 
0424     ret = request_irq(irq, irq_handler, 0, irq_name, NULL);
0425     if (ret) {
0426         uv_teardown_irq(irq);
0427         printk(KERN_ERR "%s: request_irq failed, errno=%d\n",
0428                GRU_DRIVER_ID_STR, -ret);
0429         return ret;
0430     }
0431     gru_base[blade]->bs_grus[chiplet].gs_irq[core] = irq;
0432     return 0;
0433 }
0434 
0435 static void gru_chiplet_teardown_tlb_irq(int chiplet, int cpu, int blade)
0436 {
0437     int irq, core;
0438     unsigned long mmr;
0439 
0440     mmr = gru_chiplet_cpu_to_mmr(chiplet, cpu, &core);
0441     if (mmr) {
0442         irq = gru_base[blade]->bs_grus[chiplet].gs_irq[core];
0443         if (irq) {
0444             free_irq(irq, NULL);
0445             uv_teardown_irq(irq);
0446         }
0447     }
0448 }
0449 
0450 #endif
0451 
0452 static void gru_teardown_tlb_irqs(void)
0453 {
0454     int blade;
0455     int cpu;
0456 
0457     for_each_online_cpu(cpu) {
0458         blade = uv_cpu_to_blade_id(cpu);
0459         gru_chiplet_teardown_tlb_irq(0, cpu, blade);
0460         gru_chiplet_teardown_tlb_irq(1, cpu, blade);
0461     }
0462     for_each_possible_blade(blade) {
0463         if (uv_blade_nr_possible_cpus(blade))
0464             continue;
0465         gru_chiplet_teardown_tlb_irq(0, 0, blade);
0466         gru_chiplet_teardown_tlb_irq(1, 0, blade);
0467     }
0468 }
0469 
0470 static int gru_setup_tlb_irqs(void)
0471 {
0472     int blade;
0473     int cpu;
0474     int ret;
0475 
0476     for_each_online_cpu(cpu) {
0477         blade = uv_cpu_to_blade_id(cpu);
0478         ret = gru_chiplet_setup_tlb_irq(0, "GRU0_TLB", gru0_intr, cpu, blade);
0479         if (ret != 0)
0480             goto exit1;
0481 
0482         ret = gru_chiplet_setup_tlb_irq(1, "GRU1_TLB", gru1_intr, cpu, blade);
0483         if (ret != 0)
0484             goto exit1;
0485     }
0486     for_each_possible_blade(blade) {
0487         if (uv_blade_nr_possible_cpus(blade))
0488             continue;
0489         ret = gru_chiplet_setup_tlb_irq(0, "GRU0_TLB", gru_intr_mblade, 0, blade);
0490         if (ret != 0)
0491             goto exit1;
0492 
0493         ret = gru_chiplet_setup_tlb_irq(1, "GRU1_TLB", gru_intr_mblade, 0, blade);
0494         if (ret != 0)
0495             goto exit1;
0496     }
0497 
0498     return 0;
0499 
0500 exit1:
0501     gru_teardown_tlb_irqs();
0502     return ret;
0503 }
0504 
0505 /*
0506  * gru_init
0507  *
0508  * Called at boot or module load time to initialize the GRUs.
0509  */
0510 static int __init gru_init(void)
0511 {
0512     int ret;
0513 
0514     if (!gru_supported())
0515         return 0;
0516 
0517 #if defined CONFIG_IA64
0518     gru_start_paddr = 0xd000000000UL; /* ZZZZZZZZZZZZZZZZZZZ fixme */
0519 #else
0520     gru_start_paddr = uv_read_local_mmr(UVH_RH_GAM_GRU_OVERLAY_CONFIG) &
0521                 0x7fffffffffffUL;
0522 #endif
0523     gru_start_vaddr = __va(gru_start_paddr);
0524     gru_end_paddr = gru_start_paddr + GRU_MAX_BLADES * GRU_SIZE;
0525     printk(KERN_INFO "GRU space: 0x%lx - 0x%lx\n",
0526            gru_start_paddr, gru_end_paddr);
0527     ret = misc_register(&gru_miscdev);
0528     if (ret) {
0529         printk(KERN_ERR "%s: misc_register failed\n",
0530                GRU_DRIVER_ID_STR);
0531         goto exit0;
0532     }
0533 
0534     ret = gru_proc_init();
0535     if (ret) {
0536         printk(KERN_ERR "%s: proc init failed\n", GRU_DRIVER_ID_STR);
0537         goto exit1;
0538     }
0539 
0540     ret = gru_init_tables(gru_start_paddr, gru_start_vaddr);
0541     if (ret) {
0542         printk(KERN_ERR "%s: init tables failed\n", GRU_DRIVER_ID_STR);
0543         goto exit2;
0544     }
0545 
0546     ret = gru_setup_tlb_irqs();
0547     if (ret != 0)
0548         goto exit3;
0549 
0550     gru_kservices_init();
0551 
0552     printk(KERN_INFO "%s: v%s\n", GRU_DRIVER_ID_STR,
0553            GRU_DRIVER_VERSION_STR);
0554     return 0;
0555 
0556 exit3:
0557     gru_free_tables();
0558 exit2:
0559     gru_proc_exit();
0560 exit1:
0561     misc_deregister(&gru_miscdev);
0562 exit0:
0563     return ret;
0564 
0565 }
0566 
0567 static void __exit gru_exit(void)
0568 {
0569     if (!gru_supported())
0570         return;
0571 
0572     gru_teardown_tlb_irqs();
0573     gru_kservices_exit();
0574     gru_free_tables();
0575     misc_deregister(&gru_miscdev);
0576     gru_proc_exit();
0577     mmu_notifier_synchronize();
0578 }
0579 
0580 static const struct file_operations gru_fops = {
0581     .owner      = THIS_MODULE,
0582     .unlocked_ioctl = gru_file_unlocked_ioctl,
0583     .mmap       = gru_file_mmap,
0584     .llseek     = noop_llseek,
0585 };
0586 
0587 static struct miscdevice gru_miscdev = {
0588     .minor      = MISC_DYNAMIC_MINOR,
0589     .name       = "gru",
0590     .fops       = &gru_fops,
0591 };
0592 
0593 const struct vm_operations_struct gru_vm_ops = {
0594     .close      = gru_vma_close,
0595     .fault      = gru_fault,
0596 };
0597 
0598 #ifndef MODULE
0599 fs_initcall(gru_init);
0600 #else
0601 module_init(gru_init);
0602 #endif
0603 module_exit(gru_exit);
0604 
0605 module_param(gru_options, ulong, 0644);
0606 MODULE_PARM_DESC(gru_options, "Various debug options");
0607 
0608 MODULE_AUTHOR("Silicon Graphics, Inc.");
0609 MODULE_LICENSE("GPL");
0610 MODULE_DESCRIPTION(GRU_DRIVER_ID_STR GRU_DRIVER_VERSION_STR);
0611 MODULE_VERSION(GRU_DRIVER_VERSION_STR);
0612
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!