platforms/powernv/vas-fault.c

0001 // SPDX-License-Identifier: GPL-2.0+
0002 /*
0003  * VAS Fault handling.
0004  * Copyright 2019, IBM Corporation
0005  */
0006
0007 #define pr_fmt(fmt) "vas: " fmt
0008
0009 #include <linux/kernel.h>
0010 #include <linux/types.h>
0011 #include <linux/slab.h>
0012 #include <linux/uaccess.h>
0013 #include <linux/kthread.h>
0014 #include <linux/sched/signal.h>
0015 #include <linux/mmu_context.h>
0016 #include <asm/icswx.h>
0017
0018 #include "vas.h"
0019
0020 /*
0021  * The maximum FIFO size for fault window can be 8MB
0022  * (VAS_RX_FIFO_SIZE_MAX). Using 4MB FIFO since each VAS
0023  * instance will be having fault window.
0024  * 8MB FIFO can be used if expects more faults for each VAS
0025  * instance.
0026  */
0027 #define VAS_FAULT_WIN_FIFO_SIZE (4 << 20)
0028
0029 static void dump_fifo(struct vas_instance *vinst, void *entry)
0030 {
0031     unsigned long *end = vinst->fault_fifo + vinst->fault_fifo_size;
0032     unsigned long *fifo = entry;
0033     int i;
0034
0035     pr_err("Fault fifo size %d, Max crbs %d\n", vinst->fault_fifo_size,
0036             vinst->fault_fifo_size / CRB_SIZE);
0037
0038     /* Dump 10 CRB entries or until end of FIFO */
0039     pr_err("Fault FIFO Dump:\n");
0040     for (i = 0; i < 10*(CRB_SIZE/8) && fifo < end; i += 4, fifo += 4) {
0041         pr_err("[%.3d, %p]: 0x%.16lx 0x%.16lx 0x%.16lx 0x%.16lx\n",
0042             i, fifo, *fifo, *(fifo+1), *(fifo+2), *(fifo+3));
0043     }
0044 }
0045
0046 /*
0047  * Process valid CRBs in fault FIFO.
0048  * NX process user space requests, return credit and update the status
0049  * in CRB. If it encounters transalation error when accessing CRB or
0050  * request buffers, raises interrupt on the CPU to handle the fault.
0051  * It takes credit on fault window, updates nx_fault_stamp in CRB with
0052  * the following information and pastes CRB in fault FIFO.
0053  *
0054  * pswid - window ID of the window on which the request is sent.
0055  * fault_storage_addr - fault address
0056  *
0057  * It can raise a single interrupt for multiple faults. Expects OS to
0058  * process all valid faults and return credit for each fault on user
0059  * space and fault windows. This fault FIFO control will be done with
0060  * credit mechanism. NX can continuously paste CRBs until credits are not
0061  * available on fault window. Otherwise, returns with RMA_reject.
0062  *
0063  * Total credits available on fault window: FIFO_SIZE(4MB)/CRBS_SIZE(128)
0064  *
0065  */
0066 irqreturn_t vas_fault_thread_fn(int irq, void *data)
0067 {
0068     struct vas_instance *vinst = data;
0069     struct coprocessor_request_block *crb, *entry;
0070     struct coprocessor_request_block buf;
0071     struct pnv_vas_window *window;
0072     unsigned long flags;
0073     void *fifo;
0074
0075     crb = &buf;
0076
0077     /*
0078      * VAS can interrupt with multiple page faults. So process all
0079      * valid CRBs within fault FIFO until reaches invalid CRB.
0080      * We use CCW[0] and pswid to validate CRBs:
0081      *
0082      * CCW[0]   Reserved bit. When NX pastes CRB, CCW[0]=0
0083      *      OS sets this bit to 1 after reading CRB.
0084      * pswid    NX assigns window ID. Set pswid to -1 after
0085      *      reading CRB from fault FIFO.
0086      *
0087      * We exit this function if no valid CRBs are available to process.
0088      * So acquire fault_lock and reset fifo_in_progress to 0 before
0089      * exit.
0090      * In case kernel receives another interrupt with different page
0091      * fault, interrupt handler returns with IRQ_HANDLED if
0092      * fifo_in_progress is set. Means these new faults will be
0093      * handled by the current thread. Otherwise set fifo_in_progress
0094      * and return IRQ_WAKE_THREAD to wake up thread.
0095      */
0096     while (true) {
0097         spin_lock_irqsave(&vinst->fault_lock, flags);
0098         /*
0099          * Advance the fault fifo pointer to next CRB.
0100          * Use CRB_SIZE rather than sizeof(*crb) since the latter is
0101          * aligned to CRB_ALIGN (256) but the CRB written to by VAS is
0102          * only CRB_SIZE in len.
0103          */
0104         fifo = vinst->fault_fifo + (vinst->fault_crbs * CRB_SIZE);
0105         entry = fifo;
0106
0107         if ((entry->stamp.nx.pswid == cpu_to_be32(FIFO_INVALID_ENTRY))
0108             || (entry->ccw & cpu_to_be32(CCW0_INVALID))) {
0109             vinst->fifo_in_progress = 0;
0110             spin_unlock_irqrestore(&vinst->fault_lock, flags);
0111             return IRQ_HANDLED;
0112         }
0113
0114         spin_unlock_irqrestore(&vinst->fault_lock, flags);
0115         vinst->fault_crbs++;
0116         if (vinst->fault_crbs == (vinst->fault_fifo_size / CRB_SIZE))
0117             vinst->fault_crbs = 0;
0118
0119         memcpy(crb, fifo, CRB_SIZE);
0120         entry->stamp.nx.pswid = cpu_to_be32(FIFO_INVALID_ENTRY);
0121         entry->ccw |= cpu_to_be32(CCW0_INVALID);
0122         /*
0123          * Return credit for the fault window.
0124          */
0125         vas_return_credit(vinst->fault_win, false);
0126
0127         pr_devel("VAS[%d] fault_fifo %p, fifo %p, fault_crbs %d\n",
0128                 vinst->vas_id, vinst->fault_fifo, fifo,
0129                 vinst->fault_crbs);
0130
0131         vas_dump_crb(crb);
0132         window = vas_pswid_to_window(vinst,
0133                 be32_to_cpu(crb->stamp.nx.pswid));
0134
0135         if (IS_ERR(window)) {
0136             /*
0137              * We got an interrupt about a specific send
0138              * window but we can't find that window and we can't
0139              * even clean it up (return credit on user space
0140              * window).
0141              * But we should not get here.
0142              * TODO: Disable IRQ.
0143              */
0144             dump_fifo(vinst, (void *)entry);
0145             pr_err("VAS[%d] fault_fifo %p, fifo %p, pswid 0x%x, fault_crbs %d bad CRB?\n",
0146                 vinst->vas_id, vinst->fault_fifo, fifo,
0147                 be32_to_cpu(crb->stamp.nx.pswid),
0148                 vinst->fault_crbs);
0149
0150             WARN_ON_ONCE(1);
0151         } else {
0152             /*
0153              * NX sees faults only with user space windows.
0154              */
0155             if (window->user_win)
0156                 vas_update_csb(crb, &window->vas_win.task_ref);
0157             else
0158                 WARN_ON_ONCE(!window->user_win);
0159
0160             /*
0161              * Return credit for send window after processing
0162              * fault CRB.
0163              */
0164             vas_return_credit(window, true);
0165         }
0166     }
0167 }
0168
0169 irqreturn_t vas_fault_handler(int irq, void *dev_id)
0170 {
0171     struct vas_instance *vinst = dev_id;
0172     irqreturn_t ret = IRQ_WAKE_THREAD;
0173     unsigned long flags;
0174
0175     /*
0176      * NX can generate an interrupt for multiple faults. So the
0177      * fault handler thread process all CRBs until finds invalid
0178      * entry. In case if NX sees continuous faults, it is possible
0179      * that the thread function entered with the first interrupt
0180      * can execute and process all valid CRBs.
0181      * So wake up thread only if the fault thread is not in progress.
0182      */
0183     spin_lock_irqsave(&vinst->fault_lock, flags);
0184
0185     if (vinst->fifo_in_progress)
0186         ret = IRQ_HANDLED;
0187     else
0188         vinst->fifo_in_progress = 1;
0189
0190     spin_unlock_irqrestore(&vinst->fault_lock, flags);
0191
0192     return ret;
0193 }
0194
0195 /*
0196  * Fault window is opened per VAS instance. NX pastes fault CRB in fault
0197  * FIFO upon page faults.
0198  */
0199 int vas_setup_fault_window(struct vas_instance *vinst)
0200 {
0201     struct vas_rx_win_attr attr;
0202     struct vas_window *win;
0203
0204     vinst->fault_fifo_size = VAS_FAULT_WIN_FIFO_SIZE;
0205     vinst->fault_fifo = kzalloc(vinst->fault_fifo_size, GFP_KERNEL);
0206     if (!vinst->fault_fifo) {
0207         pr_err("Unable to alloc %d bytes for fault_fifo\n",
0208                 vinst->fault_fifo_size);
0209         return -ENOMEM;
0210     }
0211
0212     /*
0213      * Invalidate all CRB entries. NX pastes valid entry for each fault.
0214      */
0215     memset(vinst->fault_fifo, FIFO_INVALID_ENTRY, vinst->fault_fifo_size);
0216     vas_init_rx_win_attr(&attr, VAS_COP_TYPE_FAULT);
0217
0218     attr.rx_fifo_size = vinst->fault_fifo_size;
0219     attr.rx_fifo = __pa(vinst->fault_fifo);
0220
0221     /*
0222      * Max creds is based on number of CRBs can fit in the FIFO.
0223      * (fault_fifo_size/CRB_SIZE). If 8MB FIFO is used, max creds
0224      * will be 0xffff since the receive creds field is 16bits wide.
0225      */
0226     attr.wcreds_max = vinst->fault_fifo_size / CRB_SIZE;
0227     attr.lnotify_lpid = 0;
0228     attr.lnotify_pid = mfspr(SPRN_PID);
0229     attr.lnotify_tid = mfspr(SPRN_PID);
0230
0231     win = vas_rx_win_open(vinst->vas_id, VAS_COP_TYPE_FAULT, &attr);
0232     if (IS_ERR(win)) {
0233         pr_err("VAS: Error %ld opening FaultWin\n", PTR_ERR(win));
0234         kfree(vinst->fault_fifo);
0235         return PTR_ERR(win);
0236     }
0237
0238     vinst->fault_win = container_of(win, struct pnv_vas_window, vas_win);
0239
0240     pr_devel("VAS: Created FaultWin %d, LPID/PID/TID [%d/%d/%d]\n",
0241             vinst->fault_win->vas_win.winid, attr.lnotify_lpid,
0242             attr.lnotify_pid, attr.lnotify_tid);
0243
0244     return 0;
0245 }