Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * Copyright 2016-17 IBM Corp.
0004  */
0005 
0006 #define pr_fmt(fmt) "vas: " fmt
0007 
0008 #include <linux/types.h>
0009 #include <linux/mutex.h>
0010 #include <linux/slab.h>
0011 #include <linux/io.h>
0012 #include <linux/log2.h>
0013 #include <linux/rcupdate.h>
0014 #include <linux/cred.h>
0015 #include <linux/sched/mm.h>
0016 #include <linux/mmu_context.h>
0017 #include <asm/switch_to.h>
0018 #include <asm/ppc-opcode.h>
0019 #include <asm/vas.h>
0020 #include "vas.h"
0021 #include "copy-paste.h"
0022 
0023 #define CREATE_TRACE_POINTS
0024 #include "vas-trace.h"
0025 
0026 /*
0027  * Compute the paste address region for the window @window using the
0028  * ->paste_base_addr and ->paste_win_id_shift we got from device tree.
0029  */
0030 void vas_win_paste_addr(struct pnv_vas_window *window, u64 *addr, int *len)
0031 {
0032     int winid;
0033     u64 base, shift;
0034 
0035     base = window->vinst->paste_base_addr;
0036     shift = window->vinst->paste_win_id_shift;
0037     winid = window->vas_win.winid;
0038 
0039     *addr  = base + (winid << shift);
0040     if (len)
0041         *len = PAGE_SIZE;
0042 
0043     pr_debug("Txwin #%d: Paste addr 0x%llx\n", winid, *addr);
0044 }
0045 
0046 static inline void get_hvwc_mmio_bar(struct pnv_vas_window *window,
0047             u64 *start, int *len)
0048 {
0049     u64 pbaddr;
0050 
0051     pbaddr = window->vinst->hvwc_bar_start;
0052     *start = pbaddr + window->vas_win.winid * VAS_HVWC_SIZE;
0053     *len = VAS_HVWC_SIZE;
0054 }
0055 
0056 static inline void get_uwc_mmio_bar(struct pnv_vas_window *window,
0057             u64 *start, int *len)
0058 {
0059     u64 pbaddr;
0060 
0061     pbaddr = window->vinst->uwc_bar_start;
0062     *start = pbaddr + window->vas_win.winid * VAS_UWC_SIZE;
0063     *len = VAS_UWC_SIZE;
0064 }
0065 
0066 /*
0067  * Map the paste bus address of the given send window into kernel address
0068  * space. Unlike MMIO regions (map_mmio_region() below), paste region must
0069  * be mapped cache-able and is only applicable to send windows.
0070  */
0071 static void *map_paste_region(struct pnv_vas_window *txwin)
0072 {
0073     int len;
0074     void *map;
0075     char *name;
0076     u64 start;
0077 
0078     name = kasprintf(GFP_KERNEL, "window-v%d-w%d", txwin->vinst->vas_id,
0079                 txwin->vas_win.winid);
0080     if (!name)
0081         goto free_name;
0082 
0083     txwin->paste_addr_name = name;
0084     vas_win_paste_addr(txwin, &start, &len);
0085 
0086     if (!request_mem_region(start, len, name)) {
0087         pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n",
0088                 __func__, start, len);
0089         goto free_name;
0090     }
0091 
0092     map = ioremap_cache(start, len);
0093     if (!map) {
0094         pr_devel("%s(): ioremap_cache(0x%llx, %d) failed\n", __func__,
0095                 start, len);
0096         goto free_name;
0097     }
0098 
0099     pr_devel("Mapped paste addr 0x%llx to kaddr 0x%p\n", start, map);
0100     return map;
0101 
0102 free_name:
0103     kfree(name);
0104     return ERR_PTR(-ENOMEM);
0105 }
0106 
0107 static void *map_mmio_region(char *name, u64 start, int len)
0108 {
0109     void *map;
0110 
0111     if (!request_mem_region(start, len, name)) {
0112         pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n",
0113                 __func__, start, len);
0114         return NULL;
0115     }
0116 
0117     map = ioremap(start, len);
0118     if (!map) {
0119         pr_devel("%s(): ioremap(0x%llx, %d) failed\n", __func__, start,
0120                 len);
0121         return NULL;
0122     }
0123 
0124     return map;
0125 }
0126 
0127 static void unmap_region(void *addr, u64 start, int len)
0128 {
0129     iounmap(addr);
0130     release_mem_region((phys_addr_t)start, len);
0131 }
0132 
0133 /*
0134  * Unmap the paste address region for a window.
0135  */
0136 static void unmap_paste_region(struct pnv_vas_window *window)
0137 {
0138     int len;
0139     u64 busaddr_start;
0140 
0141     if (window->paste_kaddr) {
0142         vas_win_paste_addr(window, &busaddr_start, &len);
0143         unmap_region(window->paste_kaddr, busaddr_start, len);
0144         window->paste_kaddr = NULL;
0145         kfree(window->paste_addr_name);
0146         window->paste_addr_name = NULL;
0147     }
0148 }
0149 
0150 /*
0151  * Unmap the MMIO regions for a window. Hold the vas_mutex so we don't
0152  * unmap when the window's debugfs dir is in use. This serializes close
0153  * of a window even on another VAS instance but since its not a critical
0154  * path, just minimize the time we hold the mutex for now. We can add
0155  * a per-instance mutex later if necessary.
0156  */
0157 static void unmap_winctx_mmio_bars(struct pnv_vas_window *window)
0158 {
0159     int len;
0160     void *uwc_map;
0161     void *hvwc_map;
0162     u64 busaddr_start;
0163 
0164     mutex_lock(&vas_mutex);
0165 
0166     hvwc_map = window->hvwc_map;
0167     window->hvwc_map = NULL;
0168 
0169     uwc_map = window->uwc_map;
0170     window->uwc_map = NULL;
0171 
0172     mutex_unlock(&vas_mutex);
0173 
0174     if (hvwc_map) {
0175         get_hvwc_mmio_bar(window, &busaddr_start, &len);
0176         unmap_region(hvwc_map, busaddr_start, len);
0177     }
0178 
0179     if (uwc_map) {
0180         get_uwc_mmio_bar(window, &busaddr_start, &len);
0181         unmap_region(uwc_map, busaddr_start, len);
0182     }
0183 }
0184 
0185 /*
0186  * Find the Hypervisor Window Context (HVWC) MMIO Base Address Region and the
0187  * OS/User Window Context (UWC) MMIO Base Address Region for the given window.
0188  * Map these bus addresses and save the mapped kernel addresses in @window.
0189  */
0190 static int map_winctx_mmio_bars(struct pnv_vas_window *window)
0191 {
0192     int len;
0193     u64 start;
0194 
0195     get_hvwc_mmio_bar(window, &start, &len);
0196     window->hvwc_map = map_mmio_region("HVWCM_Window", start, len);
0197 
0198     get_uwc_mmio_bar(window, &start, &len);
0199     window->uwc_map = map_mmio_region("UWCM_Window", start, len);
0200 
0201     if (!window->hvwc_map || !window->uwc_map) {
0202         unmap_winctx_mmio_bars(window);
0203         return -1;
0204     }
0205 
0206     return 0;
0207 }
0208 
0209 /*
0210  * Reset all valid registers in the HV and OS/User Window Contexts for
0211  * the window identified by @window.
0212  *
0213  * NOTE: We cannot really use a for loop to reset window context. Not all
0214  *   offsets in a window context are valid registers and the valid
0215  *   registers are not sequential. And, we can only write to offsets
0216  *   with valid registers.
0217  */
0218 static void reset_window_regs(struct pnv_vas_window *window)
0219 {
0220     write_hvwc_reg(window, VREG(LPID), 0ULL);
0221     write_hvwc_reg(window, VREG(PID), 0ULL);
0222     write_hvwc_reg(window, VREG(XLATE_MSR), 0ULL);
0223     write_hvwc_reg(window, VREG(XLATE_LPCR), 0ULL);
0224     write_hvwc_reg(window, VREG(XLATE_CTL), 0ULL);
0225     write_hvwc_reg(window, VREG(AMR), 0ULL);
0226     write_hvwc_reg(window, VREG(SEIDR), 0ULL);
0227     write_hvwc_reg(window, VREG(FAULT_TX_WIN), 0ULL);
0228     write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL);
0229     write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), 0ULL);
0230     write_hvwc_reg(window, VREG(PSWID), 0ULL);
0231     write_hvwc_reg(window, VREG(LFIFO_BAR), 0ULL);
0232     write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), 0ULL);
0233     write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), 0ULL);
0234     write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL);
0235     write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL);
0236     write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL);
0237     write_hvwc_reg(window, VREG(LRX_WCRED), 0ULL);
0238     write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL);
0239     write_hvwc_reg(window, VREG(TX_WCRED), 0ULL);
0240     write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL);
0241     write_hvwc_reg(window, VREG(LFIFO_SIZE), 0ULL);
0242     write_hvwc_reg(window, VREG(WINCTL), 0ULL);
0243     write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL);
0244     write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), 0ULL);
0245     write_hvwc_reg(window, VREG(TX_RSVD_BUF_COUNT), 0ULL);
0246     write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), 0ULL);
0247     write_hvwc_reg(window, VREG(LNOTIFY_CTL), 0ULL);
0248     write_hvwc_reg(window, VREG(LNOTIFY_PID), 0ULL);
0249     write_hvwc_reg(window, VREG(LNOTIFY_LPID), 0ULL);
0250     write_hvwc_reg(window, VREG(LNOTIFY_TID), 0ULL);
0251     write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), 0ULL);
0252     write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL);
0253 
0254     /* Skip read-only registers: NX_UTIL and NX_UTIL_SE */
0255 
0256     /*
0257      * The send and receive window credit adder registers are also
0258      * accessible from HVWC and have been initialized above. We don't
0259      * need to initialize from the OS/User Window Context, so skip
0260      * following calls:
0261      *
0262      *  write_uwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL);
0263      *  write_uwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL);
0264      */
0265 }
0266 
0267 /*
0268  * Initialize window context registers related to Address Translation.
0269  * These registers are common to send/receive windows although they
0270  * differ for user/kernel windows. As we resolve the TODOs we may
0271  * want to add fields to vas_winctx and move the initialization to
0272  * init_vas_winctx_regs().
0273  */
0274 static void init_xlate_regs(struct pnv_vas_window *window, bool user_win)
0275 {
0276     u64 lpcr, val;
0277 
0278     /*
0279      * MSR_TA, MSR_US are false for both kernel and user.
0280      * MSR_DR and MSR_PR are false for kernel.
0281      */
0282     val = 0ULL;
0283     val = SET_FIELD(VAS_XLATE_MSR_HV, val, 1);
0284     val = SET_FIELD(VAS_XLATE_MSR_SF, val, 1);
0285     if (user_win) {
0286         val = SET_FIELD(VAS_XLATE_MSR_DR, val, 1);
0287         val = SET_FIELD(VAS_XLATE_MSR_PR, val, 1);
0288     }
0289     write_hvwc_reg(window, VREG(XLATE_MSR), val);
0290 
0291     lpcr = mfspr(SPRN_LPCR);
0292     val = 0ULL;
0293     /*
0294      * NOTE: From Section 5.7.8.1 Segment Lookaside Buffer of the
0295      *   Power ISA, v3.0B, Page size encoding is 0 = 4KB, 5 = 64KB.
0296      *
0297      * NOTE: From Section 1.3.1, Address Translation Context of the
0298      *   Nest MMU Workbook, LPCR_SC should be 0 for Power9.
0299      */
0300     val = SET_FIELD(VAS_XLATE_LPCR_PAGE_SIZE, val, 5);
0301     val = SET_FIELD(VAS_XLATE_LPCR_ISL, val, lpcr & LPCR_ISL);
0302     val = SET_FIELD(VAS_XLATE_LPCR_TC, val, lpcr & LPCR_TC);
0303     val = SET_FIELD(VAS_XLATE_LPCR_SC, val, 0);
0304     write_hvwc_reg(window, VREG(XLATE_LPCR), val);
0305 
0306     /*
0307      * Section 1.3.1 (Address translation Context) of NMMU workbook.
0308      *  0b00    Hashed Page Table mode
0309      *  0b01    Reserved
0310      *  0b10    Radix on HPT
0311      *  0b11    Radix on Radix
0312      */
0313     val = 0ULL;
0314     val = SET_FIELD(VAS_XLATE_MODE, val, radix_enabled() ? 3 : 2);
0315     write_hvwc_reg(window, VREG(XLATE_CTL), val);
0316 
0317     /*
0318      * TODO: Can we mfspr(AMR) even for user windows?
0319      */
0320     val = 0ULL;
0321     val = SET_FIELD(VAS_AMR, val, mfspr(SPRN_AMR));
0322     write_hvwc_reg(window, VREG(AMR), val);
0323 
0324     val = 0ULL;
0325     val = SET_FIELD(VAS_SEIDR, val, 0);
0326     write_hvwc_reg(window, VREG(SEIDR), val);
0327 }
0328 
0329 /*
0330  * Initialize Reserved Send Buffer Count for the send window. It involves
0331  * writing to the register, reading it back to confirm that the hardware
0332  * has enough buffers to reserve. See section 1.3.1.2.1 of VAS workbook.
0333  *
0334  * Since we can only make a best-effort attempt to fulfill the request,
0335  * we don't return any errors if we cannot.
0336  *
0337  * TODO: Reserved (aka dedicated) send buffers are not supported yet.
0338  */
0339 static void init_rsvd_tx_buf_count(struct pnv_vas_window *txwin,
0340                 struct vas_winctx *winctx)
0341 {
0342     write_hvwc_reg(txwin, VREG(TX_RSVD_BUF_COUNT), 0ULL);
0343 }
0344 
0345 /*
0346  * init_winctx_regs()
0347  *  Initialize window context registers for a receive window.
0348  *  Except for caching control and marking window open, the registers
0349  *  are initialized in the order listed in Section 3.1.4 (Window Context
0350  *  Cache Register Details) of the VAS workbook although they don't need
0351  *  to be.
0352  *
0353  * Design note: For NX receive windows, NX allocates the FIFO buffer in OPAL
0354  *  (so that it can get a large contiguous area) and passes that buffer
0355  *  to kernel via device tree. We now write that buffer address to the
0356  *  FIFO BAR. Would it make sense to do this all in OPAL? i.e have OPAL
0357  *  write the per-chip RX FIFO addresses to the windows during boot-up
0358  *  as a one-time task? That could work for NX but what about other
0359  *  receivers?  Let the receivers tell us the rx-fifo buffers for now.
0360  */
0361 static void init_winctx_regs(struct pnv_vas_window *window,
0362                  struct vas_winctx *winctx)
0363 {
0364     u64 val;
0365     int fifo_size;
0366 
0367     reset_window_regs(window);
0368 
0369     val = 0ULL;
0370     val = SET_FIELD(VAS_LPID, val, winctx->lpid);
0371     write_hvwc_reg(window, VREG(LPID), val);
0372 
0373     val = 0ULL;
0374     val = SET_FIELD(VAS_PID_ID, val, winctx->pidr);
0375     write_hvwc_reg(window, VREG(PID), val);
0376 
0377     init_xlate_regs(window, winctx->user_win);
0378 
0379     val = 0ULL;
0380     val = SET_FIELD(VAS_FAULT_TX_WIN, val, winctx->fault_win_id);
0381     write_hvwc_reg(window, VREG(FAULT_TX_WIN), val);
0382 
0383     /* In PowerNV, interrupts go to HV. */
0384     write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL);
0385 
0386     val = 0ULL;
0387     val = SET_FIELD(VAS_HV_INTR_SRC_RA, val, winctx->irq_port);
0388     write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), val);
0389 
0390     val = 0ULL;
0391     val = SET_FIELD(VAS_PSWID_EA_HANDLE, val, winctx->pswid);
0392     write_hvwc_reg(window, VREG(PSWID), val);
0393 
0394     write_hvwc_reg(window, VREG(SPARE1), 0ULL);
0395     write_hvwc_reg(window, VREG(SPARE2), 0ULL);
0396     write_hvwc_reg(window, VREG(SPARE3), 0ULL);
0397 
0398     /*
0399      * NOTE: VAS expects the FIFO address to be copied into the LFIFO_BAR
0400      *   register as is - do NOT shift the address into VAS_LFIFO_BAR
0401      *   bit fields! Ok to set the page migration select fields -
0402      *   VAS ignores the lower 10+ bits in the address anyway, because
0403      *   the minimum FIFO size is 1K?
0404      *
0405      * See also: Design note in function header.
0406      */
0407     val = winctx->rx_fifo;
0408     val = SET_FIELD(VAS_PAGE_MIGRATION_SELECT, val, 0);
0409     write_hvwc_reg(window, VREG(LFIFO_BAR), val);
0410 
0411     val = 0ULL;
0412     val = SET_FIELD(VAS_LDATA_STAMP, val, winctx->data_stamp);
0413     write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), val);
0414 
0415     val = 0ULL;
0416     val = SET_FIELD(VAS_LDMA_TYPE, val, winctx->dma_type);
0417     val = SET_FIELD(VAS_LDMA_FIFO_DISABLE, val, winctx->fifo_disable);
0418     write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), val);
0419 
0420     write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL);
0421     write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL);
0422     write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL);
0423 
0424     val = 0ULL;
0425     val = SET_FIELD(VAS_LRX_WCRED, val, winctx->wcreds_max);
0426     write_hvwc_reg(window, VREG(LRX_WCRED), val);
0427 
0428     val = 0ULL;
0429     val = SET_FIELD(VAS_TX_WCRED, val, winctx->wcreds_max);
0430     write_hvwc_reg(window, VREG(TX_WCRED), val);
0431 
0432     write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL);
0433     write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL);
0434 
0435     fifo_size = winctx->rx_fifo_size / 1024;
0436 
0437     val = 0ULL;
0438     val = SET_FIELD(VAS_LFIFO_SIZE, val, ilog2(fifo_size));
0439     write_hvwc_reg(window, VREG(LFIFO_SIZE), val);
0440 
0441     /* Update window control and caching control registers last so
0442      * we mark the window open only after fully initializing it and
0443      * pushing context to cache.
0444      */
0445 
0446     write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL);
0447 
0448     init_rsvd_tx_buf_count(window, winctx);
0449 
0450     /* for a send window, point to the matching receive window */
0451     val = 0ULL;
0452     val = SET_FIELD(VAS_LRX_WIN_ID, val, winctx->rx_win_id);
0453     write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), val);
0454 
0455     write_hvwc_reg(window, VREG(SPARE4), 0ULL);
0456 
0457     val = 0ULL;
0458     val = SET_FIELD(VAS_NOTIFY_DISABLE, val, winctx->notify_disable);
0459     val = SET_FIELD(VAS_INTR_DISABLE, val, winctx->intr_disable);
0460     val = SET_FIELD(VAS_NOTIFY_EARLY, val, winctx->notify_early);
0461     val = SET_FIELD(VAS_NOTIFY_OSU_INTR, val, winctx->notify_os_intr_reg);
0462     write_hvwc_reg(window, VREG(LNOTIFY_CTL), val);
0463 
0464     val = 0ULL;
0465     val = SET_FIELD(VAS_LNOTIFY_PID, val, winctx->lnotify_pid);
0466     write_hvwc_reg(window, VREG(LNOTIFY_PID), val);
0467 
0468     val = 0ULL;
0469     val = SET_FIELD(VAS_LNOTIFY_LPID, val, winctx->lnotify_lpid);
0470     write_hvwc_reg(window, VREG(LNOTIFY_LPID), val);
0471 
0472     val = 0ULL;
0473     val = SET_FIELD(VAS_LNOTIFY_TID, val, winctx->lnotify_tid);
0474     write_hvwc_reg(window, VREG(LNOTIFY_TID), val);
0475 
0476     val = 0ULL;
0477     val = SET_FIELD(VAS_LNOTIFY_MIN_SCOPE, val, winctx->min_scope);
0478     val = SET_FIELD(VAS_LNOTIFY_MAX_SCOPE, val, winctx->max_scope);
0479     write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), val);
0480 
0481     /* Skip read-only registers NX_UTIL and NX_UTIL_SE */
0482 
0483     write_hvwc_reg(window, VREG(SPARE5), 0ULL);
0484     write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL);
0485     write_hvwc_reg(window, VREG(SPARE6), 0ULL);
0486 
0487     /* Finally, push window context to memory and... */
0488     val = 0ULL;
0489     val = SET_FIELD(VAS_PUSH_TO_MEM, val, 1);
0490     write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), val);
0491 
0492     /* ... mark the window open for business */
0493     val = 0ULL;
0494     val = SET_FIELD(VAS_WINCTL_REJ_NO_CREDIT, val, winctx->rej_no_credit);
0495     val = SET_FIELD(VAS_WINCTL_PIN, val, winctx->pin_win);
0496     val = SET_FIELD(VAS_WINCTL_TX_WCRED_MODE, val, winctx->tx_wcred_mode);
0497     val = SET_FIELD(VAS_WINCTL_RX_WCRED_MODE, val, winctx->rx_wcred_mode);
0498     val = SET_FIELD(VAS_WINCTL_TX_WORD_MODE, val, winctx->tx_word_mode);
0499     val = SET_FIELD(VAS_WINCTL_RX_WORD_MODE, val, winctx->rx_word_mode);
0500     val = SET_FIELD(VAS_WINCTL_FAULT_WIN, val, winctx->fault_win);
0501     val = SET_FIELD(VAS_WINCTL_NX_WIN, val, winctx->nx_win);
0502     val = SET_FIELD(VAS_WINCTL_OPEN, val, 1);
0503     write_hvwc_reg(window, VREG(WINCTL), val);
0504 }
0505 
0506 static void vas_release_window_id(struct ida *ida, int winid)
0507 {
0508     ida_free(ida, winid);
0509 }
0510 
0511 static int vas_assign_window_id(struct ida *ida)
0512 {
0513     int winid = ida_alloc_max(ida, VAS_WINDOWS_PER_CHIP - 1, GFP_KERNEL);
0514 
0515     if (winid == -ENOSPC) {
0516         pr_err("Too many (%d) open windows\n", VAS_WINDOWS_PER_CHIP);
0517         return -EAGAIN;
0518     }
0519 
0520     return winid;
0521 }
0522 
0523 static void vas_window_free(struct pnv_vas_window *window)
0524 {
0525     struct vas_instance *vinst = window->vinst;
0526     int winid = window->vas_win.winid;
0527 
0528     unmap_winctx_mmio_bars(window);
0529 
0530     vas_window_free_dbgdir(window);
0531 
0532     kfree(window);
0533 
0534     vas_release_window_id(&vinst->ida, winid);
0535 }
0536 
0537 static struct pnv_vas_window *vas_window_alloc(struct vas_instance *vinst)
0538 {
0539     int winid;
0540     struct pnv_vas_window *window;
0541 
0542     winid = vas_assign_window_id(&vinst->ida);
0543     if (winid < 0)
0544         return ERR_PTR(winid);
0545 
0546     window = kzalloc(sizeof(*window), GFP_KERNEL);
0547     if (!window)
0548         goto out_free;
0549 
0550     window->vinst = vinst;
0551     window->vas_win.winid = winid;
0552 
0553     if (map_winctx_mmio_bars(window))
0554         goto out_free;
0555 
0556     vas_window_init_dbgdir(window);
0557 
0558     return window;
0559 
0560 out_free:
0561     kfree(window);
0562     vas_release_window_id(&vinst->ida, winid);
0563     return ERR_PTR(-ENOMEM);
0564 }
0565 
0566 static void put_rx_win(struct pnv_vas_window *rxwin)
0567 {
0568     /* Better not be a send window! */
0569     WARN_ON_ONCE(rxwin->tx_win);
0570 
0571     atomic_dec(&rxwin->num_txwins);
0572 }
0573 
0574 /*
0575  * Find the user space receive window given the @pswid.
0576  *      - We must have a valid vasid and it must belong to this instance.
0577  *        (so both send and receive windows are on the same VAS instance)
0578  *      - The window must refer to an OPEN, FTW, RECEIVE window.
0579  *
0580  * NOTE: We access ->windows[] table and assume that vinst->mutex is held.
0581  */
0582 static struct pnv_vas_window *get_user_rxwin(struct vas_instance *vinst,
0583                          u32 pswid)
0584 {
0585     int vasid, winid;
0586     struct pnv_vas_window *rxwin;
0587 
0588     decode_pswid(pswid, &vasid, &winid);
0589 
0590     if (vinst->vas_id != vasid)
0591         return ERR_PTR(-EINVAL);
0592 
0593     rxwin = vinst->windows[winid];
0594 
0595     if (!rxwin || rxwin->tx_win || rxwin->vas_win.cop != VAS_COP_TYPE_FTW)
0596         return ERR_PTR(-EINVAL);
0597 
0598     return rxwin;
0599 }
0600 
0601 /*
0602  * Get the VAS receive window associated with NX engine identified
0603  * by @cop and if applicable, @pswid.
0604  *
0605  * See also function header of set_vinst_win().
0606  */
0607 static struct pnv_vas_window *get_vinst_rxwin(struct vas_instance *vinst,
0608             enum vas_cop_type cop, u32 pswid)
0609 {
0610     struct pnv_vas_window *rxwin;
0611 
0612     mutex_lock(&vinst->mutex);
0613 
0614     if (cop == VAS_COP_TYPE_FTW)
0615         rxwin = get_user_rxwin(vinst, pswid);
0616     else
0617         rxwin = vinst->rxwin[cop] ?: ERR_PTR(-EINVAL);
0618 
0619     if (!IS_ERR(rxwin))
0620         atomic_inc(&rxwin->num_txwins);
0621 
0622     mutex_unlock(&vinst->mutex);
0623 
0624     return rxwin;
0625 }
0626 
0627 /*
0628  * We have two tables of windows in a VAS instance. The first one,
0629  * ->windows[], contains all the windows in the instance and allows
0630  * looking up a window by its id. It is used to look up send windows
0631  * during fault handling and receive windows when pairing user space
0632  * send/receive windows.
0633  *
0634  * The second table, ->rxwin[], contains receive windows that are
0635  * associated with NX engines. This table has VAS_COP_TYPE_MAX
0636  * entries and is used to look up a receive window by its
0637  * coprocessor type.
0638  *
0639  * Here, we save @window in the ->windows[] table. If it is a receive
0640  * window, we also save the window in the ->rxwin[] table.
0641  */
0642 static void set_vinst_win(struct vas_instance *vinst,
0643             struct pnv_vas_window *window)
0644 {
0645     int id = window->vas_win.winid;
0646 
0647     mutex_lock(&vinst->mutex);
0648 
0649     /*
0650      * There should only be one receive window for a coprocessor type
0651      * unless its a user (FTW) window.
0652      */
0653     if (!window->user_win && !window->tx_win) {
0654         WARN_ON_ONCE(vinst->rxwin[window->vas_win.cop]);
0655         vinst->rxwin[window->vas_win.cop] = window;
0656     }
0657 
0658     WARN_ON_ONCE(vinst->windows[id] != NULL);
0659     vinst->windows[id] = window;
0660 
0661     mutex_unlock(&vinst->mutex);
0662 }
0663 
0664 /*
0665  * Clear this window from the table(s) of windows for this VAS instance.
0666  * See also function header of set_vinst_win().
0667  */
0668 static void clear_vinst_win(struct pnv_vas_window *window)
0669 {
0670     int id = window->vas_win.winid;
0671     struct vas_instance *vinst = window->vinst;
0672 
0673     mutex_lock(&vinst->mutex);
0674 
0675     if (!window->user_win && !window->tx_win) {
0676         WARN_ON_ONCE(!vinst->rxwin[window->vas_win.cop]);
0677         vinst->rxwin[window->vas_win.cop] = NULL;
0678     }
0679 
0680     WARN_ON_ONCE(vinst->windows[id] != window);
0681     vinst->windows[id] = NULL;
0682 
0683     mutex_unlock(&vinst->mutex);
0684 }
0685 
0686 static void init_winctx_for_rxwin(struct pnv_vas_window *rxwin,
0687             struct vas_rx_win_attr *rxattr,
0688             struct vas_winctx *winctx)
0689 {
0690     /*
0691      * We first zero (memset()) all fields and only set non-zero fields.
0692      * Following fields are 0/false but maybe deserve a comment:
0693      *
0694      *  ->notify_os_intr_reg    In powerNV, send intrs to HV
0695      *  ->notify_disable    False for NX windows
0696      *  ->intr_disable      False for Fault Windows
0697      *  ->xtra_write        False for NX windows
0698      *  ->notify_early      NA for NX windows
0699      *  ->rsvd_txbuf_count  NA for Rx windows
0700      *  ->lpid, ->pid, ->tid    NA for Rx windows
0701      */
0702 
0703     memset(winctx, 0, sizeof(struct vas_winctx));
0704 
0705     winctx->rx_fifo = rxattr->rx_fifo;
0706     winctx->rx_fifo_size = rxattr->rx_fifo_size;
0707     winctx->wcreds_max = rxwin->vas_win.wcreds_max;
0708     winctx->pin_win = rxattr->pin_win;
0709 
0710     winctx->nx_win = rxattr->nx_win;
0711     winctx->fault_win = rxattr->fault_win;
0712     winctx->user_win = rxattr->user_win;
0713     winctx->rej_no_credit = rxattr->rej_no_credit;
0714     winctx->rx_word_mode = rxattr->rx_win_ord_mode;
0715     winctx->tx_word_mode = rxattr->tx_win_ord_mode;
0716     winctx->rx_wcred_mode = rxattr->rx_wcred_mode;
0717     winctx->tx_wcred_mode = rxattr->tx_wcred_mode;
0718     winctx->notify_early = rxattr->notify_early;
0719 
0720     if (winctx->nx_win) {
0721         winctx->data_stamp = true;
0722         winctx->intr_disable = true;
0723         winctx->pin_win = true;
0724 
0725         WARN_ON_ONCE(winctx->fault_win);
0726         WARN_ON_ONCE(!winctx->rx_word_mode);
0727         WARN_ON_ONCE(!winctx->tx_word_mode);
0728         WARN_ON_ONCE(winctx->notify_after_count);
0729     } else if (winctx->fault_win) {
0730         winctx->notify_disable = true;
0731     } else if (winctx->user_win) {
0732         /*
0733          * Section 1.8.1 Low Latency Core-Core Wake up of
0734          * the VAS workbook:
0735          *
0736          *      - disable credit checks ([tr]x_wcred_mode = false)
0737          *      - disable FIFO writes
0738          *      - enable ASB_Notify, disable interrupt
0739          */
0740         winctx->fifo_disable = true;
0741         winctx->intr_disable = true;
0742         winctx->rx_fifo = 0;
0743     }
0744 
0745     winctx->lnotify_lpid = rxattr->lnotify_lpid;
0746     winctx->lnotify_pid = rxattr->lnotify_pid;
0747     winctx->lnotify_tid = rxattr->lnotify_tid;
0748     winctx->pswid = rxattr->pswid;
0749     winctx->dma_type = VAS_DMA_TYPE_INJECT;
0750     winctx->tc_mode = rxattr->tc_mode;
0751 
0752     winctx->min_scope = VAS_SCOPE_LOCAL;
0753     winctx->max_scope = VAS_SCOPE_VECTORED_GROUP;
0754     if (rxwin->vinst->virq)
0755         winctx->irq_port = rxwin->vinst->irq_port;
0756 }
0757 
0758 static bool rx_win_args_valid(enum vas_cop_type cop,
0759             struct vas_rx_win_attr *attr)
0760 {
0761     pr_debug("Rxattr: fault %d, notify %d, intr %d, early %d, fifo %d\n",
0762             attr->fault_win, attr->notify_disable,
0763             attr->intr_disable, attr->notify_early,
0764             attr->rx_fifo_size);
0765 
0766     if (cop >= VAS_COP_TYPE_MAX)
0767         return false;
0768 
0769     if (cop != VAS_COP_TYPE_FTW &&
0770                 attr->rx_fifo_size < VAS_RX_FIFO_SIZE_MIN)
0771         return false;
0772 
0773     if (attr->rx_fifo_size > VAS_RX_FIFO_SIZE_MAX)
0774         return false;
0775 
0776     if (!attr->wcreds_max)
0777         return false;
0778 
0779     if (attr->nx_win) {
0780         /* cannot be fault or user window if it is nx */
0781         if (attr->fault_win || attr->user_win)
0782             return false;
0783         /*
0784          * Section 3.1.4.32: NX Windows must not disable notification,
0785          *  and must not enable interrupts or early notification.
0786          */
0787         if (attr->notify_disable || !attr->intr_disable ||
0788                 attr->notify_early)
0789             return false;
0790     } else if (attr->fault_win) {
0791         /* cannot be both fault and user window */
0792         if (attr->user_win)
0793             return false;
0794 
0795         /*
0796          * Section 3.1.4.32: Fault windows must disable notification
0797          *  but not interrupts.
0798          */
0799         if (!attr->notify_disable || attr->intr_disable)
0800             return false;
0801 
0802     } else if (attr->user_win) {
0803         /*
0804          * User receive windows are only for fast-thread-wakeup
0805          * (FTW). They don't need a FIFO and must disable interrupts
0806          */
0807         if (attr->rx_fifo || attr->rx_fifo_size || !attr->intr_disable)
0808             return false;
0809     } else {
0810         /* Rx window must be one of NX or Fault or User window. */
0811         return false;
0812     }
0813 
0814     return true;
0815 }
0816 
0817 void vas_init_rx_win_attr(struct vas_rx_win_attr *rxattr, enum vas_cop_type cop)
0818 {
0819     memset(rxattr, 0, sizeof(*rxattr));
0820 
0821     if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI ||
0822         cop == VAS_COP_TYPE_GZIP || cop == VAS_COP_TYPE_GZIP_HIPRI) {
0823         rxattr->pin_win = true;
0824         rxattr->nx_win = true;
0825         rxattr->fault_win = false;
0826         rxattr->intr_disable = true;
0827         rxattr->rx_wcred_mode = true;
0828         rxattr->tx_wcred_mode = true;
0829         rxattr->rx_win_ord_mode = true;
0830         rxattr->tx_win_ord_mode = true;
0831     } else if (cop == VAS_COP_TYPE_FAULT) {
0832         rxattr->pin_win = true;
0833         rxattr->fault_win = true;
0834         rxattr->notify_disable = true;
0835         rxattr->rx_wcred_mode = true;
0836         rxattr->rx_win_ord_mode = true;
0837         rxattr->rej_no_credit = true;
0838         rxattr->tc_mode = VAS_THRESH_DISABLED;
0839     } else if (cop == VAS_COP_TYPE_FTW) {
0840         rxattr->user_win = true;
0841         rxattr->intr_disable = true;
0842 
0843         /*
0844          * As noted in the VAS Workbook we disable credit checks.
0845          * If we enable credit checks in the future, we must also
0846          * implement a mechanism to return the user credits or new
0847          * paste operations will fail.
0848          */
0849     }
0850 }
0851 EXPORT_SYMBOL_GPL(vas_init_rx_win_attr);
0852 
0853 struct vas_window *vas_rx_win_open(int vasid, enum vas_cop_type cop,
0854             struct vas_rx_win_attr *rxattr)
0855 {
0856     struct pnv_vas_window *rxwin;
0857     struct vas_winctx winctx;
0858     struct vas_instance *vinst;
0859 
0860     trace_vas_rx_win_open(current, vasid, cop, rxattr);
0861 
0862     if (!rx_win_args_valid(cop, rxattr))
0863         return ERR_PTR(-EINVAL);
0864 
0865     vinst = find_vas_instance(vasid);
0866     if (!vinst) {
0867         pr_devel("vasid %d not found!\n", vasid);
0868         return ERR_PTR(-EINVAL);
0869     }
0870     pr_devel("Found instance %d\n", vasid);
0871 
0872     rxwin = vas_window_alloc(vinst);
0873     if (IS_ERR(rxwin)) {
0874         pr_devel("Unable to allocate memory for Rx window\n");
0875         return (struct vas_window *)rxwin;
0876     }
0877 
0878     rxwin->tx_win = false;
0879     rxwin->nx_win = rxattr->nx_win;
0880     rxwin->user_win = rxattr->user_win;
0881     rxwin->vas_win.cop = cop;
0882     rxwin->vas_win.wcreds_max = rxattr->wcreds_max;
0883 
0884     init_winctx_for_rxwin(rxwin, rxattr, &winctx);
0885     init_winctx_regs(rxwin, &winctx);
0886 
0887     set_vinst_win(vinst, rxwin);
0888 
0889     return &rxwin->vas_win;
0890 }
0891 EXPORT_SYMBOL_GPL(vas_rx_win_open);
0892 
0893 void vas_init_tx_win_attr(struct vas_tx_win_attr *txattr, enum vas_cop_type cop)
0894 {
0895     memset(txattr, 0, sizeof(*txattr));
0896 
0897     if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI ||
0898         cop == VAS_COP_TYPE_GZIP || cop == VAS_COP_TYPE_GZIP_HIPRI) {
0899         txattr->rej_no_credit = false;
0900         txattr->rx_wcred_mode = true;
0901         txattr->tx_wcred_mode = true;
0902         txattr->rx_win_ord_mode = true;
0903         txattr->tx_win_ord_mode = true;
0904     } else if (cop == VAS_COP_TYPE_FTW) {
0905         txattr->user_win = true;
0906     }
0907 }
0908 EXPORT_SYMBOL_GPL(vas_init_tx_win_attr);
0909 
0910 static void init_winctx_for_txwin(struct pnv_vas_window *txwin,
0911             struct vas_tx_win_attr *txattr,
0912             struct vas_winctx *winctx)
0913 {
0914     /*
0915      * We first zero all fields and only set non-zero ones. Following
0916      * are some fields set to 0/false for the stated reason:
0917      *
0918      *  ->notify_os_intr_reg    In powernv, send intrs to HV
0919      *  ->rsvd_txbuf_count  Not supported yet.
0920      *  ->notify_disable    False for NX windows
0921      *  ->xtra_write        False for NX windows
0922      *  ->notify_early      NA for NX windows
0923      *  ->lnotify_lpid      NA for Tx windows
0924      *  ->lnotify_pid       NA for Tx windows
0925      *  ->lnotify_tid       NA for Tx windows
0926      *  ->tx_win_cred_mode  Ignore for now for NX windows
0927      *  ->rx_win_cred_mode  Ignore for now for NX windows
0928      */
0929     memset(winctx, 0, sizeof(struct vas_winctx));
0930 
0931     winctx->wcreds_max = txwin->vas_win.wcreds_max;
0932 
0933     winctx->user_win = txattr->user_win;
0934     winctx->nx_win = txwin->rxwin->nx_win;
0935     winctx->pin_win = txattr->pin_win;
0936     winctx->rej_no_credit = txattr->rej_no_credit;
0937     winctx->rsvd_txbuf_enable = txattr->rsvd_txbuf_enable;
0938 
0939     winctx->rx_wcred_mode = txattr->rx_wcred_mode;
0940     winctx->tx_wcred_mode = txattr->tx_wcred_mode;
0941     winctx->rx_word_mode = txattr->rx_win_ord_mode;
0942     winctx->tx_word_mode = txattr->tx_win_ord_mode;
0943     winctx->rsvd_txbuf_count = txattr->rsvd_txbuf_count;
0944 
0945     winctx->intr_disable = true;
0946     if (winctx->nx_win)
0947         winctx->data_stamp = true;
0948 
0949     winctx->lpid = txattr->lpid;
0950     winctx->pidr = txattr->pidr;
0951     winctx->rx_win_id = txwin->rxwin->vas_win.winid;
0952     /*
0953      * IRQ and fault window setup is successful. Set fault window
0954      * for the send window so that ready to handle faults.
0955      */
0956     if (txwin->vinst->virq)
0957         winctx->fault_win_id = txwin->vinst->fault_win->vas_win.winid;
0958 
0959     winctx->dma_type = VAS_DMA_TYPE_INJECT;
0960     winctx->tc_mode = txattr->tc_mode;
0961     winctx->min_scope = VAS_SCOPE_LOCAL;
0962     winctx->max_scope = VAS_SCOPE_VECTORED_GROUP;
0963     if (txwin->vinst->virq)
0964         winctx->irq_port = txwin->vinst->irq_port;
0965 
0966     winctx->pswid = txattr->pswid ? txattr->pswid :
0967             encode_pswid(txwin->vinst->vas_id,
0968             txwin->vas_win.winid);
0969 }
0970 
0971 static bool tx_win_args_valid(enum vas_cop_type cop,
0972             struct vas_tx_win_attr *attr)
0973 {
0974     if (attr->tc_mode != VAS_THRESH_DISABLED)
0975         return false;
0976 
0977     if (cop > VAS_COP_TYPE_MAX)
0978         return false;
0979 
0980     if (attr->wcreds_max > VAS_TX_WCREDS_MAX)
0981         return false;
0982 
0983     if (attr->user_win) {
0984         if (attr->rsvd_txbuf_count)
0985             return false;
0986 
0987         if (cop != VAS_COP_TYPE_FTW && cop != VAS_COP_TYPE_GZIP &&
0988             cop != VAS_COP_TYPE_GZIP_HIPRI)
0989             return false;
0990     }
0991 
0992     return true;
0993 }
0994 
0995 struct vas_window *vas_tx_win_open(int vasid, enum vas_cop_type cop,
0996             struct vas_tx_win_attr *attr)
0997 {
0998     int rc;
0999     struct pnv_vas_window *txwin;
1000     struct pnv_vas_window *rxwin;
1001     struct vas_winctx winctx;
1002     struct vas_instance *vinst;
1003 
1004     trace_vas_tx_win_open(current, vasid, cop, attr);
1005 
1006     if (!tx_win_args_valid(cop, attr))
1007         return ERR_PTR(-EINVAL);
1008 
1009     /*
1010      * If caller did not specify a vasid but specified the PSWID of a
1011      * receive window (applicable only to FTW windows), use the vasid
1012      * from that receive window.
1013      */
1014     if (vasid == -1 && attr->pswid)
1015         decode_pswid(attr->pswid, &vasid, NULL);
1016 
1017     vinst = find_vas_instance(vasid);
1018     if (!vinst) {
1019         pr_devel("vasid %d not found!\n", vasid);
1020         return ERR_PTR(-EINVAL);
1021     }
1022 
1023     rxwin = get_vinst_rxwin(vinst, cop, attr->pswid);
1024     if (IS_ERR(rxwin)) {
1025         pr_devel("No RxWin for vasid %d, cop %d\n", vasid, cop);
1026         return (struct vas_window *)rxwin;
1027     }
1028 
1029     txwin = vas_window_alloc(vinst);
1030     if (IS_ERR(txwin)) {
1031         rc = PTR_ERR(txwin);
1032         goto put_rxwin;
1033     }
1034 
1035     txwin->vas_win.cop = cop;
1036     txwin->tx_win = 1;
1037     txwin->rxwin = rxwin;
1038     txwin->nx_win = txwin->rxwin->nx_win;
1039     txwin->user_win = attr->user_win;
1040     txwin->vas_win.wcreds_max = attr->wcreds_max ?: VAS_WCREDS_DEFAULT;
1041 
1042     init_winctx_for_txwin(txwin, attr, &winctx);
1043 
1044     init_winctx_regs(txwin, &winctx);
1045 
1046     /*
1047      * If its a kernel send window, map the window address into the
1048      * kernel's address space. For user windows, user must issue an
1049      * mmap() to map the window into their address space.
1050      *
1051      * NOTE: If kernel ever resubmits a user CRB after handling a page
1052      *   fault, we will need to map this into kernel as well.
1053      */
1054     if (!txwin->user_win) {
1055         txwin->paste_kaddr = map_paste_region(txwin);
1056         if (IS_ERR(txwin->paste_kaddr)) {
1057             rc = PTR_ERR(txwin->paste_kaddr);
1058             goto free_window;
1059         }
1060     } else {
1061         /*
1062          * Interrupt hanlder or fault window setup failed. Means
1063          * NX can not generate fault for page fault. So not
1064          * opening for user space tx window.
1065          */
1066         if (!vinst->virq) {
1067             rc = -ENODEV;
1068             goto free_window;
1069         }
1070         rc = get_vas_user_win_ref(&txwin->vas_win.task_ref);
1071         if (rc)
1072             goto free_window;
1073 
1074         vas_user_win_add_mm_context(&txwin->vas_win.task_ref);
1075     }
1076 
1077     set_vinst_win(vinst, txwin);
1078 
1079     return &txwin->vas_win;
1080 
1081 free_window:
1082     vas_window_free(txwin);
1083 
1084 put_rxwin:
1085     put_rx_win(rxwin);
1086     return ERR_PTR(rc);
1087 
1088 }
1089 EXPORT_SYMBOL_GPL(vas_tx_win_open);
1090 
1091 int vas_copy_crb(void *crb, int offset)
1092 {
1093     return vas_copy(crb, offset);
1094 }
1095 EXPORT_SYMBOL_GPL(vas_copy_crb);
1096 
1097 #define RMA_LSMP_REPORT_ENABLE PPC_BIT(53)
1098 int vas_paste_crb(struct vas_window *vwin, int offset, bool re)
1099 {
1100     struct pnv_vas_window *txwin;
1101     int rc;
1102     void *addr;
1103     uint64_t val;
1104 
1105     txwin = container_of(vwin, struct pnv_vas_window, vas_win);
1106     trace_vas_paste_crb(current, txwin);
1107 
1108     /*
1109      * Only NX windows are supported for now and hardware assumes
1110      * report-enable flag is set for NX windows. Ensure software
1111      * complies too.
1112      */
1113     WARN_ON_ONCE(txwin->nx_win && !re);
1114 
1115     addr = txwin->paste_kaddr;
1116     if (re) {
1117         /*
1118          * Set the REPORT_ENABLE bit (equivalent to writing
1119          * to 1K offset of the paste address)
1120          */
1121         val = SET_FIELD(RMA_LSMP_REPORT_ENABLE, 0ULL, 1);
1122         addr += val;
1123     }
1124 
1125     /*
1126      * Map the raw CR value from vas_paste() to an error code (there
1127      * is just pass or fail for now though).
1128      */
1129     rc = vas_paste(addr, offset);
1130     if (rc == 2)
1131         rc = 0;
1132     else
1133         rc = -EINVAL;
1134 
1135     pr_debug("Txwin #%d: Msg count %llu\n", txwin->vas_win.winid,
1136             read_hvwc_reg(txwin, VREG(LRFIFO_PUSH)));
1137 
1138     return rc;
1139 }
1140 EXPORT_SYMBOL_GPL(vas_paste_crb);
1141 
1142 /*
1143  * If credit checking is enabled for this window, poll for the return
1144  * of window credits (i.e for NX engines to process any outstanding CRBs).
1145  * Since NX-842 waits for the CRBs to be processed before closing the
1146  * window, we should not have to wait for too long.
1147  *
1148  * TODO: We retry in 10ms intervals now. We could/should probably peek at
1149  *  the VAS_LRFIFO_PUSH_OFFSET register to get an estimate of pending
1150  *  CRBs on the FIFO and compute the delay dynamically on each retry.
1151  *  But that is not really needed until we support NX-GZIP access from
1152  *  user space. (NX-842 driver waits for CSB and Fast thread-wakeup
1153  *  doesn't use credit checking).
1154  */
1155 static void poll_window_credits(struct pnv_vas_window *window)
1156 {
1157     u64 val;
1158     int creds, mode;
1159     int count = 0;
1160 
1161     val = read_hvwc_reg(window, VREG(WINCTL));
1162     if (window->tx_win)
1163         mode = GET_FIELD(VAS_WINCTL_TX_WCRED_MODE, val);
1164     else
1165         mode = GET_FIELD(VAS_WINCTL_RX_WCRED_MODE, val);
1166 
1167     if (!mode)
1168         return;
1169 retry:
1170     if (window->tx_win) {
1171         val = read_hvwc_reg(window, VREG(TX_WCRED));
1172         creds = GET_FIELD(VAS_TX_WCRED, val);
1173     } else {
1174         val = read_hvwc_reg(window, VREG(LRX_WCRED));
1175         creds = GET_FIELD(VAS_LRX_WCRED, val);
1176     }
1177 
1178     /*
1179      * Takes around few milliseconds to complete all pending requests
1180      * and return credits.
1181      * TODO: Scan fault FIFO and invalidate CRBs points to this window
1182      *       and issue CRB Kill to stop all pending requests. Need only
1183      *       if there is a bug in NX or fault handling in kernel.
1184      */
1185     if (creds < window->vas_win.wcreds_max) {
1186         val = 0;
1187         set_current_state(TASK_UNINTERRUPTIBLE);
1188         schedule_timeout(msecs_to_jiffies(10));
1189         count++;
1190         /*
1191          * Process can not close send window until all credits are
1192          * returned.
1193          */
1194         if (!(count % 1000))
1195             pr_warn_ratelimited("VAS: pid %d stuck. Waiting for credits returned for Window(%d). creds %d, Retries %d\n",
1196                 vas_window_pid(&window->vas_win),
1197                 window->vas_win.winid,
1198                 creds, count);
1199 
1200         goto retry;
1201     }
1202 }
1203 
1204 /*
1205  * Wait for the window to go to "not-busy" state. It should only take a
1206  * short time to queue a CRB, so window should not be busy for too long.
1207  * Trying 5ms intervals.
1208  */
1209 static void poll_window_busy_state(struct pnv_vas_window *window)
1210 {
1211     int busy;
1212     u64 val;
1213     int count = 0;
1214 
1215 retry:
1216     val = read_hvwc_reg(window, VREG(WIN_STATUS));
1217     busy = GET_FIELD(VAS_WIN_BUSY, val);
1218     if (busy) {
1219         val = 0;
1220         set_current_state(TASK_UNINTERRUPTIBLE);
1221         schedule_timeout(msecs_to_jiffies(10));
1222         count++;
1223         /*
1224          * Takes around few milliseconds to process all pending
1225          * requests.
1226          */
1227         if (!(count % 1000))
1228             pr_warn_ratelimited("VAS: pid %d stuck. Window (ID=%d) is in busy state. Retries %d\n",
1229                 vas_window_pid(&window->vas_win),
1230                 window->vas_win.winid, count);
1231 
1232         goto retry;
1233     }
1234 }
1235 
1236 /*
1237  * Have the hardware cast a window out of cache and wait for it to
1238  * be completed.
1239  *
1240  * NOTE: It can take a relatively long time to cast the window context
1241  *  out of the cache. It is not strictly necessary to cast out if:
1242  *
1243  *  - we clear the "Pin Window" bit (so hardware is free to evict)
1244  *
1245  *  - we re-initialize the window context when it is reassigned.
1246  *
1247  *  We do the former in vas_win_close() and latter in vas_win_open().
1248  *  So, ignoring the cast-out for now. We can add it as needed. If
1249  *  casting out becomes necessary we should consider offloading the
1250  *  job to a worker thread, so the window close can proceed quickly.
1251  */
1252 static void poll_window_castout(struct pnv_vas_window *window)
1253 {
1254     /* stub for now */
1255 }
1256 
1257 /*
1258  * Unpin and close a window so no new requests are accepted and the
1259  * hardware can evict this window from cache if necessary.
1260  */
1261 static void unpin_close_window(struct pnv_vas_window *window)
1262 {
1263     u64 val;
1264 
1265     val = read_hvwc_reg(window, VREG(WINCTL));
1266     val = SET_FIELD(VAS_WINCTL_PIN, val, 0);
1267     val = SET_FIELD(VAS_WINCTL_OPEN, val, 0);
1268     write_hvwc_reg(window, VREG(WINCTL), val);
1269 }
1270 
1271 /*
1272  * Close a window.
1273  *
1274  * See Section 1.12.1 of VAS workbook v1.05 for details on closing window:
1275  *  - Disable new paste operations (unmap paste address)
1276  *  - Poll for the "Window Busy" bit to be cleared
1277  *  - Clear the Open/Enable bit for the Window.
1278  *  - Poll for return of window Credits (implies FIFO empty for Rx win?)
1279  *  - Unpin and cast window context out of cache
1280  *
1281  * Besides the hardware, kernel has some bookkeeping of course.
1282  */
1283 int vas_win_close(struct vas_window *vwin)
1284 {
1285     struct pnv_vas_window *window;
1286 
1287     if (!vwin)
1288         return 0;
1289 
1290     window = container_of(vwin, struct pnv_vas_window, vas_win);
1291 
1292     if (!window->tx_win && atomic_read(&window->num_txwins) != 0) {
1293         pr_devel("Attempting to close an active Rx window!\n");
1294         WARN_ON_ONCE(1);
1295         return -EBUSY;
1296     }
1297 
1298     unmap_paste_region(window);
1299 
1300     poll_window_busy_state(window);
1301 
1302     unpin_close_window(window);
1303 
1304     poll_window_credits(window);
1305 
1306     clear_vinst_win(window);
1307 
1308     poll_window_castout(window);
1309 
1310     /* if send window, drop reference to matching receive window */
1311     if (window->tx_win) {
1312         if (window->user_win) {
1313             put_vas_user_win_ref(&vwin->task_ref);
1314             mm_context_remove_vas_window(vwin->task_ref.mm);
1315         }
1316         put_rx_win(window->rxwin);
1317     }
1318 
1319     vas_window_free(window);
1320 
1321     return 0;
1322 }
1323 EXPORT_SYMBOL_GPL(vas_win_close);
1324 
1325 /*
1326  * Return credit for the given window.
1327  * Send windows and fault window uses credit mechanism as follows:
1328  *
1329  * Send windows:
1330  * - The default number of credits available for each send window is
1331  *   1024. It means 1024 requests can be issued asynchronously at the
1332  *   same time. If the credit is not available, that request will be
1333  *   returned with RMA_Busy.
1334  * - One credit is taken when NX request is issued.
1335  * - This credit is returned after NX processed that request.
1336  * - If NX encounters translation error, kernel will return the
1337  *   credit on the specific send window after processing the fault CRB.
1338  *
1339  * Fault window:
1340  * - The total number credits available is FIFO_SIZE/CRB_SIZE.
1341  *   Means 4MB/128 in the current implementation. If credit is not
1342  *   available, RMA_Reject is returned.
1343  * - A credit is taken when NX pastes CRB in fault FIFO.
1344  * - The kernel with return credit on fault window after reading entry
1345  *   from fault FIFO.
1346  */
1347 void vas_return_credit(struct pnv_vas_window *window, bool tx)
1348 {
1349     uint64_t val;
1350 
1351     val = 0ULL;
1352     if (tx) { /* send window */
1353         val = SET_FIELD(VAS_TX_WCRED, val, 1);
1354         write_hvwc_reg(window, VREG(TX_WCRED_ADDER), val);
1355     } else {
1356         val = SET_FIELD(VAS_LRX_WCRED, val, 1);
1357         write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), val);
1358     }
1359 }
1360 
1361 struct pnv_vas_window *vas_pswid_to_window(struct vas_instance *vinst,
1362         uint32_t pswid)
1363 {
1364     struct pnv_vas_window *window;
1365     int winid;
1366 
1367     if (!pswid) {
1368         pr_devel("%s: called for pswid 0!\n", __func__);
1369         return ERR_PTR(-ESRCH);
1370     }
1371 
1372     decode_pswid(pswid, NULL, &winid);
1373 
1374     if (winid >= VAS_WINDOWS_PER_CHIP)
1375         return ERR_PTR(-ESRCH);
1376 
1377     /*
1378      * If application closes the window before the hardware
1379      * returns the fault CRB, we should wait in vas_win_close()
1380      * for the pending requests. so the window must be active
1381      * and the process alive.
1382      *
1383      * If its a kernel process, we should not get any faults and
1384      * should not get here.
1385      */
1386     window = vinst->windows[winid];
1387 
1388     if (!window) {
1389         pr_err("PSWID decode: Could not find window for winid %d pswid %d vinst 0x%p\n",
1390             winid, pswid, vinst);
1391         return NULL;
1392     }
1393 
1394     /*
1395      * Do some sanity checks on the decoded window.  Window should be
1396      * NX GZIP user send window. FTW windows should not incur faults
1397      * since their CRBs are ignored (not queued on FIFO or processed
1398      * by NX).
1399      */
1400     if (!window->tx_win || !window->user_win || !window->nx_win ||
1401             window->vas_win.cop == VAS_COP_TYPE_FAULT ||
1402             window->vas_win.cop == VAS_COP_TYPE_FTW) {
1403         pr_err("PSWID decode: id %d, tx %d, user %d, nx %d, cop %d\n",
1404             winid, window->tx_win, window->user_win,
1405             window->nx_win, window->vas_win.cop);
1406         WARN_ON(1);
1407     }
1408 
1409     return window;
1410 }
1411 
1412 static struct vas_window *vas_user_win_open(int vas_id, u64 flags,
1413                 enum vas_cop_type cop_type)
1414 {
1415     struct vas_tx_win_attr txattr = {};
1416 
1417     vas_init_tx_win_attr(&txattr, cop_type);
1418 
1419     txattr.lpid = mfspr(SPRN_LPID);
1420     txattr.pidr = mfspr(SPRN_PID);
1421     txattr.user_win = true;
1422     txattr.rsvd_txbuf_count = false;
1423     txattr.pswid = false;
1424 
1425     pr_devel("Pid %d: Opening txwin, PIDR %ld\n", txattr.pidr,
1426                 mfspr(SPRN_PID));
1427 
1428     return vas_tx_win_open(vas_id, cop_type, &txattr);
1429 }
1430 
1431 static u64 vas_user_win_paste_addr(struct vas_window *txwin)
1432 {
1433     struct pnv_vas_window *win;
1434     u64 paste_addr;
1435 
1436     win = container_of(txwin, struct pnv_vas_window, vas_win);
1437     vas_win_paste_addr(win, &paste_addr, NULL);
1438 
1439     return paste_addr;
1440 }
1441 
1442 static int vas_user_win_close(struct vas_window *txwin)
1443 {
1444     vas_win_close(txwin);
1445 
1446     return 0;
1447 }
1448 
1449 static const struct vas_user_win_ops vops =  {
1450     .open_win   =   vas_user_win_open,
1451     .paste_addr =   vas_user_win_paste_addr,
1452     .close_win  =   vas_user_win_close,
1453 };
1454 
1455 /*
1456  * Supporting only nx-gzip coprocessor type now, but this API code
1457  * extended to other coprocessor types later.
1458  */
1459 int vas_register_api_powernv(struct module *mod, enum vas_cop_type cop_type,
1460                  const char *name)
1461 {
1462 
1463     return vas_register_coproc_api(mod, cop_type, name, &vops);
1464 }
1465 EXPORT_SYMBOL_GPL(vas_register_api_powernv);
1466 
1467 void vas_unregister_api_powernv(void)
1468 {
1469     vas_unregister_coproc_api();
1470 }
1471 EXPORT_SYMBOL_GPL(vas_unregister_api_powernv);
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!