chelsio/libcxgb/libcxgb_ppm.h

0001 /*
0002  * libcxgb_ppm.h: Chelsio common library for T3/T4/T5 iSCSI ddp operation
0003  *
0004  * Copyright (c) 2016 Chelsio Communications, Inc. All rights reserved.
0005  *
0006  * This software is available to you under a choice of one of two
0007  * licenses.  You may choose to be licensed under the terms of the GNU
0008  * General Public License (GPL) Version 2, available from the file
0009  * COPYING in the main directory of this source tree, or the
0010  * OpenIB.org BSD license below:
0011  *
0012  *     Redistribution and use in source and binary forms, with or
0013  *     without modification, are permitted provided that the following
0014  *     conditions are met:
0015  *
0016  *      - Redistributions of source code must retain the above
0017  *        copyright notice, this list of conditions and the following
0018  *        disclaimer.
0019  *
0020  *      - Redistributions in binary form must reproduce the above
0021  *        copyright notice, this list of conditions and the following
0022  *        disclaimer in the documentation and/or other materials
0023  *        provided with the distribution.
0024  *
0025  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
0026  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
0027  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
0028  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
0029  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
0030  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
0031  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
0032  * SOFTWARE.
0033  *
0034  * Written by: Karen Xie (kxie@chelsio.com)
0035  */
0036
0037 #ifndef __LIBCXGB_PPM_H__
0038 #define __LIBCXGB_PPM_H__
0039
0040 #include <linux/kernel.h>
0041 #include <linux/errno.h>
0042 #include <linux/types.h>
0043 #include <linux/debugfs.h>
0044 #include <linux/list.h>
0045 #include <linux/netdevice.h>
0046 #include <linux/scatterlist.h>
0047 #include <linux/skbuff.h>
0048 #include <linux/vmalloc.h>
0049 #include <linux/bitmap.h>
0050
0051 struct cxgbi_pagepod_hdr {
0052     u32 vld_tid;
0053     u32 pgsz_tag_clr;
0054     u32 max_offset;
0055     u32 page_offset;
0056     u64 rsvd;
0057 };
0058
0059 #define PPOD_PAGES_MAX          4
0060 struct cxgbi_pagepod {
0061     struct cxgbi_pagepod_hdr hdr;
0062     __be64 addr[PPOD_PAGES_MAX + 1];
0063 };
0064
0065 /* ddp tag format
0066  * for a 32-bit tag:
0067  * bit #
0068  * 31 .....   .....  0
0069  *     X   Y...Y Z...Z, where
0070  *     ^   ^^^^^ ^^^^
0071  *     |   |      |____ when ddp bit = 0: color bits
0072  *     |   |
0073  *     |   |____ when ddp bit = 0: idx into the ddp memory region
0074  *     |
0075  *     |____ ddp bit: 0 - ddp tag, 1 - non-ddp tag
0076  *
0077  *  [page selector:2] [sw/free bits] [0] [idx] [color:6]
0078  */
0079
0080 #define DDP_PGIDX_MAX       4
0081 #define DDP_PGSZ_BASE_SHIFT 12  /* base page 4K */
0082
0083 struct cxgbi_task_tag_info {
0084     unsigned char flags;
0085 #define CXGBI_PPOD_INFO_FLAG_VALID  0x1
0086 #define CXGBI_PPOD_INFO_FLAG_MAPPED 0x2
0087     unsigned char cid;
0088     unsigned short pg_shift;
0089     unsigned int npods;
0090     unsigned int idx;
0091     unsigned int tag;
0092     struct cxgbi_pagepod_hdr hdr;
0093     int nents;
0094     int nr_pages;
0095     struct scatterlist *sgl;
0096 };
0097
0098 struct cxgbi_tag_format {
0099     unsigned char pgsz_order[DDP_PGIDX_MAX];
0100     unsigned char pgsz_idx_dflt;
0101     unsigned char free_bits:4;
0102     unsigned char color_bits:4;
0103     unsigned char idx_bits;
0104     unsigned char rsvd_bits;
0105     unsigned int  no_ddp_mask;
0106     unsigned int  idx_mask;
0107     unsigned int  color_mask;
0108     unsigned int  idx_clr_mask;
0109     unsigned int  rsvd_mask;
0110 };
0111
0112 struct cxgbi_ppod_data {
0113     unsigned char pg_idx:2;
0114     unsigned char color:6;
0115     unsigned char chan_id;
0116     unsigned short npods;
0117     unsigned long caller_data;
0118 };
0119
0120 /* per cpu ppm pool */
0121 struct cxgbi_ppm_pool {
0122     unsigned int base;      /* base index */
0123     unsigned int next;      /* next possible free index */
0124     spinlock_t lock;        /* ppm pool lock */
0125     unsigned long bmap[];
0126 } ____cacheline_aligned_in_smp;
0127
0128 struct cxgbi_ppm {
0129     struct kref refcnt;
0130     struct net_device *ndev;    /* net_device, 1st port */
0131     struct pci_dev *pdev;
0132     void *lldev;
0133     void **ppm_pp;
0134     struct cxgbi_tag_format tformat;
0135     unsigned int ppmax;
0136     unsigned int llimit;
0137     unsigned int base_idx;
0138
0139     unsigned int pool_rsvd;
0140     unsigned int pool_index_max;
0141     struct cxgbi_ppm_pool __percpu *pool;
0142     /* map lock */
0143     spinlock_t map_lock;        /* ppm map lock */
0144     unsigned int bmap_index_max;
0145     unsigned int next;
0146     unsigned int max_index_in_edram;
0147     unsigned long *ppod_bmap;
0148     struct cxgbi_ppod_data ppod_data[];
0149 };
0150
0151 #define DDP_THRESHOLD       512
0152
0153 #define PPOD_PAGES_SHIFT    2       /*  4 pages per pod */
0154
0155 #define IPPOD_SIZE               sizeof(struct cxgbi_pagepod)  /*  64 */
0156 #define PPOD_SIZE_SHIFT         6
0157
0158 /* page pods are allocated in groups of this size (must be power of 2) */
0159 #define PPOD_CLUSTER_SIZE   16U
0160
0161 #define ULPMEM_DSGL_MAX_NPPODS  16  /*  1024/PPOD_SIZE */
0162 #define ULPMEM_IDATA_MAX_NPPODS 3   /* (PPOD_SIZE * 3 + ulptx hdr) < 256B */
0163 #define PCIE_MEMWIN_MAX_NPPODS  16  /*  1024/PPOD_SIZE */
0164
0165 #define PPOD_COLOR_SHIFT    0
0166 #define PPOD_COLOR(x)       ((x) << PPOD_COLOR_SHIFT)
0167
0168 #define PPOD_IDX_SHIFT          6
0169 #define PPOD_IDX_MAX_SIZE       24
0170
0171 #define PPOD_TID_SHIFT      0
0172 #define PPOD_TID(x)     ((x) << PPOD_TID_SHIFT)
0173
0174 #define PPOD_TAG_SHIFT      6
0175 #define PPOD_TAG(x)     ((x) << PPOD_TAG_SHIFT)
0176
0177 #define PPOD_VALID_SHIFT    24
0178 #define PPOD_VALID(x)       ((x) << PPOD_VALID_SHIFT)
0179 #define PPOD_VALID_FLAG     PPOD_VALID(1U)
0180
0181 #define PPOD_PI_EXTRACT_CTL_SHIFT   31
0182 #define PPOD_PI_EXTRACT_CTL(x)      ((x) << PPOD_PI_EXTRACT_CTL_SHIFT)
0183 #define PPOD_PI_EXTRACT_CTL_FLAG    V_PPOD_PI_EXTRACT_CTL(1U)
0184
0185 #define PPOD_PI_TYPE_SHIFT      29
0186 #define PPOD_PI_TYPE_MASK       0x3
0187 #define PPOD_PI_TYPE(x)         ((x) << PPOD_PI_TYPE_SHIFT)
0188
0189 #define PPOD_PI_CHECK_CTL_SHIFT     27
0190 #define PPOD_PI_CHECK_CTL_MASK      0x3
0191 #define PPOD_PI_CHECK_CTL(x)        ((x) << PPOD_PI_CHECK_CTL_SHIFT)
0192
0193 #define PPOD_PI_REPORT_CTL_SHIFT    25
0194 #define PPOD_PI_REPORT_CTL_MASK     0x3
0195 #define PPOD_PI_REPORT_CTL(x)       ((x) << PPOD_PI_REPORT_CTL_SHIFT)
0196
0197 static inline int cxgbi_ppm_is_ddp_tag(struct cxgbi_ppm *ppm, u32 tag)
0198 {
0199     return !(tag & ppm->tformat.no_ddp_mask);
0200 }
0201
0202 static inline int cxgbi_ppm_sw_tag_is_usable(struct cxgbi_ppm *ppm,
0203                          u32 tag)
0204 {
0205     /* the sw tag must be using <= 31 bits */
0206     return !(tag & 0x80000000U);
0207 }
0208
0209 static inline int cxgbi_ppm_make_non_ddp_tag(struct cxgbi_ppm *ppm,
0210                          u32 sw_tag,
0211                          u32 *final_tag)
0212 {
0213     struct cxgbi_tag_format *tformat = &ppm->tformat;
0214
0215     if (!cxgbi_ppm_sw_tag_is_usable(ppm, sw_tag)) {
0216         pr_info("sw_tag 0x%x NOT usable.\n", sw_tag);
0217         return -EINVAL;
0218     }
0219
0220     if (!sw_tag) {
0221         *final_tag = tformat->no_ddp_mask;
0222     } else {
0223         unsigned int shift = tformat->idx_bits + tformat->color_bits;
0224         u32 lower = sw_tag & tformat->idx_clr_mask;
0225         u32 upper = (sw_tag >> shift) << (shift + 1);
0226
0227         *final_tag = upper | tformat->no_ddp_mask | lower;
0228     }
0229     return 0;
0230 }
0231
0232 static inline u32 cxgbi_ppm_decode_non_ddp_tag(struct cxgbi_ppm *ppm,
0233                            u32 tag)
0234 {
0235     struct cxgbi_tag_format *tformat = &ppm->tformat;
0236     unsigned int shift = tformat->idx_bits + tformat->color_bits;
0237     u32 lower = tag & tformat->idx_clr_mask;
0238     u32 upper = (tag >> tformat->rsvd_bits) << shift;
0239
0240     return upper | lower;
0241 }
0242
0243 static inline u32 cxgbi_ppm_ddp_tag_get_idx(struct cxgbi_ppm *ppm,
0244                         u32 ddp_tag)
0245 {
0246     u32 hw_idx = (ddp_tag >> PPOD_IDX_SHIFT) &
0247             ppm->tformat.idx_mask;
0248
0249     return hw_idx - ppm->base_idx;
0250 }
0251
0252 static inline u32 cxgbi_ppm_make_ddp_tag(unsigned int hw_idx,
0253                      unsigned char color)
0254 {
0255     return (hw_idx << PPOD_IDX_SHIFT) | ((u32)color);
0256 }
0257
0258 static inline unsigned long
0259 cxgbi_ppm_get_tag_caller_data(struct cxgbi_ppm *ppm,
0260                   u32 ddp_tag)
0261 {
0262     u32 idx = cxgbi_ppm_ddp_tag_get_idx(ppm, ddp_tag);
0263
0264     return ppm->ppod_data[idx].caller_data;
0265 }
0266
0267 /* sw bits are the free bits */
0268 static inline int cxgbi_ppm_ddp_tag_update_sw_bits(struct cxgbi_ppm *ppm,
0269                            u32 val, u32 orig_tag,
0270                            u32 *final_tag)
0271 {
0272     struct cxgbi_tag_format *tformat = &ppm->tformat;
0273     u32 v = val >> tformat->free_bits;
0274
0275     if (v) {
0276         pr_info("sw_bits 0x%x too large, avail bits %u.\n",
0277             val, tformat->free_bits);
0278         return -EINVAL;
0279     }
0280     if (!cxgbi_ppm_is_ddp_tag(ppm, orig_tag))
0281         return -EINVAL;
0282
0283     *final_tag = (val << tformat->rsvd_bits) |
0284              (orig_tag & ppm->tformat.rsvd_mask);
0285     return 0;
0286 }
0287
0288 static inline void cxgbi_ppm_ppod_clear(struct cxgbi_pagepod *ppod)
0289 {
0290     ppod->hdr.vld_tid = 0U;
0291 }
0292
0293 static inline void cxgbi_tagmask_check(unsigned int tagmask,
0294                        struct cxgbi_tag_format *tformat)
0295 {
0296     unsigned int bits = fls(tagmask);
0297
0298     /* reserve top most 2 bits for page selector */
0299     tformat->free_bits = 32 - 2 - bits;
0300     tformat->rsvd_bits = bits;
0301     tformat->color_bits = PPOD_IDX_SHIFT;
0302     tformat->idx_bits = bits - 1 - PPOD_IDX_SHIFT;
0303     tformat->no_ddp_mask = 1 << (bits - 1);
0304     tformat->idx_mask = (1 << tformat->idx_bits) - 1;
0305     tformat->color_mask = (1 << PPOD_IDX_SHIFT) - 1;
0306     tformat->idx_clr_mask = (1 << (bits - 1)) - 1;
0307     tformat->rsvd_mask = (1 << bits) - 1;
0308
0309     pr_info("ippm: tagmask 0x%x, rsvd %u=%u+%u+1, mask 0x%x,0x%x, "
0310         "pg %u,%u,%u,%u.\n",
0311         tagmask, tformat->rsvd_bits, tformat->idx_bits,
0312         tformat->color_bits, tformat->no_ddp_mask, tformat->rsvd_mask,
0313         tformat->pgsz_order[0], tformat->pgsz_order[1],
0314         tformat->pgsz_order[2], tformat->pgsz_order[3]);
0315 }
0316
0317 int cxgbi_ppm_find_page_index(struct cxgbi_ppm *ppm, unsigned long pgsz);
0318 void cxgbi_ppm_make_ppod_hdr(struct cxgbi_ppm *ppm, u32 tag,
0319                  unsigned int tid, unsigned int offset,
0320                  unsigned int length,
0321                  struct cxgbi_pagepod_hdr *hdr);
0322 void cxgbi_ppm_ppod_release(struct cxgbi_ppm *, u32 idx);
0323 int cxgbi_ppm_ppods_reserve(struct cxgbi_ppm *, unsigned short nr_pages,
0324                 u32 per_tag_pg_idx, u32 *ppod_idx, u32 *ddp_tag,
0325                 unsigned long caller_data);
0326 int cxgbi_ppm_init(void **ppm_pp, struct net_device *, struct pci_dev *,
0327            void *lldev, struct cxgbi_tag_format *,
0328            unsigned int iscsi_size, unsigned int llimit,
0329            unsigned int start, unsigned int reserve_factor,
0330            unsigned int edram_start, unsigned int edram_size);
0331 int cxgbi_ppm_release(struct cxgbi_ppm *ppm);
0332 void cxgbi_tagmask_check(unsigned int tagmask, struct cxgbi_tag_format *);
0333 unsigned int cxgbi_tagmask_set(unsigned int ppmax);
0334
0335 #endif  /*__LIBCXGB_PPM_H__*/