Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * Driver for IBM PowerNV compression accelerator
0004  *
0005  * Copyright (C) 2015 Dan Streetman, IBM Corp
0006  */
0007 
0008 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
0009 
0010 #include "nx-842.h"
0011 
0012 #include <linux/timer.h>
0013 
0014 #include <asm/prom.h>
0015 #include <asm/icswx.h>
0016 #include <asm/vas.h>
0017 #include <asm/reg.h>
0018 #include <asm/opal-api.h>
0019 #include <asm/opal.h>
0020 
0021 MODULE_LICENSE("GPL");
0022 MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");
0023 MODULE_DESCRIPTION("H/W Compression driver for IBM PowerNV processors");
0024 MODULE_ALIAS_CRYPTO("842");
0025 MODULE_ALIAS_CRYPTO("842-nx");
0026 
0027 #define WORKMEM_ALIGN   (CRB_ALIGN)
0028 #define CSB_WAIT_MAX    (5000) /* ms */
0029 #define VAS_RETRIES (10)
0030 
0031 struct nx842_workmem {
0032     /* Below fields must be properly aligned */
0033     struct coprocessor_request_block crb; /* CRB_ALIGN align */
0034     struct data_descriptor_entry ddl_in[DDL_LEN_MAX]; /* DDE_ALIGN align */
0035     struct data_descriptor_entry ddl_out[DDL_LEN_MAX]; /* DDE_ALIGN align */
0036     /* Above fields must be properly aligned */
0037 
0038     ktime_t start;
0039 
0040     char padding[WORKMEM_ALIGN]; /* unused, to allow alignment */
0041 } __packed __aligned(WORKMEM_ALIGN);
0042 
0043 struct nx_coproc {
0044     unsigned int chip_id;
0045     unsigned int ct;    /* Can be 842 or GZIP high/normal*/
0046     unsigned int ci;    /* Coprocessor instance, used with icswx */
0047     struct {
0048         struct vas_window *rxwin;
0049         int id;
0050     } vas;
0051     struct list_head list;
0052 };
0053 
0054 /*
0055  * Send the request to NX engine on the chip for the corresponding CPU
0056  * where the process is executing. Use with VAS function.
0057  */
0058 static DEFINE_PER_CPU(struct vas_window *, cpu_txwin);
0059 
0060 /* no cpu hotplug on powernv, so this list never changes after init */
0061 static LIST_HEAD(nx_coprocs);
0062 static unsigned int nx842_ct;   /* used in icswx function */
0063 
0064 /*
0065  * Using same values as in skiboot or coprocessor type representing
0066  * in NX workbook.
0067  */
0068 #define NX_CT_GZIP  (2) /* on P9 and later */
0069 #define NX_CT_842   (3)
0070 
0071 static int (*nx842_powernv_exec)(const unsigned char *in,
0072                 unsigned int inlen, unsigned char *out,
0073                 unsigned int *outlenp, void *workmem, int fc);
0074 
0075 /**
0076  * setup_indirect_dde - Setup an indirect DDE
0077  *
0078  * The DDE is setup with the DDE count, byte count, and address of
0079  * first direct DDE in the list.
0080  */
0081 static void setup_indirect_dde(struct data_descriptor_entry *dde,
0082                    struct data_descriptor_entry *ddl,
0083                    unsigned int dde_count, unsigned int byte_count)
0084 {
0085     dde->flags = 0;
0086     dde->count = dde_count;
0087     dde->index = 0;
0088     dde->length = cpu_to_be32(byte_count);
0089     dde->address = cpu_to_be64(nx842_get_pa(ddl));
0090 }
0091 
0092 /**
0093  * setup_direct_dde - Setup single DDE from buffer
0094  *
0095  * The DDE is setup with the buffer and length.  The buffer must be properly
0096  * aligned.  The used length is returned.
0097  * Returns:
0098  *   N    Successfully set up DDE with N bytes
0099  */
0100 static unsigned int setup_direct_dde(struct data_descriptor_entry *dde,
0101                      unsigned long pa, unsigned int len)
0102 {
0103     unsigned int l = min_t(unsigned int, len, LEN_ON_PAGE(pa));
0104 
0105     dde->flags = 0;
0106     dde->count = 0;
0107     dde->index = 0;
0108     dde->length = cpu_to_be32(l);
0109     dde->address = cpu_to_be64(pa);
0110 
0111     return l;
0112 }
0113 
0114 /**
0115  * setup_ddl - Setup DDL from buffer
0116  *
0117  * Returns:
0118  *   0      Successfully set up DDL
0119  */
0120 static int setup_ddl(struct data_descriptor_entry *dde,
0121              struct data_descriptor_entry *ddl,
0122              unsigned char *buf, unsigned int len,
0123              bool in)
0124 {
0125     unsigned long pa = nx842_get_pa(buf);
0126     int i, ret, total_len = len;
0127 
0128     if (!IS_ALIGNED(pa, DDE_BUFFER_ALIGN)) {
0129         pr_debug("%s buffer pa 0x%lx not 0x%x-byte aligned\n",
0130              in ? "input" : "output", pa, DDE_BUFFER_ALIGN);
0131         return -EINVAL;
0132     }
0133 
0134     /* only need to check last mult; since buffer must be
0135      * DDE_BUFFER_ALIGN aligned, and that is a multiple of
0136      * DDE_BUFFER_SIZE_MULT, and pre-last page DDE buffers
0137      * are guaranteed a multiple of DDE_BUFFER_SIZE_MULT.
0138      */
0139     if (len % DDE_BUFFER_LAST_MULT) {
0140         pr_debug("%s buffer len 0x%x not a multiple of 0x%x\n",
0141              in ? "input" : "output", len, DDE_BUFFER_LAST_MULT);
0142         if (in)
0143             return -EINVAL;
0144         len = round_down(len, DDE_BUFFER_LAST_MULT);
0145     }
0146 
0147     /* use a single direct DDE */
0148     if (len <= LEN_ON_PAGE(pa)) {
0149         ret = setup_direct_dde(dde, pa, len);
0150         WARN_ON(ret < len);
0151         return 0;
0152     }
0153 
0154     /* use the DDL */
0155     for (i = 0; i < DDL_LEN_MAX && len > 0; i++) {
0156         ret = setup_direct_dde(&ddl[i], pa, len);
0157         buf += ret;
0158         len -= ret;
0159         pa = nx842_get_pa(buf);
0160     }
0161 
0162     if (len > 0) {
0163         pr_debug("0x%x total %s bytes 0x%x too many for DDL.\n",
0164              total_len, in ? "input" : "output", len);
0165         if (in)
0166             return -EMSGSIZE;
0167         total_len -= len;
0168     }
0169     setup_indirect_dde(dde, ddl, i, total_len);
0170 
0171     return 0;
0172 }
0173 
0174 #define CSB_ERR(csb, msg, ...)                  \
0175     pr_err("ERROR: " msg " : %02x %02x %02x %02x %08x\n",   \
0176            ##__VA_ARGS__, (csb)->flags,         \
0177            (csb)->cs, (csb)->cc, (csb)->ce,         \
0178            be32_to_cpu((csb)->count))
0179 
0180 #define CSB_ERR_ADDR(csb, msg, ...)             \
0181     CSB_ERR(csb, msg " at %lx", ##__VA_ARGS__,      \
0182         (unsigned long)be64_to_cpu((csb)->address))
0183 
0184 /**
0185  * wait_for_csb
0186  */
0187 static int wait_for_csb(struct nx842_workmem *wmem,
0188             struct coprocessor_status_block *csb)
0189 {
0190     ktime_t start = wmem->start, now = ktime_get();
0191     ktime_t timeout = ktime_add_ms(start, CSB_WAIT_MAX);
0192 
0193     while (!(READ_ONCE(csb->flags) & CSB_V)) {
0194         cpu_relax();
0195         now = ktime_get();
0196         if (ktime_after(now, timeout))
0197             break;
0198     }
0199 
0200     /* hw has updated csb and output buffer */
0201     barrier();
0202 
0203     /* check CSB flags */
0204     if (!(csb->flags & CSB_V)) {
0205         CSB_ERR(csb, "CSB still not valid after %ld us, giving up",
0206             (long)ktime_us_delta(now, start));
0207         return -ETIMEDOUT;
0208     }
0209     if (csb->flags & CSB_F) {
0210         CSB_ERR(csb, "Invalid CSB format");
0211         return -EPROTO;
0212     }
0213     if (csb->flags & CSB_CH) {
0214         CSB_ERR(csb, "Invalid CSB chaining state");
0215         return -EPROTO;
0216     }
0217 
0218     /* verify CSB completion sequence is 0 */
0219     if (csb->cs) {
0220         CSB_ERR(csb, "Invalid CSB completion sequence");
0221         return -EPROTO;
0222     }
0223 
0224     /* check CSB Completion Code */
0225     switch (csb->cc) {
0226     /* no error */
0227     case CSB_CC_SUCCESS:
0228         break;
0229     case CSB_CC_TPBC_GT_SPBC:
0230         /* not an error, but the compressed data is
0231          * larger than the uncompressed data :(
0232          */
0233         break;
0234 
0235     /* input data errors */
0236     case CSB_CC_OPERAND_OVERLAP:
0237         /* input and output buffers overlap */
0238         CSB_ERR(csb, "Operand Overlap error");
0239         return -EINVAL;
0240     case CSB_CC_INVALID_OPERAND:
0241         CSB_ERR(csb, "Invalid operand");
0242         return -EINVAL;
0243     case CSB_CC_NOSPC:
0244         /* output buffer too small */
0245         return -ENOSPC;
0246     case CSB_CC_ABORT:
0247         CSB_ERR(csb, "Function aborted");
0248         return -EINTR;
0249     case CSB_CC_CRC_MISMATCH:
0250         CSB_ERR(csb, "CRC mismatch");
0251         return -EINVAL;
0252     case CSB_CC_TEMPL_INVALID:
0253         CSB_ERR(csb, "Compressed data template invalid");
0254         return -EINVAL;
0255     case CSB_CC_TEMPL_OVERFLOW:
0256         CSB_ERR(csb, "Compressed data template shows data past end");
0257         return -EINVAL;
0258     case CSB_CC_EXCEED_BYTE_COUNT:  /* P9 or later */
0259         /*
0260          * DDE byte count exceeds the limit specified in Maximum
0261          * byte count register.
0262          */
0263         CSB_ERR(csb, "DDE byte count exceeds the limit");
0264         return -EINVAL;
0265 
0266     /* these should not happen */
0267     case CSB_CC_INVALID_ALIGN:
0268         /* setup_ddl should have detected this */
0269         CSB_ERR_ADDR(csb, "Invalid alignment");
0270         return -EINVAL;
0271     case CSB_CC_DATA_LENGTH:
0272         /* setup_ddl should have detected this */
0273         CSB_ERR(csb, "Invalid data length");
0274         return -EINVAL;
0275     case CSB_CC_WR_TRANSLATION:
0276     case CSB_CC_TRANSLATION:
0277     case CSB_CC_TRANSLATION_DUP1:
0278     case CSB_CC_TRANSLATION_DUP2:
0279     case CSB_CC_TRANSLATION_DUP3:
0280     case CSB_CC_TRANSLATION_DUP4:
0281     case CSB_CC_TRANSLATION_DUP5:
0282     case CSB_CC_TRANSLATION_DUP6:
0283         /* should not happen, we use physical addrs */
0284         CSB_ERR_ADDR(csb, "Translation error");
0285         return -EPROTO;
0286     case CSB_CC_WR_PROTECTION:
0287     case CSB_CC_PROTECTION:
0288     case CSB_CC_PROTECTION_DUP1:
0289     case CSB_CC_PROTECTION_DUP2:
0290     case CSB_CC_PROTECTION_DUP3:
0291     case CSB_CC_PROTECTION_DUP4:
0292     case CSB_CC_PROTECTION_DUP5:
0293     case CSB_CC_PROTECTION_DUP6:
0294         /* should not happen, we use physical addrs */
0295         CSB_ERR_ADDR(csb, "Protection error");
0296         return -EPROTO;
0297     case CSB_CC_PRIVILEGE:
0298         /* shouldn't happen, we're in HYP mode */
0299         CSB_ERR(csb, "Insufficient Privilege error");
0300         return -EPROTO;
0301     case CSB_CC_EXCESSIVE_DDE:
0302         /* shouldn't happen, setup_ddl doesn't use many dde's */
0303         CSB_ERR(csb, "Too many DDEs in DDL");
0304         return -EINVAL;
0305     case CSB_CC_TRANSPORT:
0306     case CSB_CC_INVALID_CRB:    /* P9 or later */
0307         /* shouldn't happen, we setup CRB correctly */
0308         CSB_ERR(csb, "Invalid CRB");
0309         return -EINVAL;
0310     case CSB_CC_INVALID_DDE:    /* P9 or later */
0311         /*
0312          * shouldn't happen, setup_direct/indirect_dde creates
0313          * DDE right
0314          */
0315         CSB_ERR(csb, "Invalid DDE");
0316         return -EINVAL;
0317     case CSB_CC_SEGMENTED_DDL:
0318         /* shouldn't happen, setup_ddl creates DDL right */
0319         CSB_ERR(csb, "Segmented DDL error");
0320         return -EINVAL;
0321     case CSB_CC_DDE_OVERFLOW:
0322         /* shouldn't happen, setup_ddl creates DDL right */
0323         CSB_ERR(csb, "DDE overflow error");
0324         return -EINVAL;
0325     case CSB_CC_SESSION:
0326         /* should not happen with ICSWX */
0327         CSB_ERR(csb, "Session violation error");
0328         return -EPROTO;
0329     case CSB_CC_CHAIN:
0330         /* should not happen, we don't use chained CRBs */
0331         CSB_ERR(csb, "Chained CRB error");
0332         return -EPROTO;
0333     case CSB_CC_SEQUENCE:
0334         /* should not happen, we don't use chained CRBs */
0335         CSB_ERR(csb, "CRB sequence number error");
0336         return -EPROTO;
0337     case CSB_CC_UNKNOWN_CODE:
0338         CSB_ERR(csb, "Unknown subfunction code");
0339         return -EPROTO;
0340 
0341     /* hardware errors */
0342     case CSB_CC_RD_EXTERNAL:
0343     case CSB_CC_RD_EXTERNAL_DUP1:
0344     case CSB_CC_RD_EXTERNAL_DUP2:
0345     case CSB_CC_RD_EXTERNAL_DUP3:
0346         CSB_ERR_ADDR(csb, "Read error outside coprocessor");
0347         return -EPROTO;
0348     case CSB_CC_WR_EXTERNAL:
0349         CSB_ERR_ADDR(csb, "Write error outside coprocessor");
0350         return -EPROTO;
0351     case CSB_CC_INTERNAL:
0352         CSB_ERR(csb, "Internal error in coprocessor");
0353         return -EPROTO;
0354     case CSB_CC_PROVISION:
0355         CSB_ERR(csb, "Storage provision error");
0356         return -EPROTO;
0357     case CSB_CC_HW:
0358         CSB_ERR(csb, "Correctable hardware error");
0359         return -EPROTO;
0360     case CSB_CC_HW_EXPIRED_TIMER:   /* P9 or later */
0361         CSB_ERR(csb, "Job did not finish within allowed time");
0362         return -EPROTO;
0363 
0364     default:
0365         CSB_ERR(csb, "Invalid CC %d", csb->cc);
0366         return -EPROTO;
0367     }
0368 
0369     /* check Completion Extension state */
0370     if (csb->ce & CSB_CE_TERMINATION) {
0371         CSB_ERR(csb, "CSB request was terminated");
0372         return -EPROTO;
0373     }
0374     if (csb->ce & CSB_CE_INCOMPLETE) {
0375         CSB_ERR(csb, "CSB request not complete");
0376         return -EPROTO;
0377     }
0378     if (!(csb->ce & CSB_CE_TPBC)) {
0379         CSB_ERR(csb, "TPBC not provided, unknown target length");
0380         return -EPROTO;
0381     }
0382 
0383     /* successful completion */
0384     pr_debug_ratelimited("Processed %u bytes in %lu us\n",
0385                  be32_to_cpu(csb->count),
0386                  (unsigned long)ktime_us_delta(now, start));
0387 
0388     return 0;
0389 }
0390 
0391 static int nx842_config_crb(const unsigned char *in, unsigned int inlen,
0392             unsigned char *out, unsigned int outlen,
0393             struct nx842_workmem *wmem)
0394 {
0395     struct coprocessor_request_block *crb;
0396     struct coprocessor_status_block *csb;
0397     u64 csb_addr;
0398     int ret;
0399 
0400     crb = &wmem->crb;
0401     csb = &crb->csb;
0402 
0403     /* Clear any previous values */
0404     memset(crb, 0, sizeof(*crb));
0405 
0406     /* set up DDLs */
0407     ret = setup_ddl(&crb->source, wmem->ddl_in,
0408             (unsigned char *)in, inlen, true);
0409     if (ret)
0410         return ret;
0411 
0412     ret = setup_ddl(&crb->target, wmem->ddl_out,
0413             out, outlen, false);
0414     if (ret)
0415         return ret;
0416 
0417     /* set up CRB's CSB addr */
0418     csb_addr = nx842_get_pa(csb) & CRB_CSB_ADDRESS;
0419     csb_addr |= CRB_CSB_AT; /* Addrs are phys */
0420     crb->csb_addr = cpu_to_be64(csb_addr);
0421 
0422     return 0;
0423 }
0424 
0425 /**
0426  * nx842_exec_icswx - compress/decompress data using the 842 algorithm
0427  *
0428  * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems.
0429  * This compresses or decompresses the provided input buffer into the provided
0430  * output buffer.
0431  *
0432  * Upon return from this function @outlen contains the length of the
0433  * output data.  If there is an error then @outlen will be 0 and an
0434  * error will be specified by the return code from this function.
0435  *
0436  * The @workmem buffer should only be used by one function call at a time.
0437  *
0438  * @in: input buffer pointer
0439  * @inlen: input buffer size
0440  * @out: output buffer pointer
0441  * @outlenp: output buffer size pointer
0442  * @workmem: working memory buffer pointer, size determined by
0443  *           nx842_powernv_driver.workmem_size
0444  * @fc: function code, see CCW Function Codes in nx-842.h
0445  *
0446  * Returns:
0447  *   0      Success, output of length @outlenp stored in the buffer at @out
0448  *   -ENODEV    Hardware unavailable
0449  *   -ENOSPC    Output buffer is to small
0450  *   -EMSGSIZE  Input buffer too large
0451  *   -EINVAL    buffer constraints do not fix nx842_constraints
0452  *   -EPROTO    hardware error during operation
0453  *   -ETIMEDOUT hardware did not complete operation in reasonable time
0454  *   -EINTR operation was aborted
0455  */
0456 static int nx842_exec_icswx(const unsigned char *in, unsigned int inlen,
0457                   unsigned char *out, unsigned int *outlenp,
0458                   void *workmem, int fc)
0459 {
0460     struct coprocessor_request_block *crb;
0461     struct coprocessor_status_block *csb;
0462     struct nx842_workmem *wmem;
0463     int ret;
0464     u32 ccw;
0465     unsigned int outlen = *outlenp;
0466 
0467     wmem = PTR_ALIGN(workmem, WORKMEM_ALIGN);
0468 
0469     *outlenp = 0;
0470 
0471     /* shoudn't happen, we don't load without a coproc */
0472     if (!nx842_ct) {
0473         pr_err_ratelimited("coprocessor CT is 0");
0474         return -ENODEV;
0475     }
0476 
0477     ret = nx842_config_crb(in, inlen, out, outlen, wmem);
0478     if (ret)
0479         return ret;
0480 
0481     crb = &wmem->crb;
0482     csb = &crb->csb;
0483 
0484     /* set up CCW */
0485     ccw = 0;
0486     ccw = SET_FIELD(CCW_CT, ccw, nx842_ct);
0487     ccw = SET_FIELD(CCW_CI_842, ccw, 0); /* use 0 for hw auto-selection */
0488     ccw = SET_FIELD(CCW_FC_842, ccw, fc);
0489 
0490     wmem->start = ktime_get();
0491 
0492     /* do ICSWX */
0493     ret = icswx(cpu_to_be32(ccw), crb);
0494 
0495     pr_debug_ratelimited("icswx CR %x ccw %x crb->ccw %x\n", ret,
0496                  (unsigned int)ccw,
0497                  (unsigned int)be32_to_cpu(crb->ccw));
0498 
0499     /*
0500      * NX842 coprocessor sets 3rd bit in CR register with XER[S0].
0501      * XER[S0] is the integer summary overflow bit which is nothing
0502      * to do NX. Since this bit can be set with other return values,
0503      * mask this bit.
0504      */
0505     ret &= ~ICSWX_XERS0;
0506 
0507     switch (ret) {
0508     case ICSWX_INITIATED:
0509         ret = wait_for_csb(wmem, csb);
0510         break;
0511     case ICSWX_BUSY:
0512         pr_debug_ratelimited("842 Coprocessor busy\n");
0513         ret = -EBUSY;
0514         break;
0515     case ICSWX_REJECTED:
0516         pr_err_ratelimited("ICSWX rejected\n");
0517         ret = -EPROTO;
0518         break;
0519     }
0520 
0521     if (!ret)
0522         *outlenp = be32_to_cpu(csb->count);
0523 
0524     return ret;
0525 }
0526 
0527 /**
0528  * nx842_exec_vas - compress/decompress data using the 842 algorithm
0529  *
0530  * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems.
0531  * This compresses or decompresses the provided input buffer into the provided
0532  * output buffer.
0533  *
0534  * Upon return from this function @outlen contains the length of the
0535  * output data.  If there is an error then @outlen will be 0 and an
0536  * error will be specified by the return code from this function.
0537  *
0538  * The @workmem buffer should only be used by one function call at a time.
0539  *
0540  * @in: input buffer pointer
0541  * @inlen: input buffer size
0542  * @out: output buffer pointer
0543  * @outlenp: output buffer size pointer
0544  * @workmem: working memory buffer pointer, size determined by
0545  *           nx842_powernv_driver.workmem_size
0546  * @fc: function code, see CCW Function Codes in nx-842.h
0547  *
0548  * Returns:
0549  *   0      Success, output of length @outlenp stored in the buffer
0550  *      at @out
0551  *   -ENODEV    Hardware unavailable
0552  *   -ENOSPC    Output buffer is to small
0553  *   -EMSGSIZE  Input buffer too large
0554  *   -EINVAL    buffer constraints do not fix nx842_constraints
0555  *   -EPROTO    hardware error during operation
0556  *   -ETIMEDOUT hardware did not complete operation in reasonable time
0557  *   -EINTR operation was aborted
0558  */
0559 static int nx842_exec_vas(const unsigned char *in, unsigned int inlen,
0560                   unsigned char *out, unsigned int *outlenp,
0561                   void *workmem, int fc)
0562 {
0563     struct coprocessor_request_block *crb;
0564     struct coprocessor_status_block *csb;
0565     struct nx842_workmem *wmem;
0566     struct vas_window *txwin;
0567     int ret, i = 0;
0568     u32 ccw;
0569     unsigned int outlen = *outlenp;
0570 
0571     wmem = PTR_ALIGN(workmem, WORKMEM_ALIGN);
0572 
0573     *outlenp = 0;
0574 
0575     crb = &wmem->crb;
0576     csb = &crb->csb;
0577 
0578     ret = nx842_config_crb(in, inlen, out, outlen, wmem);
0579     if (ret)
0580         return ret;
0581 
0582     ccw = 0;
0583     ccw = SET_FIELD(CCW_FC_842, ccw, fc);
0584     crb->ccw = cpu_to_be32(ccw);
0585 
0586     do {
0587         wmem->start = ktime_get();
0588         preempt_disable();
0589         txwin = this_cpu_read(cpu_txwin);
0590 
0591         /*
0592          * VAS copy CRB into L2 cache. Refer <asm/vas.h>.
0593          * @crb and @offset.
0594          */
0595         vas_copy_crb(crb, 0);
0596 
0597         /*
0598          * VAS paste previously copied CRB to NX.
0599          * @txwin, @offset and @last (must be true).
0600          */
0601         ret = vas_paste_crb(txwin, 0, 1);
0602         preempt_enable();
0603         /*
0604          * Retry copy/paste function for VAS failures.
0605          */
0606     } while (ret && (i++ < VAS_RETRIES));
0607 
0608     if (ret) {
0609         pr_err_ratelimited("VAS copy/paste failed\n");
0610         return ret;
0611     }
0612 
0613     ret = wait_for_csb(wmem, csb);
0614     if (!ret)
0615         *outlenp = be32_to_cpu(csb->count);
0616 
0617     return ret;
0618 }
0619 
0620 /**
0621  * nx842_powernv_compress - Compress data using the 842 algorithm
0622  *
0623  * Compression provided by the NX842 coprocessor on IBM PowerNV systems.
0624  * The input buffer is compressed and the result is stored in the
0625  * provided output buffer.
0626  *
0627  * Upon return from this function @outlen contains the length of the
0628  * compressed data.  If there is an error then @outlen will be 0 and an
0629  * error will be specified by the return code from this function.
0630  *
0631  * @in: input buffer pointer
0632  * @inlen: input buffer size
0633  * @out: output buffer pointer
0634  * @outlenp: output buffer size pointer
0635  * @workmem: working memory buffer pointer, size determined by
0636  *           nx842_powernv_driver.workmem_size
0637  *
0638  * Returns: see @nx842_powernv_exec()
0639  */
0640 static int nx842_powernv_compress(const unsigned char *in, unsigned int inlen,
0641                   unsigned char *out, unsigned int *outlenp,
0642                   void *wmem)
0643 {
0644     return nx842_powernv_exec(in, inlen, out, outlenp,
0645                       wmem, CCW_FC_842_COMP_CRC);
0646 }
0647 
0648 /**
0649  * nx842_powernv_decompress - Decompress data using the 842 algorithm
0650  *
0651  * Decompression provided by the NX842 coprocessor on IBM PowerNV systems.
0652  * The input buffer is decompressed and the result is stored in the
0653  * provided output buffer.
0654  *
0655  * Upon return from this function @outlen contains the length of the
0656  * decompressed data.  If there is an error then @outlen will be 0 and an
0657  * error will be specified by the return code from this function.
0658  *
0659  * @in: input buffer pointer
0660  * @inlen: input buffer size
0661  * @out: output buffer pointer
0662  * @outlenp: output buffer size pointer
0663  * @wmem: working memory buffer pointer, size determined by
0664  *        nx842_powernv_driver.workmem_size
0665  *
0666  * Returns: see @nx842_powernv_exec()
0667  */
0668 static int nx842_powernv_decompress(const unsigned char *in, unsigned int inlen,
0669                     unsigned char *out, unsigned int *outlenp,
0670                     void *wmem)
0671 {
0672     return nx842_powernv_exec(in, inlen, out, outlenp,
0673                       wmem, CCW_FC_842_DECOMP_CRC);
0674 }
0675 
0676 static inline void nx_add_coprocs_list(struct nx_coproc *coproc,
0677                     int chipid)
0678 {
0679     coproc->chip_id = chipid;
0680     INIT_LIST_HEAD(&coproc->list);
0681     list_add(&coproc->list, &nx_coprocs);
0682 }
0683 
0684 static struct vas_window *nx_alloc_txwin(struct nx_coproc *coproc)
0685 {
0686     struct vas_window *txwin = NULL;
0687     struct vas_tx_win_attr txattr;
0688 
0689     /*
0690      * Kernel requests will be high priority. So open send
0691      * windows only for high priority RxFIFO entries.
0692      */
0693     vas_init_tx_win_attr(&txattr, coproc->ct);
0694     txattr.lpid = 0;    /* lpid is 0 for kernel requests */
0695 
0696     /*
0697      * Open a VAS send window which is used to send request to NX.
0698      */
0699     txwin = vas_tx_win_open(coproc->vas.id, coproc->ct, &txattr);
0700     if (IS_ERR(txwin))
0701         pr_err("ibm,nx-842: Can not open TX window: %ld\n",
0702                 PTR_ERR(txwin));
0703 
0704     return txwin;
0705 }
0706 
0707 /*
0708  * Identify chip ID for each CPU, open send wndow for the corresponding NX
0709  * engine and save txwin in percpu cpu_txwin.
0710  * cpu_txwin is used in copy/paste operation for each compression /
0711  * decompression request.
0712  */
0713 static int nx_open_percpu_txwins(void)
0714 {
0715     struct nx_coproc *coproc, *n;
0716     unsigned int i, chip_id;
0717 
0718     for_each_possible_cpu(i) {
0719         struct vas_window *txwin = NULL;
0720 
0721         chip_id = cpu_to_chip_id(i);
0722 
0723         list_for_each_entry_safe(coproc, n, &nx_coprocs, list) {
0724             /*
0725              * Kernel requests use only high priority FIFOs. So
0726              * open send windows for these FIFOs.
0727              * GZIP is not supported in kernel right now.
0728              */
0729 
0730             if (coproc->ct != VAS_COP_TYPE_842_HIPRI)
0731                 continue;
0732 
0733             if (coproc->chip_id == chip_id) {
0734                 txwin = nx_alloc_txwin(coproc);
0735                 if (IS_ERR(txwin))
0736                     return PTR_ERR(txwin);
0737 
0738                 per_cpu(cpu_txwin, i) = txwin;
0739                 break;
0740             }
0741         }
0742 
0743         if (!per_cpu(cpu_txwin, i)) {
0744             /* shouldn't happen, Each chip will have NX engine */
0745             pr_err("NX engine is not available for CPU %d\n", i);
0746             return -EINVAL;
0747         }
0748     }
0749 
0750     return 0;
0751 }
0752 
0753 static int __init nx_set_ct(struct nx_coproc *coproc, const char *priority,
0754                 int high, int normal)
0755 {
0756     if (!strcmp(priority, "High"))
0757         coproc->ct = high;
0758     else if (!strcmp(priority, "Normal"))
0759         coproc->ct = normal;
0760     else {
0761         pr_err("Invalid RxFIFO priority value\n");
0762         return -EINVAL;
0763     }
0764 
0765     return 0;
0766 }
0767 
0768 static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id,
0769                     int vasid, int type, int *ct)
0770 {
0771     struct vas_window *rxwin = NULL;
0772     struct vas_rx_win_attr rxattr;
0773     u32 lpid, pid, tid, fifo_size;
0774     struct nx_coproc *coproc;
0775     u64 rx_fifo;
0776     const char *priority;
0777     int ret;
0778 
0779     ret = of_property_read_u64(dn, "rx-fifo-address", &rx_fifo);
0780     if (ret) {
0781         pr_err("Missing rx-fifo-address property\n");
0782         return ret;
0783     }
0784 
0785     ret = of_property_read_u32(dn, "rx-fifo-size", &fifo_size);
0786     if (ret) {
0787         pr_err("Missing rx-fifo-size property\n");
0788         return ret;
0789     }
0790 
0791     ret = of_property_read_u32(dn, "lpid", &lpid);
0792     if (ret) {
0793         pr_err("Missing lpid property\n");
0794         return ret;
0795     }
0796 
0797     ret = of_property_read_u32(dn, "pid", &pid);
0798     if (ret) {
0799         pr_err("Missing pid property\n");
0800         return ret;
0801     }
0802 
0803     ret = of_property_read_u32(dn, "tid", &tid);
0804     if (ret) {
0805         pr_err("Missing tid property\n");
0806         return ret;
0807     }
0808 
0809     ret = of_property_read_string(dn, "priority", &priority);
0810     if (ret) {
0811         pr_err("Missing priority property\n");
0812         return ret;
0813     }
0814 
0815     coproc = kzalloc(sizeof(*coproc), GFP_KERNEL);
0816     if (!coproc)
0817         return -ENOMEM;
0818 
0819     if (type == NX_CT_842)
0820         ret = nx_set_ct(coproc, priority, VAS_COP_TYPE_842_HIPRI,
0821             VAS_COP_TYPE_842);
0822     else if (type == NX_CT_GZIP)
0823         ret = nx_set_ct(coproc, priority, VAS_COP_TYPE_GZIP_HIPRI,
0824                 VAS_COP_TYPE_GZIP);
0825 
0826     if (ret)
0827         goto err_out;
0828 
0829     vas_init_rx_win_attr(&rxattr, coproc->ct);
0830     rxattr.rx_fifo = rx_fifo;
0831     rxattr.rx_fifo_size = fifo_size;
0832     rxattr.lnotify_lpid = lpid;
0833     rxattr.lnotify_pid = pid;
0834     rxattr.lnotify_tid = tid;
0835     /*
0836      * Maximum RX window credits can not be more than #CRBs in
0837      * RxFIFO. Otherwise, can get checkstop if RxFIFO overruns.
0838      */
0839     rxattr.wcreds_max = fifo_size / CRB_SIZE;
0840 
0841     /*
0842      * Open a VAS receice window which is used to configure RxFIFO
0843      * for NX.
0844      */
0845     rxwin = vas_rx_win_open(vasid, coproc->ct, &rxattr);
0846     if (IS_ERR(rxwin)) {
0847         ret = PTR_ERR(rxwin);
0848         pr_err("setting RxFIFO with VAS failed: %d\n",
0849             ret);
0850         goto err_out;
0851     }
0852 
0853     coproc->vas.rxwin = rxwin;
0854     coproc->vas.id = vasid;
0855     nx_add_coprocs_list(coproc, chip_id);
0856 
0857     /*
0858      * (lpid, pid, tid) combination has to be unique for each
0859      * coprocessor instance in the system. So to make it
0860      * unique, skiboot uses coprocessor type such as 842 or
0861      * GZIP for pid and provides this value to kernel in pid
0862      * device-tree property.
0863      */
0864     *ct = pid;
0865 
0866     return 0;
0867 
0868 err_out:
0869     kfree(coproc);
0870     return ret;
0871 }
0872 
0873 static int __init nx_coproc_init(int chip_id, int ct_842, int ct_gzip)
0874 {
0875     int ret = 0;
0876 
0877     if (opal_check_token(OPAL_NX_COPROC_INIT)) {
0878         ret = opal_nx_coproc_init(chip_id, ct_842);
0879 
0880         if (!ret)
0881             ret = opal_nx_coproc_init(chip_id, ct_gzip);
0882 
0883         if (ret) {
0884             ret = opal_error_code(ret);
0885             pr_err("Failed to initialize NX for chip(%d): %d\n",
0886                 chip_id, ret);
0887         }
0888     } else
0889         pr_warn("Firmware doesn't support NX initialization\n");
0890 
0891     return ret;
0892 }
0893 
0894 static int __init find_nx_device_tree(struct device_node *dn, int chip_id,
0895                     int vasid, int type, char *devname,
0896                     int *ct)
0897 {
0898     int ret = 0;
0899 
0900     if (of_device_is_compatible(dn, devname)) {
0901         ret  = vas_cfg_coproc_info(dn, chip_id, vasid, type, ct);
0902         if (ret)
0903             of_node_put(dn);
0904     }
0905 
0906     return ret;
0907 }
0908 
0909 static int __init nx_powernv_probe_vas(struct device_node *pn)
0910 {
0911     int chip_id, vasid, ret = 0;
0912     int ct_842 = 0, ct_gzip = 0;
0913     struct device_node *dn;
0914 
0915     chip_id = of_get_ibm_chip_id(pn);
0916     if (chip_id < 0) {
0917         pr_err("ibm,chip-id missing\n");
0918         return -EINVAL;
0919     }
0920 
0921     vasid = chip_to_vas_id(chip_id);
0922     if (vasid < 0) {
0923         pr_err("Unable to map chip_id %d to vasid\n", chip_id);
0924         return -EINVAL;
0925     }
0926 
0927     for_each_child_of_node(pn, dn) {
0928         ret = find_nx_device_tree(dn, chip_id, vasid, NX_CT_842,
0929                     "ibm,p9-nx-842", &ct_842);
0930 
0931         if (!ret)
0932             ret = find_nx_device_tree(dn, chip_id, vasid,
0933                 NX_CT_GZIP, "ibm,p9-nx-gzip", &ct_gzip);
0934 
0935         if (ret) {
0936             of_node_put(dn);
0937             return ret;
0938         }
0939     }
0940 
0941     if (!ct_842 || !ct_gzip) {
0942         pr_err("NX FIFO nodes are missing\n");
0943         return -EINVAL;
0944     }
0945 
0946     /*
0947      * Initialize NX instance for both high and normal priority FIFOs.
0948      */
0949     ret = nx_coproc_init(chip_id, ct_842, ct_gzip);
0950 
0951     return ret;
0952 }
0953 
0954 static int __init nx842_powernv_probe(struct device_node *dn)
0955 {
0956     struct nx_coproc *coproc;
0957     unsigned int ct, ci;
0958     int chip_id;
0959 
0960     chip_id = of_get_ibm_chip_id(dn);
0961     if (chip_id < 0) {
0962         pr_err("ibm,chip-id missing\n");
0963         return -EINVAL;
0964     }
0965 
0966     if (of_property_read_u32(dn, "ibm,842-coprocessor-type", &ct)) {
0967         pr_err("ibm,842-coprocessor-type missing\n");
0968         return -EINVAL;
0969     }
0970 
0971     if (of_property_read_u32(dn, "ibm,842-coprocessor-instance", &ci)) {
0972         pr_err("ibm,842-coprocessor-instance missing\n");
0973         return -EINVAL;
0974     }
0975 
0976     coproc = kzalloc(sizeof(*coproc), GFP_KERNEL);
0977     if (!coproc)
0978         return -ENOMEM;
0979 
0980     coproc->ct = ct;
0981     coproc->ci = ci;
0982     nx_add_coprocs_list(coproc, chip_id);
0983 
0984     pr_info("coprocessor found on chip %d, CT %d CI %d\n", chip_id, ct, ci);
0985 
0986     if (!nx842_ct)
0987         nx842_ct = ct;
0988     else if (nx842_ct != ct)
0989         pr_err("NX842 chip %d, CT %d != first found CT %d\n",
0990                chip_id, ct, nx842_ct);
0991 
0992     return 0;
0993 }
0994 
0995 static void nx_delete_coprocs(void)
0996 {
0997     struct nx_coproc *coproc, *n;
0998     struct vas_window *txwin;
0999     int i;
1000 
1001     /*
1002      * close percpu txwins that are opened for the corresponding coproc.
1003      */
1004     for_each_possible_cpu(i) {
1005         txwin = per_cpu(cpu_txwin, i);
1006         if (txwin)
1007             vas_win_close(txwin);
1008 
1009         per_cpu(cpu_txwin, i) = NULL;
1010     }
1011 
1012     list_for_each_entry_safe(coproc, n, &nx_coprocs, list) {
1013         if (coproc->vas.rxwin)
1014             vas_win_close(coproc->vas.rxwin);
1015 
1016         list_del(&coproc->list);
1017         kfree(coproc);
1018     }
1019 }
1020 
1021 static struct nx842_constraints nx842_powernv_constraints = {
1022     .alignment =    DDE_BUFFER_ALIGN,
1023     .multiple = DDE_BUFFER_LAST_MULT,
1024     .minimum =  DDE_BUFFER_LAST_MULT,
1025     .maximum =  (DDL_LEN_MAX - 1) * PAGE_SIZE,
1026 };
1027 
1028 static struct nx842_driver nx842_powernv_driver = {
1029     .name =     KBUILD_MODNAME,
1030     .owner =    THIS_MODULE,
1031     .workmem_size = sizeof(struct nx842_workmem),
1032     .constraints =  &nx842_powernv_constraints,
1033     .compress = nx842_powernv_compress,
1034     .decompress =   nx842_powernv_decompress,
1035 };
1036 
1037 static int nx842_powernv_crypto_init(struct crypto_tfm *tfm)
1038 {
1039     return nx842_crypto_init(tfm, &nx842_powernv_driver);
1040 }
1041 
1042 static struct crypto_alg nx842_powernv_alg = {
1043     .cra_name       = "842",
1044     .cra_driver_name    = "842-nx",
1045     .cra_priority       = 300,
1046     .cra_flags      = CRYPTO_ALG_TYPE_COMPRESS,
1047     .cra_ctxsize        = sizeof(struct nx842_crypto_ctx),
1048     .cra_module     = THIS_MODULE,
1049     .cra_init       = nx842_powernv_crypto_init,
1050     .cra_exit       = nx842_crypto_exit,
1051     .cra_u          = { .compress = {
1052     .coa_compress       = nx842_crypto_compress,
1053     .coa_decompress     = nx842_crypto_decompress } }
1054 };
1055 
1056 static __init int nx_compress_powernv_init(void)
1057 {
1058     struct device_node *dn;
1059     int ret;
1060 
1061     /* verify workmem size/align restrictions */
1062     BUILD_BUG_ON(WORKMEM_ALIGN % CRB_ALIGN);
1063     BUILD_BUG_ON(CRB_ALIGN % DDE_ALIGN);
1064     BUILD_BUG_ON(CRB_SIZE % DDE_ALIGN);
1065     /* verify buffer size/align restrictions */
1066     BUILD_BUG_ON(PAGE_SIZE % DDE_BUFFER_ALIGN);
1067     BUILD_BUG_ON(DDE_BUFFER_ALIGN % DDE_BUFFER_SIZE_MULT);
1068     BUILD_BUG_ON(DDE_BUFFER_SIZE_MULT % DDE_BUFFER_LAST_MULT);
1069 
1070     for_each_compatible_node(dn, NULL, "ibm,power9-nx") {
1071         ret = nx_powernv_probe_vas(dn);
1072         if (ret) {
1073             nx_delete_coprocs();
1074             of_node_put(dn);
1075             return ret;
1076         }
1077     }
1078 
1079     if (list_empty(&nx_coprocs)) {
1080         for_each_compatible_node(dn, NULL, "ibm,power-nx")
1081             nx842_powernv_probe(dn);
1082 
1083         if (!nx842_ct)
1084             return -ENODEV;
1085 
1086         nx842_powernv_exec = nx842_exec_icswx;
1087     } else {
1088         /*
1089          * Register VAS user space API for NX GZIP so
1090          * that user space can use GZIP engine.
1091          * Using high FIFO priority for kernel requests and
1092          * normal FIFO priority is assigned for userspace.
1093          * 842 compression is supported only in kernel.
1094          */
1095         ret = vas_register_api_powernv(THIS_MODULE, VAS_COP_TYPE_GZIP,
1096                            "nx-gzip");
1097 
1098         /*
1099          * GZIP is not supported in kernel right now.
1100          * So open tx windows only for 842.
1101          */
1102         if (!ret)
1103             ret = nx_open_percpu_txwins();
1104 
1105         if (ret) {
1106             nx_delete_coprocs();
1107             return ret;
1108         }
1109 
1110         nx842_powernv_exec = nx842_exec_vas;
1111     }
1112 
1113     ret = crypto_register_alg(&nx842_powernv_alg);
1114     if (ret) {
1115         nx_delete_coprocs();
1116         return ret;
1117     }
1118 
1119     return 0;
1120 }
1121 module_init(nx_compress_powernv_init);
1122 
1123 static void __exit nx_compress_powernv_exit(void)
1124 {
1125     /*
1126      * GZIP engine is supported only in power9 or later and nx842_ct
1127      * is used on power8 (icswx).
1128      * VAS API for NX GZIP is registered during init for user space
1129      * use. So delete this API use for GZIP engine.
1130      */
1131     if (!nx842_ct)
1132         vas_unregister_api_powernv();
1133 
1134     crypto_unregister_alg(&nx842_powernv_alg);
1135 
1136     nx_delete_coprocs();
1137 }
1138 module_exit(nx_compress_powernv_exit);
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!