crypto/nx/nx-842.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * Cryptographic API for the NX-842 hardware compression.
0004  *
0005  * Copyright (C) IBM Corporation, 2011-2015
0006  *
0007  * Designer of the Power data compression engine:
0008  *   Bulent Abali <abali@us.ibm.com>
0009  *
0010  * Original Authors: Robert Jennings <rcj@linux.vnet.ibm.com>
0011  *                   Seth Jennings <sjenning@linux.vnet.ibm.com>
0012  *
0013  * Rewrite: Dan Streetman <ddstreet@ieee.org>
0014  *
0015  * This is an interface to the NX-842 compression hardware in PowerPC
0016  * processors.  Most of the complexity of this drvier is due to the fact that
0017  * the NX-842 compression hardware requires the input and output data buffers
0018  * to be specifically aligned, to be a specific multiple in length, and within
0019  * specific minimum and maximum lengths.  Those restrictions, provided by the
0020  * nx-842 driver via nx842_constraints, mean this driver must use bounce
0021  * buffers and headers to correct misaligned in or out buffers, and to split
0022  * input buffers that are too large.
0023  *
0024  * This driver will fall back to software decompression if the hardware
0025  * decompression fails, so this driver's decompression should never fail as
0026  * long as the provided compressed buffer is valid.  Any compressed buffer
0027  * created by this driver will have a header (except ones where the input
0028  * perfectly matches the constraints); so users of this driver cannot simply
0029  * pass a compressed buffer created by this driver over to the 842 software
0030  * decompression library.  Instead, users must use this driver to decompress;
0031  * if the hardware fails or is unavailable, the compressed buffer will be
0032  * parsed and the header removed, and the raw 842 buffer(s) passed to the 842
0033  * software decompression library.
0034  *
0035  * This does not fall back to software compression, however, since the caller
0036  * of this function is specifically requesting hardware compression; if the
0037  * hardware compression fails, the caller can fall back to software
0038  * compression, and the raw 842 compressed buffer that the software compressor
0039  * creates can be passed to this driver for hardware decompression; any
0040  * buffer without our specific header magic is assumed to be a raw 842 buffer
0041  * and passed directly to the hardware.  Note that the software compression
0042  * library will produce a compressed buffer that is incompatible with the
0043  * hardware decompressor if the original input buffer length is not a multiple
0044  * of 8; if such a compressed buffer is passed to this driver for
0045  * decompression, the hardware will reject it and this driver will then pass
0046  * it over to the software library for decompression.
0047  */
0048
0049 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
0050
0051 #include <linux/vmalloc.h>
0052 #include <linux/sw842.h>
0053 #include <linux/spinlock.h>
0054
0055 #include "nx-842.h"
0056
0057 /* The first 5 bits of this magic are 0x1f, which is an invalid 842 5-bit
0058  * template (see lib/842/842.h), so this magic number will never appear at
0059  * the start of a raw 842 compressed buffer.  That is important, as any buffer
0060  * passed to us without this magic is assumed to be a raw 842 compressed
0061  * buffer, and passed directly to the hardware to decompress.
0062  */
0063 #define NX842_CRYPTO_MAGIC  (0xf842)
0064 #define NX842_CRYPTO_HEADER_SIZE(g)             \
0065     (sizeof(struct nx842_crypto_header) +           \
0066      sizeof(struct nx842_crypto_header_group) * (g))
0067 #define NX842_CRYPTO_HEADER_MAX_SIZE                \
0068     NX842_CRYPTO_HEADER_SIZE(NX842_CRYPTO_GROUP_MAX)
0069
0070 /* bounce buffer size */
0071 #define BOUNCE_BUFFER_ORDER (2)
0072 #define BOUNCE_BUFFER_SIZE                  \
0073     ((unsigned int)(PAGE_SIZE << BOUNCE_BUFFER_ORDER))
0074
0075 /* try longer on comp because we can fallback to sw decomp if hw is busy */
0076 #define COMP_BUSY_TIMEOUT   (250) /* ms */
0077 #define DECOMP_BUSY_TIMEOUT (50) /* ms */
0078
0079 struct nx842_crypto_param {
0080     u8 *in;
0081     unsigned int iremain;
0082     u8 *out;
0083     unsigned int oremain;
0084     unsigned int ototal;
0085 };
0086
0087 static int update_param(struct nx842_crypto_param *p,
0088             unsigned int slen, unsigned int dlen)
0089 {
0090     if (p->iremain < slen)
0091         return -EOVERFLOW;
0092     if (p->oremain < dlen)
0093         return -ENOSPC;
0094
0095     p->in += slen;
0096     p->iremain -= slen;
0097     p->out += dlen;
0098     p->oremain -= dlen;
0099     p->ototal += dlen;
0100
0101     return 0;
0102 }
0103
0104 int nx842_crypto_init(struct crypto_tfm *tfm, struct nx842_driver *driver)
0105 {
0106     struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);
0107
0108     spin_lock_init(&ctx->lock);
0109     ctx->driver = driver;
0110     ctx->wmem = kmalloc(driver->workmem_size, GFP_KERNEL);
0111     ctx->sbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER);
0112     ctx->dbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER);
0113     if (!ctx->wmem || !ctx->sbounce || !ctx->dbounce) {
0114         kfree(ctx->wmem);
0115         free_page((unsigned long)ctx->sbounce);
0116         free_page((unsigned long)ctx->dbounce);
0117         return -ENOMEM;
0118     }
0119
0120     return 0;
0121 }
0122 EXPORT_SYMBOL_GPL(nx842_crypto_init);
0123
0124 void nx842_crypto_exit(struct crypto_tfm *tfm)
0125 {
0126     struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);
0127
0128     kfree(ctx->wmem);
0129     free_page((unsigned long)ctx->sbounce);
0130     free_page((unsigned long)ctx->dbounce);
0131 }
0132 EXPORT_SYMBOL_GPL(nx842_crypto_exit);
0133
0134 static void check_constraints(struct nx842_constraints *c)
0135 {
0136     /* limit maximum, to always have enough bounce buffer to decompress */
0137     if (c->maximum > BOUNCE_BUFFER_SIZE)
0138         c->maximum = BOUNCE_BUFFER_SIZE;
0139 }
0140
0141 static int nx842_crypto_add_header(struct nx842_crypto_header *hdr, u8 *buf)
0142 {
0143     int s = NX842_CRYPTO_HEADER_SIZE(hdr->groups);
0144
0145     /* compress should have added space for header */
0146     if (s > be16_to_cpu(hdr->group[0].padding)) {
0147         pr_err("Internal error: no space for header\n");
0148         return -EINVAL;
0149     }
0150
0151     memcpy(buf, hdr, s);
0152
0153     print_hex_dump_debug("header ", DUMP_PREFIX_OFFSET, 16, 1, buf, s, 0);
0154
0155     return 0;
0156 }
0157
0158 static int compress(struct nx842_crypto_ctx *ctx,
0159             struct nx842_crypto_param *p,
0160             struct nx842_crypto_header_group *g,
0161             struct nx842_constraints *c,
0162             u16 *ignore,
0163             unsigned int hdrsize)
0164 {
0165     unsigned int slen = p->iremain, dlen = p->oremain, tmplen;
0166     unsigned int adj_slen = slen;
0167     u8 *src = p->in, *dst = p->out;
0168     int ret, dskip = 0;
0169     ktime_t timeout;
0170
0171     if (p->iremain == 0)
0172         return -EOVERFLOW;
0173
0174     if (p->oremain == 0 || hdrsize + c->minimum > dlen)
0175         return -ENOSPC;
0176
0177     if (slen % c->multiple)
0178         adj_slen = round_up(slen, c->multiple);
0179     if (slen < c->minimum)
0180         adj_slen = c->minimum;
0181     if (slen > c->maximum)
0182         adj_slen = slen = c->maximum;
0183     if (adj_slen > slen || (u64)src % c->alignment) {
0184         adj_slen = min(adj_slen, BOUNCE_BUFFER_SIZE);
0185         slen = min(slen, BOUNCE_BUFFER_SIZE);
0186         if (adj_slen > slen)
0187             memset(ctx->sbounce + slen, 0, adj_slen - slen);
0188         memcpy(ctx->sbounce, src, slen);
0189         src = ctx->sbounce;
0190         slen = adj_slen;
0191         pr_debug("using comp sbounce buffer, len %x\n", slen);
0192     }
0193
0194     dst += hdrsize;
0195     dlen -= hdrsize;
0196
0197     if ((u64)dst % c->alignment) {
0198         dskip = (int)(PTR_ALIGN(dst, c->alignment) - dst);
0199         dst += dskip;
0200         dlen -= dskip;
0201     }
0202     if (dlen % c->multiple)
0203         dlen = round_down(dlen, c->multiple);
0204     if (dlen < c->minimum) {
0205 nospc:
0206         dst = ctx->dbounce;
0207         dlen = min(p->oremain, BOUNCE_BUFFER_SIZE);
0208         dlen = round_down(dlen, c->multiple);
0209         dskip = 0;
0210         pr_debug("using comp dbounce buffer, len %x\n", dlen);
0211     }
0212     if (dlen > c->maximum)
0213         dlen = c->maximum;
0214
0215     tmplen = dlen;
0216     timeout = ktime_add_ms(ktime_get(), COMP_BUSY_TIMEOUT);
0217     do {
0218         dlen = tmplen; /* reset dlen, if we're retrying */
0219         ret = ctx->driver->compress(src, slen, dst, &dlen, ctx->wmem);
0220         /* possibly we should reduce the slen here, instead of
0221          * retrying with the dbounce buffer?
0222          */
0223         if (ret == -ENOSPC && dst != ctx->dbounce)
0224             goto nospc;
0225     } while (ret == -EBUSY && ktime_before(ktime_get(), timeout));
0226     if (ret)
0227         return ret;
0228
0229     dskip += hdrsize;
0230
0231     if (dst == ctx->dbounce)
0232         memcpy(p->out + dskip, dst, dlen);
0233
0234     g->padding = cpu_to_be16(dskip);
0235     g->compressed_length = cpu_to_be32(dlen);
0236     g->uncompressed_length = cpu_to_be32(slen);
0237
0238     if (p->iremain < slen) {
0239         *ignore = slen - p->iremain;
0240         slen = p->iremain;
0241     }
0242
0243     pr_debug("compress slen %x ignore %x dlen %x padding %x\n",
0244          slen, *ignore, dlen, dskip);
0245
0246     return update_param(p, slen, dskip + dlen);
0247 }
0248
0249 int nx842_crypto_compress(struct crypto_tfm *tfm,
0250               const u8 *src, unsigned int slen,
0251               u8 *dst, unsigned int *dlen)
0252 {
0253     struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);
0254     struct nx842_crypto_header *hdr = &ctx->header;
0255     struct nx842_crypto_param p;
0256     struct nx842_constraints c = *ctx->driver->constraints;
0257     unsigned int groups, hdrsize, h;
0258     int ret, n;
0259     bool add_header;
0260     u16 ignore = 0;
0261
0262     check_constraints(&c);
0263
0264     p.in = (u8 *)src;
0265     p.iremain = slen;
0266     p.out = dst;
0267     p.oremain = *dlen;
0268     p.ototal = 0;
0269
0270     *dlen = 0;
0271
0272     groups = min_t(unsigned int, NX842_CRYPTO_GROUP_MAX,
0273                DIV_ROUND_UP(p.iremain, c.maximum));
0274     hdrsize = NX842_CRYPTO_HEADER_SIZE(groups);
0275
0276     spin_lock_bh(&ctx->lock);
0277
0278     /* skip adding header if the buffers meet all constraints */
0279     add_header = (p.iremain % c.multiple    ||
0280               p.iremain < c.minimum ||
0281               p.iremain > c.maximum ||
0282               (u64)p.in % c.alignment   ||
0283               p.oremain % c.multiple    ||
0284               p.oremain < c.minimum ||
0285               p.oremain > c.maximum ||
0286               (u64)p.out % c.alignment);
0287
0288     hdr->magic = cpu_to_be16(NX842_CRYPTO_MAGIC);
0289     hdr->groups = 0;
0290     hdr->ignore = 0;
0291
0292     while (p.iremain > 0) {
0293         n = hdr->groups++;
0294         ret = -ENOSPC;
0295         if (hdr->groups > NX842_CRYPTO_GROUP_MAX)
0296             goto unlock;
0297
0298         /* header goes before first group */
0299         h = !n && add_header ? hdrsize : 0;
0300
0301         if (ignore)
0302             pr_warn("internal error, ignore is set %x\n", ignore);
0303
0304         ret = compress(ctx, &p, &hdr->group[n], &c, &ignore, h);
0305         if (ret)
0306             goto unlock;
0307     }
0308
0309     if (!add_header && hdr->groups > 1) {
0310         pr_err("Internal error: No header but multiple groups\n");
0311         ret = -EINVAL;
0312         goto unlock;
0313     }
0314
0315     /* ignore indicates the input stream needed to be padded */
0316     hdr->ignore = cpu_to_be16(ignore);
0317     if (ignore)
0318         pr_debug("marked %d bytes as ignore\n", ignore);
0319
0320     if (add_header)
0321         ret = nx842_crypto_add_header(hdr, dst);
0322     if (ret)
0323         goto unlock;
0324
0325     *dlen = p.ototal;
0326
0327     pr_debug("compress total slen %x dlen %x\n", slen, *dlen);
0328
0329 unlock:
0330     spin_unlock_bh(&ctx->lock);
0331     return ret;
0332 }
0333 EXPORT_SYMBOL_GPL(nx842_crypto_compress);
0334
0335 static int decompress(struct nx842_crypto_ctx *ctx,
0336               struct nx842_crypto_param *p,
0337               struct nx842_crypto_header_group *g,
0338               struct nx842_constraints *c,
0339               u16 ignore)
0340 {
0341     unsigned int slen = be32_to_cpu(g->compressed_length);
0342     unsigned int required_len = be32_to_cpu(g->uncompressed_length);
0343     unsigned int dlen = p->oremain, tmplen;
0344     unsigned int adj_slen = slen;
0345     u8 *src = p->in, *dst = p->out;
0346     u16 padding = be16_to_cpu(g->padding);
0347     int ret, spadding = 0;
0348     ktime_t timeout;
0349
0350     if (!slen || !required_len)
0351         return -EINVAL;
0352
0353     if (p->iremain <= 0 || padding + slen > p->iremain)
0354         return -EOVERFLOW;
0355
0356     if (p->oremain <= 0 || required_len - ignore > p->oremain)
0357         return -ENOSPC;
0358
0359     src += padding;
0360
0361     if (slen % c->multiple)
0362         adj_slen = round_up(slen, c->multiple);
0363     if (slen < c->minimum)
0364         adj_slen = c->minimum;
0365     if (slen > c->maximum)
0366         goto usesw;
0367     if (slen < adj_slen || (u64)src % c->alignment) {
0368         /* we can append padding bytes because the 842 format defines
0369          * an "end" template (see lib/842/842_decompress.c) and will
0370          * ignore any bytes following it.
0371          */
0372         if (slen < adj_slen)
0373             memset(ctx->sbounce + slen, 0, adj_slen - slen);
0374         memcpy(ctx->sbounce, src, slen);
0375         src = ctx->sbounce;
0376         spadding = adj_slen - slen;
0377         slen = adj_slen;
0378         pr_debug("using decomp sbounce buffer, len %x\n", slen);
0379     }
0380
0381     if (dlen % c->multiple)
0382         dlen = round_down(dlen, c->multiple);
0383     if (dlen < required_len || (u64)dst % c->alignment) {
0384         dst = ctx->dbounce;
0385         dlen = min(required_len, BOUNCE_BUFFER_SIZE);
0386         pr_debug("using decomp dbounce buffer, len %x\n", dlen);
0387     }
0388     if (dlen < c->minimum)
0389         goto usesw;
0390     if (dlen > c->maximum)
0391         dlen = c->maximum;
0392
0393     tmplen = dlen;
0394     timeout = ktime_add_ms(ktime_get(), DECOMP_BUSY_TIMEOUT);
0395     do {
0396         dlen = tmplen; /* reset dlen, if we're retrying */
0397         ret = ctx->driver->decompress(src, slen, dst, &dlen, ctx->wmem);
0398     } while (ret == -EBUSY && ktime_before(ktime_get(), timeout));
0399     if (ret) {
0400 usesw:
0401         /* reset everything, sw doesn't have constraints */
0402         src = p->in + padding;
0403         slen = be32_to_cpu(g->compressed_length);
0404         spadding = 0;
0405         dst = p->out;
0406         dlen = p->oremain;
0407         if (dlen < required_len) { /* have ignore bytes */
0408             dst = ctx->dbounce;
0409             dlen = BOUNCE_BUFFER_SIZE;
0410         }
0411         pr_info_ratelimited("using software 842 decompression\n");
0412         ret = sw842_decompress(src, slen, dst, &dlen);
0413     }
0414     if (ret)
0415         return ret;
0416
0417     slen -= spadding;
0418
0419     dlen -= ignore;
0420     if (ignore)
0421         pr_debug("ignoring last %x bytes\n", ignore);
0422
0423     if (dst == ctx->dbounce)
0424         memcpy(p->out, dst, dlen);
0425
0426     pr_debug("decompress slen %x padding %x dlen %x ignore %x\n",
0427          slen, padding, dlen, ignore);
0428
0429     return update_param(p, slen + padding, dlen);
0430 }
0431
0432 int nx842_crypto_decompress(struct crypto_tfm *tfm,
0433                 const u8 *src, unsigned int slen,
0434                 u8 *dst, unsigned int *dlen)
0435 {
0436     struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);
0437     struct nx842_crypto_header *hdr;
0438     struct nx842_crypto_param p;
0439     struct nx842_constraints c = *ctx->driver->constraints;
0440     int n, ret, hdr_len;
0441     u16 ignore = 0;
0442
0443     check_constraints(&c);
0444
0445     p.in = (u8 *)src;
0446     p.iremain = slen;
0447     p.out = dst;
0448     p.oremain = *dlen;
0449     p.ototal = 0;
0450
0451     *dlen = 0;
0452
0453     hdr = (struct nx842_crypto_header *)src;
0454
0455     spin_lock_bh(&ctx->lock);
0456
0457     /* If it doesn't start with our header magic number, assume it's a raw
0458      * 842 compressed buffer and pass it directly to the hardware driver
0459      */
0460     if (be16_to_cpu(hdr->magic) != NX842_CRYPTO_MAGIC) {
0461         struct nx842_crypto_header_group g = {
0462             .padding =      0,
0463             .compressed_length =    cpu_to_be32(p.iremain),
0464             .uncompressed_length =  cpu_to_be32(p.oremain),
0465         };
0466
0467         ret = decompress(ctx, &p, &g, &c, 0);
0468         if (ret)
0469             goto unlock;
0470
0471         goto success;
0472     }
0473
0474     if (!hdr->groups) {
0475         pr_err("header has no groups\n");
0476         ret = -EINVAL;
0477         goto unlock;
0478     }
0479     if (hdr->groups > NX842_CRYPTO_GROUP_MAX) {
0480         pr_err("header has too many groups %x, max %x\n",
0481                hdr->groups, NX842_CRYPTO_GROUP_MAX);
0482         ret = -EINVAL;
0483         goto unlock;
0484     }
0485
0486     hdr_len = NX842_CRYPTO_HEADER_SIZE(hdr->groups);
0487     if (hdr_len > slen) {
0488         ret = -EOVERFLOW;
0489         goto unlock;
0490     }
0491
0492     memcpy(&ctx->header, src, hdr_len);
0493     hdr = &ctx->header;
0494
0495     for (n = 0; n < hdr->groups; n++) {
0496         /* ignore applies to last group */
0497         if (n + 1 == hdr->groups)
0498             ignore = be16_to_cpu(hdr->ignore);
0499
0500         ret = decompress(ctx, &p, &hdr->group[n], &c, ignore);
0501         if (ret)
0502             goto unlock;
0503     }
0504
0505 success:
0506     *dlen = p.ototal;
0507
0508     pr_debug("decompress total slen %x dlen %x\n", slen, *dlen);
0509
0510     ret = 0;
0511
0512 unlock:
0513     spin_unlock_bh(&ctx->lock);
0514
0515     return ret;
0516 }
0517 EXPORT_SYMBOL_GPL(nx842_crypto_decompress);
0518
0519 MODULE_LICENSE("GPL");
0520 MODULE_DESCRIPTION("IBM PowerPC Nest (NX) 842 Hardware Compression Driver");
0521 MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");