block/zram/zcomp.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * Copyright (C) 2014 Sergey Senozhatsky.
0004  */
0005
0006 #include <linux/kernel.h>
0007 #include <linux/string.h>
0008 #include <linux/err.h>
0009 #include <linux/slab.h>
0010 #include <linux/wait.h>
0011 #include <linux/sched.h>
0012 #include <linux/cpu.h>
0013 #include <linux/crypto.h>
0014
0015 #include "zcomp.h"
0016
0017 static const char * const backends[] = {
0018 #if IS_ENABLED(CONFIG_CRYPTO_LZO)
0019     "lzo",
0020     "lzo-rle",
0021 #endif
0022 #if IS_ENABLED(CONFIG_CRYPTO_LZ4)
0023     "lz4",
0024 #endif
0025 #if IS_ENABLED(CONFIG_CRYPTO_LZ4HC)
0026     "lz4hc",
0027 #endif
0028 #if IS_ENABLED(CONFIG_CRYPTO_842)
0029     "842",
0030 #endif
0031 #if IS_ENABLED(CONFIG_CRYPTO_ZSTD)
0032     "zstd",
0033 #endif
0034 };
0035
0036 static void zcomp_strm_free(struct zcomp_strm *zstrm)
0037 {
0038     if (!IS_ERR_OR_NULL(zstrm->tfm))
0039         crypto_free_comp(zstrm->tfm);
0040     free_pages((unsigned long)zstrm->buffer, 1);
0041     zstrm->tfm = NULL;
0042     zstrm->buffer = NULL;
0043 }
0044
0045 /*
0046  * Initialize zcomp_strm structure with ->tfm initialized by backend, and
0047  * ->buffer. Return a negative value on error.
0048  */
0049 static int zcomp_strm_init(struct zcomp_strm *zstrm, struct zcomp *comp)
0050 {
0051     zstrm->tfm = crypto_alloc_comp(comp->name, 0, 0);
0052     /*
0053      * allocate 2 pages. 1 for compressed data, plus 1 extra for the
0054      * case when compressed size is larger than the original one
0055      */
0056     zstrm->buffer = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1);
0057     if (IS_ERR_OR_NULL(zstrm->tfm) || !zstrm->buffer) {
0058         zcomp_strm_free(zstrm);
0059         return -ENOMEM;
0060     }
0061     return 0;
0062 }
0063
0064 bool zcomp_available_algorithm(const char *comp)
0065 {
0066     /*
0067      * Crypto does not ignore a trailing new line symbol,
0068      * so make sure you don't supply a string containing
0069      * one.
0070      * This also means that we permit zcomp initialisation
0071      * with any compressing algorithm known to crypto api.
0072      */
0073     return crypto_has_comp(comp, 0, 0) == 1;
0074 }
0075
0076 /* show available compressors */
0077 ssize_t zcomp_available_show(const char *comp, char *buf)
0078 {
0079     bool known_algorithm = false;
0080     ssize_t sz = 0;
0081     int i;
0082
0083     for (i = 0; i < ARRAY_SIZE(backends); i++) {
0084         if (!strcmp(comp, backends[i])) {
0085             known_algorithm = true;
0086             sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2,
0087                     "[%s] ", backends[i]);
0088         } else {
0089             sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2,
0090                     "%s ", backends[i]);
0091         }
0092     }
0093
0094     /*
0095      * Out-of-tree module known to crypto api or a missing
0096      * entry in `backends'.
0097      */
0098     if (!known_algorithm && crypto_has_comp(comp, 0, 0) == 1)
0099         sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2,
0100                 "[%s] ", comp);
0101
0102     sz += scnprintf(buf + sz, PAGE_SIZE - sz, "\n");
0103     return sz;
0104 }
0105
0106 struct zcomp_strm *zcomp_stream_get(struct zcomp *comp)
0107 {
0108     local_lock(&comp->stream->lock);
0109     return this_cpu_ptr(comp->stream);
0110 }
0111
0112 void zcomp_stream_put(struct zcomp *comp)
0113 {
0114     local_unlock(&comp->stream->lock);
0115 }
0116
0117 int zcomp_compress(struct zcomp_strm *zstrm,
0118         const void *src, unsigned int *dst_len)
0119 {
0120     /*
0121      * Our dst memory (zstrm->buffer) is always `2 * PAGE_SIZE' sized
0122      * because sometimes we can endup having a bigger compressed data
0123      * due to various reasons: for example compression algorithms tend
0124      * to add some padding to the compressed buffer. Speaking of padding,
0125      * comp algorithm `842' pads the compressed length to multiple of 8
0126      * and returns -ENOSP when the dst memory is not big enough, which
0127      * is not something that ZRAM wants to see. We can handle the
0128      * `compressed_size > PAGE_SIZE' case easily in ZRAM, but when we
0129      * receive -ERRNO from the compressing backend we can't help it
0130      * anymore. To make `842' happy we need to tell the exact size of
0131      * the dst buffer, zram_drv will take care of the fact that
0132      * compressed buffer is too big.
0133      */
0134     *dst_len = PAGE_SIZE * 2;
0135
0136     return crypto_comp_compress(zstrm->tfm,
0137             src, PAGE_SIZE,
0138             zstrm->buffer, dst_len);
0139 }
0140
0141 int zcomp_decompress(struct zcomp_strm *zstrm,
0142         const void *src, unsigned int src_len, void *dst)
0143 {
0144     unsigned int dst_len = PAGE_SIZE;
0145
0146     return crypto_comp_decompress(zstrm->tfm,
0147             src, src_len,
0148             dst, &dst_len);
0149 }
0150
0151 int zcomp_cpu_up_prepare(unsigned int cpu, struct hlist_node *node)
0152 {
0153     struct zcomp *comp = hlist_entry(node, struct zcomp, node);
0154     struct zcomp_strm *zstrm;
0155     int ret;
0156
0157     zstrm = per_cpu_ptr(comp->stream, cpu);
0158     local_lock_init(&zstrm->lock);
0159
0160     ret = zcomp_strm_init(zstrm, comp);
0161     if (ret)
0162         pr_err("Can't allocate a compression stream\n");
0163     return ret;
0164 }
0165
0166 int zcomp_cpu_dead(unsigned int cpu, struct hlist_node *node)
0167 {
0168     struct zcomp *comp = hlist_entry(node, struct zcomp, node);
0169     struct zcomp_strm *zstrm;
0170
0171     zstrm = per_cpu_ptr(comp->stream, cpu);
0172     zcomp_strm_free(zstrm);
0173     return 0;
0174 }
0175
0176 static int zcomp_init(struct zcomp *comp)
0177 {
0178     int ret;
0179
0180     comp->stream = alloc_percpu(struct zcomp_strm);
0181     if (!comp->stream)
0182         return -ENOMEM;
0183
0184     ret = cpuhp_state_add_instance(CPUHP_ZCOMP_PREPARE, &comp->node);
0185     if (ret < 0)
0186         goto cleanup;
0187     return 0;
0188
0189 cleanup:
0190     free_percpu(comp->stream);
0191     return ret;
0192 }
0193
0194 void zcomp_destroy(struct zcomp *comp)
0195 {
0196     cpuhp_state_remove_instance(CPUHP_ZCOMP_PREPARE, &comp->node);
0197     free_percpu(comp->stream);
0198     kfree(comp);
0199 }
0200
0201 /*
0202  * search available compressors for requested algorithm.
0203  * allocate new zcomp and initialize it. return compressing
0204  * backend pointer or ERR_PTR if things went bad. ERR_PTR(-EINVAL)
0205  * if requested algorithm is not supported, ERR_PTR(-ENOMEM) in
0206  * case of allocation error, or any other error potentially
0207  * returned by zcomp_init().
0208  */
0209 struct zcomp *zcomp_create(const char *compress)
0210 {
0211     struct zcomp *comp;
0212     int error;
0213
0214     /*
0215      * Crypto API will execute /sbin/modprobe if the compression module
0216      * is not loaded yet. We must do it here, otherwise we are about to
0217      * call /sbin/modprobe under CPU hot-plug lock.
0218      */
0219     if (!zcomp_available_algorithm(compress))
0220         return ERR_PTR(-EINVAL);
0221
0222     comp = kzalloc(sizeof(struct zcomp), GFP_KERNEL);
0223     if (!comp)
0224         return ERR_PTR(-ENOMEM);
0225
0226     comp->name = compress;
0227     error = zcomp_init(comp);
0228     if (error) {
0229         kfree(comp);
0230         return ERR_PTR(error);
0231     }
0232     return comp;
0233 }