fs/btrfs/zlib.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Copyright (C) 2008 Oracle.  All rights reserved.
0004  *
0005  * Based on jffs2 zlib code:
0006  * Copyright © 2001-2007 Red Hat, Inc.
0007  * Created by David Woodhouse <dwmw2@infradead.org>
0008  */
0009
0010 #include <linux/kernel.h>
0011 #include <linux/slab.h>
0012 #include <linux/zlib.h>
0013 #include <linux/zutil.h>
0014 #include <linux/mm.h>
0015 #include <linux/init.h>
0016 #include <linux/err.h>
0017 #include <linux/sched.h>
0018 #include <linux/pagemap.h>
0019 #include <linux/bio.h>
0020 #include <linux/refcount.h>
0021 #include "compression.h"
0022
0023 /* workspace buffer size for s390 zlib hardware support */
0024 #define ZLIB_DFLTCC_BUF_SIZE    (4 * PAGE_SIZE)
0025
0026 struct workspace {
0027     z_stream strm;
0028     char *buf;
0029     unsigned int buf_size;
0030     struct list_head list;
0031     int level;
0032 };
0033
0034 static struct workspace_manager wsm;
0035
0036 struct list_head *zlib_get_workspace(unsigned int level)
0037 {
0038     struct list_head *ws = btrfs_get_workspace(BTRFS_COMPRESS_ZLIB, level);
0039     struct workspace *workspace = list_entry(ws, struct workspace, list);
0040
0041     workspace->level = level;
0042
0043     return ws;
0044 }
0045
0046 void zlib_free_workspace(struct list_head *ws)
0047 {
0048     struct workspace *workspace = list_entry(ws, struct workspace, list);
0049
0050     kvfree(workspace->strm.workspace);
0051     kfree(workspace->buf);
0052     kfree(workspace);
0053 }
0054
0055 struct list_head *zlib_alloc_workspace(unsigned int level)
0056 {
0057     struct workspace *workspace;
0058     int workspacesize;
0059
0060     workspace = kzalloc(sizeof(*workspace), GFP_KERNEL);
0061     if (!workspace)
0062         return ERR_PTR(-ENOMEM);
0063
0064     workspacesize = max(zlib_deflate_workspacesize(MAX_WBITS, MAX_MEM_LEVEL),
0065             zlib_inflate_workspacesize());
0066     workspace->strm.workspace = kvmalloc(workspacesize, GFP_KERNEL);
0067     workspace->level = level;
0068     workspace->buf = NULL;
0069     /*
0070      * In case of s390 zlib hardware support, allocate lager workspace
0071      * buffer. If allocator fails, fall back to a single page buffer.
0072      */
0073     if (zlib_deflate_dfltcc_enabled()) {
0074         workspace->buf = kmalloc(ZLIB_DFLTCC_BUF_SIZE,
0075                      __GFP_NOMEMALLOC | __GFP_NORETRY |
0076                      __GFP_NOWARN | GFP_NOIO);
0077         workspace->buf_size = ZLIB_DFLTCC_BUF_SIZE;
0078     }
0079     if (!workspace->buf) {
0080         workspace->buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
0081         workspace->buf_size = PAGE_SIZE;
0082     }
0083     if (!workspace->strm.workspace || !workspace->buf)
0084         goto fail;
0085
0086     INIT_LIST_HEAD(&workspace->list);
0087
0088     return &workspace->list;
0089 fail:
0090     zlib_free_workspace(&workspace->list);
0091     return ERR_PTR(-ENOMEM);
0092 }
0093
0094 int zlib_compress_pages(struct list_head *ws, struct address_space *mapping,
0095         u64 start, struct page **pages, unsigned long *out_pages,
0096         unsigned long *total_in, unsigned long *total_out)
0097 {
0098     struct workspace *workspace = list_entry(ws, struct workspace, list);
0099     int ret;
0100     char *data_in = NULL;
0101     char *cpage_out;
0102     int nr_pages = 0;
0103     struct page *in_page = NULL;
0104     struct page *out_page = NULL;
0105     unsigned long bytes_left;
0106     unsigned int in_buf_pages;
0107     unsigned long len = *total_out;
0108     unsigned long nr_dest_pages = *out_pages;
0109     const unsigned long max_out = nr_dest_pages * PAGE_SIZE;
0110
0111     *out_pages = 0;
0112     *total_out = 0;
0113     *total_in = 0;
0114
0115     if (Z_OK != zlib_deflateInit(&workspace->strm, workspace->level)) {
0116         pr_warn("BTRFS: deflateInit failed\n");
0117         ret = -EIO;
0118         goto out;
0119     }
0120
0121     workspace->strm.total_in = 0;
0122     workspace->strm.total_out = 0;
0123
0124     out_page = alloc_page(GFP_NOFS);
0125     if (out_page == NULL) {
0126         ret = -ENOMEM;
0127         goto out;
0128     }
0129     cpage_out = page_address(out_page);
0130     pages[0] = out_page;
0131     nr_pages = 1;
0132
0133     workspace->strm.next_in = workspace->buf;
0134     workspace->strm.avail_in = 0;
0135     workspace->strm.next_out = cpage_out;
0136     workspace->strm.avail_out = PAGE_SIZE;
0137
0138     while (workspace->strm.total_in < len) {
0139         /*
0140          * Get next input pages and copy the contents to
0141          * the workspace buffer if required.
0142          */
0143         if (workspace->strm.avail_in == 0) {
0144             bytes_left = len - workspace->strm.total_in;
0145             in_buf_pages = min(DIV_ROUND_UP(bytes_left, PAGE_SIZE),
0146                        workspace->buf_size / PAGE_SIZE);
0147             if (in_buf_pages > 1) {
0148                 int i;
0149
0150                 for (i = 0; i < in_buf_pages; i++) {
0151                     if (data_in) {
0152                         kunmap_local(data_in);
0153                         put_page(in_page);
0154                     }
0155                     in_page = find_get_page(mapping,
0156                                 start >> PAGE_SHIFT);
0157                     data_in = kmap_local_page(in_page);
0158                     memcpy(workspace->buf + i * PAGE_SIZE,
0159                            data_in, PAGE_SIZE);
0160                     start += PAGE_SIZE;
0161                 }
0162                 workspace->strm.next_in = workspace->buf;
0163             } else {
0164                 if (data_in) {
0165                     kunmap_local(data_in);
0166                     put_page(in_page);
0167                 }
0168                 in_page = find_get_page(mapping,
0169                             start >> PAGE_SHIFT);
0170                 data_in = kmap_local_page(in_page);
0171                 start += PAGE_SIZE;
0172                 workspace->strm.next_in = data_in;
0173             }
0174             workspace->strm.avail_in = min(bytes_left,
0175                                (unsigned long) workspace->buf_size);
0176         }
0177
0178         ret = zlib_deflate(&workspace->strm, Z_SYNC_FLUSH);
0179         if (ret != Z_OK) {
0180             pr_debug("BTRFS: deflate in loop returned %d\n",
0181                    ret);
0182             zlib_deflateEnd(&workspace->strm);
0183             ret = -EIO;
0184             goto out;
0185         }
0186
0187         /* we're making it bigger, give up */
0188         if (workspace->strm.total_in > 8192 &&
0189             workspace->strm.total_in <
0190             workspace->strm.total_out) {
0191             ret = -E2BIG;
0192             goto out;
0193         }
0194         /* we need another page for writing out.  Test this
0195          * before the total_in so we will pull in a new page for
0196          * the stream end if required
0197          */
0198         if (workspace->strm.avail_out == 0) {
0199             if (nr_pages == nr_dest_pages) {
0200                 ret = -E2BIG;
0201                 goto out;
0202             }
0203             out_page = alloc_page(GFP_NOFS);
0204             if (out_page == NULL) {
0205                 ret = -ENOMEM;
0206                 goto out;
0207             }
0208             cpage_out = page_address(out_page);
0209             pages[nr_pages] = out_page;
0210             nr_pages++;
0211             workspace->strm.avail_out = PAGE_SIZE;
0212             workspace->strm.next_out = cpage_out;
0213         }
0214         /* we're all done */
0215         if (workspace->strm.total_in >= len)
0216             break;
0217         if (workspace->strm.total_out > max_out)
0218             break;
0219     }
0220     workspace->strm.avail_in = 0;
0221     /*
0222      * Call deflate with Z_FINISH flush parameter providing more output
0223      * space but no more input data, until it returns with Z_STREAM_END.
0224      */
0225     while (ret != Z_STREAM_END) {
0226         ret = zlib_deflate(&workspace->strm, Z_FINISH);
0227         if (ret == Z_STREAM_END)
0228             break;
0229         if (ret != Z_OK && ret != Z_BUF_ERROR) {
0230             zlib_deflateEnd(&workspace->strm);
0231             ret = -EIO;
0232             goto out;
0233         } else if (workspace->strm.avail_out == 0) {
0234             /* get another page for the stream end */
0235             if (nr_pages == nr_dest_pages) {
0236                 ret = -E2BIG;
0237                 goto out;
0238             }
0239             out_page = alloc_page(GFP_NOFS);
0240             if (out_page == NULL) {
0241                 ret = -ENOMEM;
0242                 goto out;
0243             }
0244             cpage_out = page_address(out_page);
0245             pages[nr_pages] = out_page;
0246             nr_pages++;
0247             workspace->strm.avail_out = PAGE_SIZE;
0248             workspace->strm.next_out = cpage_out;
0249         }
0250     }
0251     zlib_deflateEnd(&workspace->strm);
0252
0253     if (workspace->strm.total_out >= workspace->strm.total_in) {
0254         ret = -E2BIG;
0255         goto out;
0256     }
0257
0258     ret = 0;
0259     *total_out = workspace->strm.total_out;
0260     *total_in = workspace->strm.total_in;
0261 out:
0262     *out_pages = nr_pages;
0263     if (data_in) {
0264         kunmap_local(data_in);
0265         put_page(in_page);
0266     }
0267
0268     return ret;
0269 }
0270
0271 int zlib_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
0272 {
0273     struct workspace *workspace = list_entry(ws, struct workspace, list);
0274     int ret = 0, ret2;
0275     int wbits = MAX_WBITS;
0276     char *data_in;
0277     size_t total_out = 0;
0278     unsigned long page_in_index = 0;
0279     size_t srclen = cb->compressed_len;
0280     unsigned long total_pages_in = DIV_ROUND_UP(srclen, PAGE_SIZE);
0281     unsigned long buf_start;
0282     struct page **pages_in = cb->compressed_pages;
0283
0284     data_in = kmap_local_page(pages_in[page_in_index]);
0285     workspace->strm.next_in = data_in;
0286     workspace->strm.avail_in = min_t(size_t, srclen, PAGE_SIZE);
0287     workspace->strm.total_in = 0;
0288
0289     workspace->strm.total_out = 0;
0290     workspace->strm.next_out = workspace->buf;
0291     workspace->strm.avail_out = workspace->buf_size;
0292
0293     /* If it's deflate, and it's got no preset dictionary, then
0294        we can tell zlib to skip the adler32 check. */
0295     if (srclen > 2 && !(data_in[1] & PRESET_DICT) &&
0296         ((data_in[0] & 0x0f) == Z_DEFLATED) &&
0297         !(((data_in[0]<<8) + data_in[1]) % 31)) {
0298
0299         wbits = -((data_in[0] >> 4) + 8);
0300         workspace->strm.next_in += 2;
0301         workspace->strm.avail_in -= 2;
0302     }
0303
0304     if (Z_OK != zlib_inflateInit2(&workspace->strm, wbits)) {
0305         pr_warn("BTRFS: inflateInit failed\n");
0306         kunmap_local(data_in);
0307         return -EIO;
0308     }
0309     while (workspace->strm.total_in < srclen) {
0310         ret = zlib_inflate(&workspace->strm, Z_NO_FLUSH);
0311         if (ret != Z_OK && ret != Z_STREAM_END)
0312             break;
0313
0314         buf_start = total_out;
0315         total_out = workspace->strm.total_out;
0316
0317         /* we didn't make progress in this inflate call, we're done */
0318         if (buf_start == total_out)
0319             break;
0320
0321         ret2 = btrfs_decompress_buf2page(workspace->buf,
0322                 total_out - buf_start, cb, buf_start);
0323         if (ret2 == 0) {
0324             ret = 0;
0325             goto done;
0326         }
0327
0328         workspace->strm.next_out = workspace->buf;
0329         workspace->strm.avail_out = workspace->buf_size;
0330
0331         if (workspace->strm.avail_in == 0) {
0332             unsigned long tmp;
0333             kunmap_local(data_in);
0334             page_in_index++;
0335             if (page_in_index >= total_pages_in) {
0336                 data_in = NULL;
0337                 break;
0338             }
0339             data_in = kmap_local_page(pages_in[page_in_index]);
0340             workspace->strm.next_in = data_in;
0341             tmp = srclen - workspace->strm.total_in;
0342             workspace->strm.avail_in = min(tmp, PAGE_SIZE);
0343         }
0344     }
0345     if (ret != Z_STREAM_END)
0346         ret = -EIO;
0347     else
0348         ret = 0;
0349 done:
0350     zlib_inflateEnd(&workspace->strm);
0351     if (data_in)
0352         kunmap_local(data_in);
0353     if (!ret)
0354         zero_fill_bio(cb->orig_bio);
0355     return ret;
0356 }
0357
0358 int zlib_decompress(struct list_head *ws, unsigned char *data_in,
0359         struct page *dest_page, unsigned long start_byte, size_t srclen,
0360         size_t destlen)
0361 {
0362     struct workspace *workspace = list_entry(ws, struct workspace, list);
0363     int ret = 0;
0364     int wbits = MAX_WBITS;
0365     unsigned long bytes_left;
0366     unsigned long total_out = 0;
0367     unsigned long pg_offset = 0;
0368
0369     destlen = min_t(unsigned long, destlen, PAGE_SIZE);
0370     bytes_left = destlen;
0371
0372     workspace->strm.next_in = data_in;
0373     workspace->strm.avail_in = srclen;
0374     workspace->strm.total_in = 0;
0375
0376     workspace->strm.next_out = workspace->buf;
0377     workspace->strm.avail_out = workspace->buf_size;
0378     workspace->strm.total_out = 0;
0379     /* If it's deflate, and it's got no preset dictionary, then
0380        we can tell zlib to skip the adler32 check. */
0381     if (srclen > 2 && !(data_in[1] & PRESET_DICT) &&
0382         ((data_in[0] & 0x0f) == Z_DEFLATED) &&
0383         !(((data_in[0]<<8) + data_in[1]) % 31)) {
0384
0385         wbits = -((data_in[0] >> 4) + 8);
0386         workspace->strm.next_in += 2;
0387         workspace->strm.avail_in -= 2;
0388     }
0389
0390     if (Z_OK != zlib_inflateInit2(&workspace->strm, wbits)) {
0391         pr_warn("BTRFS: inflateInit failed\n");
0392         return -EIO;
0393     }
0394
0395     while (bytes_left > 0) {
0396         unsigned long buf_start;
0397         unsigned long buf_offset;
0398         unsigned long bytes;
0399
0400         ret = zlib_inflate(&workspace->strm, Z_NO_FLUSH);
0401         if (ret != Z_OK && ret != Z_STREAM_END)
0402             break;
0403
0404         buf_start = total_out;
0405         total_out = workspace->strm.total_out;
0406
0407         if (total_out == buf_start) {
0408             ret = -EIO;
0409             break;
0410         }
0411
0412         if (total_out <= start_byte)
0413             goto next;
0414
0415         if (total_out > start_byte && buf_start < start_byte)
0416             buf_offset = start_byte - buf_start;
0417         else
0418             buf_offset = 0;
0419
0420         bytes = min(PAGE_SIZE - pg_offset,
0421                 PAGE_SIZE - (buf_offset % PAGE_SIZE));
0422         bytes = min(bytes, bytes_left);
0423
0424         memcpy_to_page(dest_page, pg_offset,
0425                    workspace->buf + buf_offset, bytes);
0426
0427         pg_offset += bytes;
0428         bytes_left -= bytes;
0429 next:
0430         workspace->strm.next_out = workspace->buf;
0431         workspace->strm.avail_out = workspace->buf_size;
0432     }
0433
0434     if (ret != Z_STREAM_END && bytes_left != 0)
0435         ret = -EIO;
0436     else
0437         ret = 0;
0438
0439     zlib_inflateEnd(&workspace->strm);
0440
0441     /*
0442      * this should only happen if zlib returned fewer bytes than we
0443      * expected.  btrfs_get_block is responsible for zeroing from the
0444      * end of the inline extent (destlen) to the end of the page
0445      */
0446     if (pg_offset < destlen) {
0447         memzero_page(dest_page, pg_offset, destlen - pg_offset);
0448     }
0449     return ret;
0450 }
0451
0452 const struct btrfs_compress_op btrfs_zlib_compress = {
0453     .workspace_manager  = &wsm,
0454     .max_level      = 9,
0455     .default_level      = BTRFS_ZLIB_DEFAULT_LEVEL,
0456 };