fs/erofs/pcpubuf.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright (C) Gao Xiang <xiang@kernel.org>
0004  *
0005  * For low-latency decompression algorithms (e.g. lz4), reserve consecutive
0006  * per-CPU virtual memory (in pages) in advance to store such inplace I/O
0007  * data if inplace decompression is failed (due to unmet inplace margin for
0008  * example).
0009  */
0010 #include "internal.h"
0011
0012 struct erofs_pcpubuf {
0013     raw_spinlock_t lock;
0014     void *ptr;
0015     struct page **pages;
0016     unsigned int nrpages;
0017 };
0018
0019 static DEFINE_PER_CPU(struct erofs_pcpubuf, erofs_pcb);
0020
0021 void *erofs_get_pcpubuf(unsigned int requiredpages)
0022     __acquires(pcb->lock)
0023 {
0024     struct erofs_pcpubuf *pcb = &get_cpu_var(erofs_pcb);
0025
0026     raw_spin_lock(&pcb->lock);
0027     /* check if the per-CPU buffer is too small */
0028     if (requiredpages > pcb->nrpages) {
0029         raw_spin_unlock(&pcb->lock);
0030         put_cpu_var(erofs_pcb);
0031         /* (for sparse checker) pretend pcb->lock is still taken */
0032         __acquire(pcb->lock);
0033         return NULL;
0034     }
0035     return pcb->ptr;
0036 }
0037
0038 void erofs_put_pcpubuf(void *ptr) __releases(pcb->lock)
0039 {
0040     struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, smp_processor_id());
0041
0042     DBG_BUGON(pcb->ptr != ptr);
0043     raw_spin_unlock(&pcb->lock);
0044     put_cpu_var(erofs_pcb);
0045 }
0046
0047 /* the next step: support per-CPU page buffers hotplug */
0048 int erofs_pcpubuf_growsize(unsigned int nrpages)
0049 {
0050     static DEFINE_MUTEX(pcb_resize_mutex);
0051     static unsigned int pcb_nrpages;
0052     struct page *pagepool = NULL;
0053     int delta, cpu, ret, i;
0054
0055     mutex_lock(&pcb_resize_mutex);
0056     delta = nrpages - pcb_nrpages;
0057     ret = 0;
0058     /* avoid shrinking pcpubuf, since no idea how many fses rely on */
0059     if (delta <= 0)
0060         goto out;
0061
0062     for_each_possible_cpu(cpu) {
0063         struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
0064         struct page **pages, **oldpages;
0065         void *ptr, *old_ptr;
0066
0067         pages = kmalloc_array(nrpages, sizeof(*pages), GFP_KERNEL);
0068         if (!pages) {
0069             ret = -ENOMEM;
0070             break;
0071         }
0072
0073         for (i = 0; i < nrpages; ++i) {
0074             pages[i] = erofs_allocpage(&pagepool, GFP_KERNEL);
0075             if (!pages[i]) {
0076                 ret = -ENOMEM;
0077                 oldpages = pages;
0078                 goto free_pagearray;
0079             }
0080         }
0081         ptr = vmap(pages, nrpages, VM_MAP, PAGE_KERNEL);
0082         if (!ptr) {
0083             ret = -ENOMEM;
0084             oldpages = pages;
0085             goto free_pagearray;
0086         }
0087         raw_spin_lock(&pcb->lock);
0088         old_ptr = pcb->ptr;
0089         pcb->ptr = ptr;
0090         oldpages = pcb->pages;
0091         pcb->pages = pages;
0092         i = pcb->nrpages;
0093         pcb->nrpages = nrpages;
0094         raw_spin_unlock(&pcb->lock);
0095
0096         if (!oldpages) {
0097             DBG_BUGON(old_ptr);
0098             continue;
0099         }
0100
0101         if (old_ptr)
0102             vunmap(old_ptr);
0103 free_pagearray:
0104         while (i)
0105             erofs_pagepool_add(&pagepool, oldpages[--i]);
0106         kfree(oldpages);
0107         if (ret)
0108             break;
0109     }
0110     pcb_nrpages = nrpages;
0111     erofs_release_pages(&pagepool);
0112 out:
0113     mutex_unlock(&pcb_resize_mutex);
0114     return ret;
0115 }
0116
0117 void erofs_pcpubuf_init(void)
0118 {
0119     int cpu;
0120
0121     for_each_possible_cpu(cpu) {
0122         struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
0123
0124         raw_spin_lock_init(&pcb->lock);
0125     }
0126 }
0127
0128 void erofs_pcpubuf_exit(void)
0129 {
0130     int cpu, i;
0131
0132     for_each_possible_cpu(cpu) {
0133         struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
0134
0135         if (pcb->ptr) {
0136             vunmap(pcb->ptr);
0137             pcb->ptr = NULL;
0138         }
0139         if (!pcb->pages)
0140             continue;
0141
0142         for (i = 0; i < pcb->nrpages; ++i)
0143             if (pcb->pages[i])
0144                 put_page(pcb->pages[i]);
0145         kfree(pcb->pages);
0146         pcb->pages = NULL;
0147     }
0148 }