include/linux/bvec.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /*
0003  * bvec iterator
0004  *
0005  * Copyright (C) 2001 Ming Lei <ming.lei@canonical.com>
0006  */
0007 #ifndef __LINUX_BVEC_H
0008 #define __LINUX_BVEC_H
0009
0010 #include <linux/highmem.h>
0011 #include <linux/bug.h>
0012 #include <linux/errno.h>
0013 #include <linux/limits.h>
0014 #include <linux/minmax.h>
0015 #include <linux/mm.h>
0016 #include <linux/types.h>
0017
0018 struct page;
0019
0020 /**
0021  * struct bio_vec - a contiguous range of physical memory addresses
0022  * @bv_page:   First page associated with the address range.
0023  * @bv_len:    Number of bytes in the address range.
0024  * @bv_offset: Start of the address range relative to the start of @bv_page.
0025  *
0026  * The following holds for a bvec if n * PAGE_SIZE < bv_offset + bv_len:
0027  *
0028  *   nth_page(@bv_page, n) == @bv_page + n
0029  *
0030  * This holds because page_is_mergeable() checks the above property.
0031  */
0032 struct bio_vec {
0033     struct page *bv_page;
0034     unsigned int    bv_len;
0035     unsigned int    bv_offset;
0036 };
0037
0038 struct bvec_iter {
0039     sector_t        bi_sector;  /* device address in 512 byte
0040                            sectors */
0041     unsigned int        bi_size;    /* residual I/O count */
0042
0043     unsigned int        bi_idx;     /* current index into bvl_vec */
0044
0045     unsigned int            bi_bvec_done;   /* number of bytes completed in
0046                            current bvec */
0047 } __packed;
0048
0049 struct bvec_iter_all {
0050     struct bio_vec  bv;
0051     int     idx;
0052     unsigned    done;
0053 };
0054
0055 /*
0056  * various member access, note that bio_data should of course not be used
0057  * on highmem page vectors
0058  */
0059 #define __bvec_iter_bvec(bvec, iter)    (&(bvec)[(iter).bi_idx])
0060
0061 /* multi-page (mp_bvec) helpers */
0062 #define mp_bvec_iter_page(bvec, iter)               \
0063     (__bvec_iter_bvec((bvec), (iter))->bv_page)
0064
0065 #define mp_bvec_iter_len(bvec, iter)                \
0066     min((iter).bi_size,                 \
0067         __bvec_iter_bvec((bvec), (iter))->bv_len - (iter).bi_bvec_done)
0068
0069 #define mp_bvec_iter_offset(bvec, iter)             \
0070     (__bvec_iter_bvec((bvec), (iter))->bv_offset + (iter).bi_bvec_done)
0071
0072 #define mp_bvec_iter_page_idx(bvec, iter)           \
0073     (mp_bvec_iter_offset((bvec), (iter)) / PAGE_SIZE)
0074
0075 #define mp_bvec_iter_bvec(bvec, iter)               \
0076 ((struct bio_vec) {                     \
0077     .bv_page    = mp_bvec_iter_page((bvec), (iter)),    \
0078     .bv_len     = mp_bvec_iter_len((bvec), (iter)), \
0079     .bv_offset  = mp_bvec_iter_offset((bvec), (iter)),  \
0080 })
0081
0082 /* For building single-page bvec in flight */
0083  #define bvec_iter_offset(bvec, iter)               \
0084     (mp_bvec_iter_offset((bvec), (iter)) % PAGE_SIZE)
0085
0086 #define bvec_iter_len(bvec, iter)               \
0087     min_t(unsigned, mp_bvec_iter_len((bvec), (iter)),       \
0088           PAGE_SIZE - bvec_iter_offset((bvec), (iter)))
0089
0090 #define bvec_iter_page(bvec, iter)              \
0091     (mp_bvec_iter_page((bvec), (iter)) +            \
0092      mp_bvec_iter_page_idx((bvec), (iter)))
0093
0094 #define bvec_iter_bvec(bvec, iter)              \
0095 ((struct bio_vec) {                     \
0096     .bv_page    = bvec_iter_page((bvec), (iter)),   \
0097     .bv_len     = bvec_iter_len((bvec), (iter)),    \
0098     .bv_offset  = bvec_iter_offset((bvec), (iter)), \
0099 })
0100
0101 static inline bool bvec_iter_advance(const struct bio_vec *bv,
0102         struct bvec_iter *iter, unsigned bytes)
0103 {
0104     unsigned int idx = iter->bi_idx;
0105
0106     if (WARN_ONCE(bytes > iter->bi_size,
0107              "Attempted to advance past end of bvec iter\n")) {
0108         iter->bi_size = 0;
0109         return false;
0110     }
0111
0112     iter->bi_size -= bytes;
0113     bytes += iter->bi_bvec_done;
0114
0115     while (bytes && bytes >= bv[idx].bv_len) {
0116         bytes -= bv[idx].bv_len;
0117         idx++;
0118     }
0119
0120     iter->bi_idx = idx;
0121     iter->bi_bvec_done = bytes;
0122     return true;
0123 }
0124
0125 /*
0126  * A simpler version of bvec_iter_advance(), @bytes should not span
0127  * across multiple bvec entries, i.e. bytes <= bv[i->bi_idx].bv_len
0128  */
0129 static inline void bvec_iter_advance_single(const struct bio_vec *bv,
0130                 struct bvec_iter *iter, unsigned int bytes)
0131 {
0132     unsigned int done = iter->bi_bvec_done + bytes;
0133
0134     if (done == bv[iter->bi_idx].bv_len) {
0135         done = 0;
0136         iter->bi_idx++;
0137     }
0138     iter->bi_bvec_done = done;
0139     iter->bi_size -= bytes;
0140 }
0141
0142 #define for_each_bvec(bvl, bio_vec, iter, start)            \
0143     for (iter = (start);                        \
0144          (iter).bi_size &&                      \
0145         ((bvl = bvec_iter_bvec((bio_vec), (iter))), 1); \
0146          bvec_iter_advance_single((bio_vec), &(iter), (bvl).bv_len))
0147
0148 /* for iterating one bio from start to end */
0149 #define BVEC_ITER_ALL_INIT (struct bvec_iter)               \
0150 {                                   \
0151     .bi_sector  = 0,                        \
0152     .bi_size    = UINT_MAX,                 \
0153     .bi_idx     = 0,                        \
0154     .bi_bvec_done   = 0,                        \
0155 }
0156
0157 static inline struct bio_vec *bvec_init_iter_all(struct bvec_iter_all *iter_all)
0158 {
0159     iter_all->done = 0;
0160     iter_all->idx = 0;
0161
0162     return &iter_all->bv;
0163 }
0164
0165 static inline void bvec_advance(const struct bio_vec *bvec,
0166                 struct bvec_iter_all *iter_all)
0167 {
0168     struct bio_vec *bv = &iter_all->bv;
0169
0170     if (iter_all->done) {
0171         bv->bv_page++;
0172         bv->bv_offset = 0;
0173     } else {
0174         bv->bv_page = bvec->bv_page + (bvec->bv_offset >> PAGE_SHIFT);
0175         bv->bv_offset = bvec->bv_offset & ~PAGE_MASK;
0176     }
0177     bv->bv_len = min_t(unsigned int, PAGE_SIZE - bv->bv_offset,
0178                bvec->bv_len - iter_all->done);
0179     iter_all->done += bv->bv_len;
0180
0181     if (iter_all->done == bvec->bv_len) {
0182         iter_all->idx++;
0183         iter_all->done = 0;
0184     }
0185 }
0186
0187 /**
0188  * bvec_kmap_local - map a bvec into the kernel virtual address space
0189  * @bvec: bvec to map
0190  *
0191  * Must be called on single-page bvecs only.  Call kunmap_local on the returned
0192  * address to unmap.
0193  */
0194 static inline void *bvec_kmap_local(struct bio_vec *bvec)
0195 {
0196     return kmap_local_page(bvec->bv_page) + bvec->bv_offset;
0197 }
0198
0199 /**
0200  * memcpy_from_bvec - copy data from a bvec
0201  * @bvec: bvec to copy from
0202  *
0203  * Must be called on single-page bvecs only.
0204  */
0205 static inline void memcpy_from_bvec(char *to, struct bio_vec *bvec)
0206 {
0207     memcpy_from_page(to, bvec->bv_page, bvec->bv_offset, bvec->bv_len);
0208 }
0209
0210 /**
0211  * memcpy_to_bvec - copy data to a bvec
0212  * @bvec: bvec to copy to
0213  *
0214  * Must be called on single-page bvecs only.
0215  */
0216 static inline void memcpy_to_bvec(struct bio_vec *bvec, const char *from)
0217 {
0218     memcpy_to_page(bvec->bv_page, bvec->bv_offset, from, bvec->bv_len);
0219 }
0220
0221 /**
0222  * memzero_bvec - zero all data in a bvec
0223  * @bvec: bvec to zero
0224  *
0225  * Must be called on single-page bvecs only.
0226  */
0227 static inline void memzero_bvec(struct bio_vec *bvec)
0228 {
0229     memzero_page(bvec->bv_page, bvec->bv_offset, bvec->bv_len);
0230 }
0231
0232 /**
0233  * bvec_virt - return the virtual address for a bvec
0234  * @bvec: bvec to return the virtual address for
0235  *
0236  * Note: the caller must ensure that @bvec->bv_page is not a highmem page.
0237  */
0238 static inline void *bvec_virt(struct bio_vec *bvec)
0239 {
0240     WARN_ON_ONCE(PageHighMem(bvec->bv_page));
0241     return page_address(bvec->bv_page) + bvec->bv_offset;
0242 }
0243
0244 #endif /* __LINUX_BVEC_H */