Back to home page

LXR

 
 

    


0001 /*
0002  * bio-integrity.c - bio data integrity extensions
0003  *
0004  * Copyright (C) 2007, 2008, 2009 Oracle Corporation
0005  * Written by: Martin K. Petersen <martin.petersen@oracle.com>
0006  *
0007  * This program is free software; you can redistribute it and/or
0008  * modify it under the terms of the GNU General Public License version
0009  * 2 as published by the Free Software Foundation.
0010  *
0011  * This program is distributed in the hope that it will be useful, but
0012  * WITHOUT ANY WARRANTY; without even the implied warranty of
0013  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
0014  * General Public License for more details.
0015  *
0016  * You should have received a copy of the GNU General Public License
0017  * along with this program; see the file COPYING.  If not, write to
0018  * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139,
0019  * USA.
0020  *
0021  */
0022 
0023 #include <linux/blkdev.h>
0024 #include <linux/mempool.h>
0025 #include <linux/export.h>
0026 #include <linux/bio.h>
0027 #include <linux/workqueue.h>
0028 #include <linux/slab.h>
0029 #include "blk.h"
0030 
0031 #define BIP_INLINE_VECS 4
0032 
0033 static struct kmem_cache *bip_slab;
0034 static struct workqueue_struct *kintegrityd_wq;
0035 
0036 void blk_flush_integrity(void)
0037 {
0038     flush_workqueue(kintegrityd_wq);
0039 }
0040 
0041 /**
0042  * bio_integrity_alloc - Allocate integrity payload and attach it to bio
0043  * @bio:    bio to attach integrity metadata to
0044  * @gfp_mask:   Memory allocation mask
0045  * @nr_vecs:    Number of integrity metadata scatter-gather elements
0046  *
0047  * Description: This function prepares a bio for attaching integrity
0048  * metadata.  nr_vecs specifies the maximum number of pages containing
0049  * integrity metadata that can be attached.
0050  */
0051 struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio,
0052                           gfp_t gfp_mask,
0053                           unsigned int nr_vecs)
0054 {
0055     struct bio_integrity_payload *bip;
0056     struct bio_set *bs = bio->bi_pool;
0057     unsigned inline_vecs;
0058 
0059     if (!bs || !bs->bio_integrity_pool) {
0060         bip = kmalloc(sizeof(struct bio_integrity_payload) +
0061                   sizeof(struct bio_vec) * nr_vecs, gfp_mask);
0062         inline_vecs = nr_vecs;
0063     } else {
0064         bip = mempool_alloc(bs->bio_integrity_pool, gfp_mask);
0065         inline_vecs = BIP_INLINE_VECS;
0066     }
0067 
0068     if (unlikely(!bip))
0069         return ERR_PTR(-ENOMEM);
0070 
0071     memset(bip, 0, sizeof(*bip));
0072 
0073     if (nr_vecs > inline_vecs) {
0074         unsigned long idx = 0;
0075 
0076         bip->bip_vec = bvec_alloc(gfp_mask, nr_vecs, &idx,
0077                       bs->bvec_integrity_pool);
0078         if (!bip->bip_vec)
0079             goto err;
0080         bip->bip_max_vcnt = bvec_nr_vecs(idx);
0081         bip->bip_slab = idx;
0082     } else {
0083         bip->bip_vec = bip->bip_inline_vecs;
0084         bip->bip_max_vcnt = inline_vecs;
0085     }
0086 
0087     bip->bip_bio = bio;
0088     bio->bi_integrity = bip;
0089     bio->bi_opf |= REQ_INTEGRITY;
0090 
0091     return bip;
0092 err:
0093     mempool_free(bip, bs->bio_integrity_pool);
0094     return ERR_PTR(-ENOMEM);
0095 }
0096 EXPORT_SYMBOL(bio_integrity_alloc);
0097 
0098 /**
0099  * bio_integrity_free - Free bio integrity payload
0100  * @bio:    bio containing bip to be freed
0101  *
0102  * Description: Used to free the integrity portion of a bio. Usually
0103  * called from bio_free().
0104  */
0105 void bio_integrity_free(struct bio *bio)
0106 {
0107     struct bio_integrity_payload *bip = bio_integrity(bio);
0108     struct bio_set *bs = bio->bi_pool;
0109 
0110     if (bip->bip_flags & BIP_BLOCK_INTEGRITY)
0111         kfree(page_address(bip->bip_vec->bv_page) +
0112               bip->bip_vec->bv_offset);
0113 
0114     if (bs && bs->bio_integrity_pool) {
0115         bvec_free(bs->bvec_integrity_pool, bip->bip_vec, bip->bip_slab);
0116 
0117         mempool_free(bip, bs->bio_integrity_pool);
0118     } else {
0119         kfree(bip);
0120     }
0121 
0122     bio->bi_integrity = NULL;
0123 }
0124 EXPORT_SYMBOL(bio_integrity_free);
0125 
0126 /**
0127  * bio_integrity_add_page - Attach integrity metadata
0128  * @bio:    bio to update
0129  * @page:   page containing integrity metadata
0130  * @len:    number of bytes of integrity metadata in page
0131  * @offset: start offset within page
0132  *
0133  * Description: Attach a page containing integrity metadata to bio.
0134  */
0135 int bio_integrity_add_page(struct bio *bio, struct page *page,
0136                unsigned int len, unsigned int offset)
0137 {
0138     struct bio_integrity_payload *bip = bio_integrity(bio);
0139     struct bio_vec *iv;
0140 
0141     if (bip->bip_vcnt >= bip->bip_max_vcnt) {
0142         printk(KERN_ERR "%s: bip_vec full\n", __func__);
0143         return 0;
0144     }
0145 
0146     iv = bip->bip_vec + bip->bip_vcnt;
0147 
0148     if (bip->bip_vcnt &&
0149         bvec_gap_to_prev(bdev_get_queue(bio->bi_bdev),
0150                  &bip->bip_vec[bip->bip_vcnt - 1], offset))
0151         return 0;
0152 
0153     iv->bv_page = page;
0154     iv->bv_len = len;
0155     iv->bv_offset = offset;
0156     bip->bip_vcnt++;
0157 
0158     return len;
0159 }
0160 EXPORT_SYMBOL(bio_integrity_add_page);
0161 
0162 /**
0163  * bio_integrity_enabled - Check whether integrity can be passed
0164  * @bio:    bio to check
0165  *
0166  * Description: Determines whether bio_integrity_prep() can be called
0167  * on this bio or not.  bio data direction and target device must be
0168  * set prior to calling.  The functions honors the write_generate and
0169  * read_verify flags in sysfs.
0170  */
0171 bool bio_integrity_enabled(struct bio *bio)
0172 {
0173     struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
0174 
0175     if (bio_op(bio) != REQ_OP_READ && bio_op(bio) != REQ_OP_WRITE)
0176         return false;
0177 
0178     /* Already protected? */
0179     if (bio_integrity(bio))
0180         return false;
0181 
0182     if (bi == NULL)
0183         return false;
0184 
0185     if (bio_data_dir(bio) == READ && bi->profile->verify_fn != NULL &&
0186         (bi->flags & BLK_INTEGRITY_VERIFY))
0187         return true;
0188 
0189     if (bio_data_dir(bio) == WRITE && bi->profile->generate_fn != NULL &&
0190         (bi->flags & BLK_INTEGRITY_GENERATE))
0191         return true;
0192 
0193     return false;
0194 }
0195 EXPORT_SYMBOL(bio_integrity_enabled);
0196 
0197 /**
0198  * bio_integrity_intervals - Return number of integrity intervals for a bio
0199  * @bi:     blk_integrity profile for device
0200  * @sectors:    Size of the bio in 512-byte sectors
0201  *
0202  * Description: The block layer calculates everything in 512 byte
0203  * sectors but integrity metadata is done in terms of the data integrity
0204  * interval size of the storage device.  Convert the block layer sectors
0205  * to the appropriate number of integrity intervals.
0206  */
0207 static inline unsigned int bio_integrity_intervals(struct blk_integrity *bi,
0208                            unsigned int sectors)
0209 {
0210     return sectors >> (bi->interval_exp - 9);
0211 }
0212 
0213 static inline unsigned int bio_integrity_bytes(struct blk_integrity *bi,
0214                            unsigned int sectors)
0215 {
0216     return bio_integrity_intervals(bi, sectors) * bi->tuple_size;
0217 }
0218 
0219 /**
0220  * bio_integrity_process - Process integrity metadata for a bio
0221  * @bio:    bio to generate/verify integrity metadata for
0222  * @proc_fn:    Pointer to the relevant processing function
0223  */
0224 static int bio_integrity_process(struct bio *bio,
0225                  integrity_processing_fn *proc_fn)
0226 {
0227     struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
0228     struct blk_integrity_iter iter;
0229     struct bvec_iter bviter;
0230     struct bio_vec bv;
0231     struct bio_integrity_payload *bip = bio_integrity(bio);
0232     unsigned int ret = 0;
0233     void *prot_buf = page_address(bip->bip_vec->bv_page) +
0234         bip->bip_vec->bv_offset;
0235 
0236     iter.disk_name = bio->bi_bdev->bd_disk->disk_name;
0237     iter.interval = 1 << bi->interval_exp;
0238     iter.seed = bip_get_seed(bip);
0239     iter.prot_buf = prot_buf;
0240 
0241     bio_for_each_segment(bv, bio, bviter) {
0242         void *kaddr = kmap_atomic(bv.bv_page);
0243 
0244         iter.data_buf = kaddr + bv.bv_offset;
0245         iter.data_size = bv.bv_len;
0246 
0247         ret = proc_fn(&iter);
0248         if (ret) {
0249             kunmap_atomic(kaddr);
0250             return ret;
0251         }
0252 
0253         kunmap_atomic(kaddr);
0254     }
0255     return ret;
0256 }
0257 
0258 /**
0259  * bio_integrity_prep - Prepare bio for integrity I/O
0260  * @bio:    bio to prepare
0261  *
0262  * Description: Allocates a buffer for integrity metadata, maps the
0263  * pages and attaches them to a bio.  The bio must have data
0264  * direction, target device and start sector set priot to calling.  In
0265  * the WRITE case, integrity metadata will be generated using the
0266  * block device's integrity function.  In the READ case, the buffer
0267  * will be prepared for DMA and a suitable end_io handler set up.
0268  */
0269 int bio_integrity_prep(struct bio *bio)
0270 {
0271     struct bio_integrity_payload *bip;
0272     struct blk_integrity *bi;
0273     struct request_queue *q;
0274     void *buf;
0275     unsigned long start, end;
0276     unsigned int len, nr_pages;
0277     unsigned int bytes, offset, i;
0278     unsigned int intervals;
0279 
0280     bi = bdev_get_integrity(bio->bi_bdev);
0281     q = bdev_get_queue(bio->bi_bdev);
0282     BUG_ON(bi == NULL);
0283     BUG_ON(bio_integrity(bio));
0284 
0285     intervals = bio_integrity_intervals(bi, bio_sectors(bio));
0286 
0287     /* Allocate kernel buffer for protection data */
0288     len = intervals * bi->tuple_size;
0289     buf = kmalloc(len, GFP_NOIO | q->bounce_gfp);
0290     if (unlikely(buf == NULL)) {
0291         printk(KERN_ERR "could not allocate integrity buffer\n");
0292         return -ENOMEM;
0293     }
0294 
0295     end = (((unsigned long) buf) + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
0296     start = ((unsigned long) buf) >> PAGE_SHIFT;
0297     nr_pages = end - start;
0298 
0299     /* Allocate bio integrity payload and integrity vectors */
0300     bip = bio_integrity_alloc(bio, GFP_NOIO, nr_pages);
0301     if (IS_ERR(bip)) {
0302         printk(KERN_ERR "could not allocate data integrity bioset\n");
0303         kfree(buf);
0304         return PTR_ERR(bip);
0305     }
0306 
0307     bip->bip_flags |= BIP_BLOCK_INTEGRITY;
0308     bip->bip_iter.bi_size = len;
0309     bip_set_seed(bip, bio->bi_iter.bi_sector);
0310 
0311     if (bi->flags & BLK_INTEGRITY_IP_CHECKSUM)
0312         bip->bip_flags |= BIP_IP_CHECKSUM;
0313 
0314     /* Map it */
0315     offset = offset_in_page(buf);
0316     for (i = 0 ; i < nr_pages ; i++) {
0317         int ret;
0318         bytes = PAGE_SIZE - offset;
0319 
0320         if (len <= 0)
0321             break;
0322 
0323         if (bytes > len)
0324             bytes = len;
0325 
0326         ret = bio_integrity_add_page(bio, virt_to_page(buf),
0327                          bytes, offset);
0328 
0329         if (ret == 0)
0330             return 0;
0331 
0332         if (ret < bytes)
0333             break;
0334 
0335         buf += bytes;
0336         len -= bytes;
0337         offset = 0;
0338     }
0339 
0340     /* Install custom I/O completion handler if read verify is enabled */
0341     if (bio_data_dir(bio) == READ) {
0342         bip->bip_end_io = bio->bi_end_io;
0343         bio->bi_end_io = bio_integrity_endio;
0344     }
0345 
0346     /* Auto-generate integrity metadata if this is a write */
0347     if (bio_data_dir(bio) == WRITE)
0348         bio_integrity_process(bio, bi->profile->generate_fn);
0349 
0350     return 0;
0351 }
0352 EXPORT_SYMBOL(bio_integrity_prep);
0353 
0354 /**
0355  * bio_integrity_verify_fn - Integrity I/O completion worker
0356  * @work:   Work struct stored in bio to be verified
0357  *
0358  * Description: This workqueue function is called to complete a READ
0359  * request.  The function verifies the transferred integrity metadata
0360  * and then calls the original bio end_io function.
0361  */
0362 static void bio_integrity_verify_fn(struct work_struct *work)
0363 {
0364     struct bio_integrity_payload *bip =
0365         container_of(work, struct bio_integrity_payload, bip_work);
0366     struct bio *bio = bip->bip_bio;
0367     struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
0368 
0369     bio->bi_error = bio_integrity_process(bio, bi->profile->verify_fn);
0370 
0371     /* Restore original bio completion handler */
0372     bio->bi_end_io = bip->bip_end_io;
0373     bio_endio(bio);
0374 }
0375 
0376 /**
0377  * bio_integrity_endio - Integrity I/O completion function
0378  * @bio:    Protected bio
0379  * @error:  Pointer to errno
0380  *
0381  * Description: Completion for integrity I/O
0382  *
0383  * Normally I/O completion is done in interrupt context.  However,
0384  * verifying I/O integrity is a time-consuming task which must be run
0385  * in process context.  This function postpones completion
0386  * accordingly.
0387  */
0388 void bio_integrity_endio(struct bio *bio)
0389 {
0390     struct bio_integrity_payload *bip = bio_integrity(bio);
0391 
0392     BUG_ON(bip->bip_bio != bio);
0393 
0394     /* In case of an I/O error there is no point in verifying the
0395      * integrity metadata.  Restore original bio end_io handler
0396      * and run it.
0397      */
0398     if (bio->bi_error) {
0399         bio->bi_end_io = bip->bip_end_io;
0400         bio_endio(bio);
0401 
0402         return;
0403     }
0404 
0405     INIT_WORK(&bip->bip_work, bio_integrity_verify_fn);
0406     queue_work(kintegrityd_wq, &bip->bip_work);
0407 }
0408 EXPORT_SYMBOL(bio_integrity_endio);
0409 
0410 /**
0411  * bio_integrity_advance - Advance integrity vector
0412  * @bio:    bio whose integrity vector to update
0413  * @bytes_done: number of data bytes that have been completed
0414  *
0415  * Description: This function calculates how many integrity bytes the
0416  * number of completed data bytes correspond to and advances the
0417  * integrity vector accordingly.
0418  */
0419 void bio_integrity_advance(struct bio *bio, unsigned int bytes_done)
0420 {
0421     struct bio_integrity_payload *bip = bio_integrity(bio);
0422     struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
0423     unsigned bytes = bio_integrity_bytes(bi, bytes_done >> 9);
0424 
0425     bvec_iter_advance(bip->bip_vec, &bip->bip_iter, bytes);
0426 }
0427 EXPORT_SYMBOL(bio_integrity_advance);
0428 
0429 /**
0430  * bio_integrity_trim - Trim integrity vector
0431  * @bio:    bio whose integrity vector to update
0432  * @offset: offset to first data sector
0433  * @sectors:    number of data sectors
0434  *
0435  * Description: Used to trim the integrity vector in a cloned bio.
0436  * The ivec will be advanced corresponding to 'offset' data sectors
0437  * and the length will be truncated corresponding to 'len' data
0438  * sectors.
0439  */
0440 void bio_integrity_trim(struct bio *bio, unsigned int offset,
0441             unsigned int sectors)
0442 {
0443     struct bio_integrity_payload *bip = bio_integrity(bio);
0444     struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
0445 
0446     bio_integrity_advance(bio, offset << 9);
0447     bip->bip_iter.bi_size = bio_integrity_bytes(bi, sectors);
0448 }
0449 EXPORT_SYMBOL(bio_integrity_trim);
0450 
0451 /**
0452  * bio_integrity_clone - Callback for cloning bios with integrity metadata
0453  * @bio:    New bio
0454  * @bio_src:    Original bio
0455  * @gfp_mask:   Memory allocation mask
0456  *
0457  * Description: Called to allocate a bip when cloning a bio
0458  */
0459 int bio_integrity_clone(struct bio *bio, struct bio *bio_src,
0460             gfp_t gfp_mask)
0461 {
0462     struct bio_integrity_payload *bip_src = bio_integrity(bio_src);
0463     struct bio_integrity_payload *bip;
0464 
0465     BUG_ON(bip_src == NULL);
0466 
0467     bip = bio_integrity_alloc(bio, gfp_mask, bip_src->bip_vcnt);
0468     if (IS_ERR(bip))
0469         return PTR_ERR(bip);
0470 
0471     memcpy(bip->bip_vec, bip_src->bip_vec,
0472            bip_src->bip_vcnt * sizeof(struct bio_vec));
0473 
0474     bip->bip_vcnt = bip_src->bip_vcnt;
0475     bip->bip_iter = bip_src->bip_iter;
0476 
0477     return 0;
0478 }
0479 EXPORT_SYMBOL(bio_integrity_clone);
0480 
0481 int bioset_integrity_create(struct bio_set *bs, int pool_size)
0482 {
0483     if (bs->bio_integrity_pool)
0484         return 0;
0485 
0486     bs->bio_integrity_pool = mempool_create_slab_pool(pool_size, bip_slab);
0487     if (!bs->bio_integrity_pool)
0488         return -1;
0489 
0490     bs->bvec_integrity_pool = biovec_create_pool(pool_size);
0491     if (!bs->bvec_integrity_pool) {
0492         mempool_destroy(bs->bio_integrity_pool);
0493         return -1;
0494     }
0495 
0496     return 0;
0497 }
0498 EXPORT_SYMBOL(bioset_integrity_create);
0499 
0500 void bioset_integrity_free(struct bio_set *bs)
0501 {
0502     if (bs->bio_integrity_pool)
0503         mempool_destroy(bs->bio_integrity_pool);
0504 
0505     if (bs->bvec_integrity_pool)
0506         mempool_destroy(bs->bvec_integrity_pool);
0507 }
0508 EXPORT_SYMBOL(bioset_integrity_free);
0509 
0510 void __init bio_integrity_init(void)
0511 {
0512     /*
0513      * kintegrityd won't block much but may burn a lot of CPU cycles.
0514      * Make it highpri CPU intensive wq with max concurrency of 1.
0515      */
0516     kintegrityd_wq = alloc_workqueue("kintegrityd", WQ_MEM_RECLAIM |
0517                      WQ_HIGHPRI | WQ_CPU_INTENSIVE, 1);
0518     if (!kintegrityd_wq)
0519         panic("Failed to create kintegrityd\n");
0520 
0521     bip_slab = kmem_cache_create("bio_integrity_payload",
0522                      sizeof(struct bio_integrity_payload) +
0523                      sizeof(struct bio_vec) * BIP_INLINE_VECS,
0524                      0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
0525 }