Back to home page

LXR

 
 

    


0001 /*
0002  * Functions related to generic helpers functions
0003  */
0004 #include <linux/kernel.h>
0005 #include <linux/module.h>
0006 #include <linux/bio.h>
0007 #include <linux/blkdev.h>
0008 #include <linux/scatterlist.h>
0009 
0010 #include "blk.h"
0011 
0012 static struct bio *next_bio(struct bio *bio, unsigned int nr_pages,
0013         gfp_t gfp)
0014 {
0015     struct bio *new = bio_alloc(gfp, nr_pages);
0016 
0017     if (bio) {
0018         bio_chain(bio, new);
0019         submit_bio(bio);
0020     }
0021 
0022     return new;
0023 }
0024 
0025 int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
0026         sector_t nr_sects, gfp_t gfp_mask, int flags,
0027         struct bio **biop)
0028 {
0029     struct request_queue *q = bdev_get_queue(bdev);
0030     struct bio *bio = *biop;
0031     unsigned int granularity;
0032     unsigned int op;
0033     int alignment;
0034     sector_t bs_mask;
0035 
0036     if (!q)
0037         return -ENXIO;
0038 
0039     if (flags & BLKDEV_DISCARD_SECURE) {
0040         if (flags & BLKDEV_DISCARD_ZERO)
0041             return -EOPNOTSUPP;
0042         if (!blk_queue_secure_erase(q))
0043             return -EOPNOTSUPP;
0044         op = REQ_OP_SECURE_ERASE;
0045     } else {
0046         if (!blk_queue_discard(q))
0047             return -EOPNOTSUPP;
0048         if ((flags & BLKDEV_DISCARD_ZERO) &&
0049             !q->limits.discard_zeroes_data)
0050             return -EOPNOTSUPP;
0051         op = REQ_OP_DISCARD;
0052     }
0053 
0054     bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
0055     if ((sector | nr_sects) & bs_mask)
0056         return -EINVAL;
0057 
0058     /* Zero-sector (unknown) and one-sector granularities are the same.  */
0059     granularity = max(q->limits.discard_granularity >> 9, 1U);
0060     alignment = (bdev_discard_alignment(bdev) >> 9) % granularity;
0061 
0062     while (nr_sects) {
0063         unsigned int req_sects;
0064         sector_t end_sect, tmp;
0065 
0066         /* Make sure bi_size doesn't overflow */
0067         req_sects = min_t(sector_t, nr_sects, UINT_MAX >> 9);
0068 
0069         /**
0070          * If splitting a request, and the next starting sector would be
0071          * misaligned, stop the discard at the previous aligned sector.
0072          */
0073         end_sect = sector + req_sects;
0074         tmp = end_sect;
0075         if (req_sects < nr_sects &&
0076             sector_div(tmp, granularity) != alignment) {
0077             end_sect = end_sect - alignment;
0078             sector_div(end_sect, granularity);
0079             end_sect = end_sect * granularity + alignment;
0080             req_sects = end_sect - sector;
0081         }
0082 
0083         bio = next_bio(bio, 0, gfp_mask);
0084         bio->bi_iter.bi_sector = sector;
0085         bio->bi_bdev = bdev;
0086         bio_set_op_attrs(bio, op, 0);
0087 
0088         bio->bi_iter.bi_size = req_sects << 9;
0089         nr_sects -= req_sects;
0090         sector = end_sect;
0091 
0092         /*
0093          * We can loop for a long time in here, if someone does
0094          * full device discards (like mkfs). Be nice and allow
0095          * us to schedule out to avoid softlocking if preempt
0096          * is disabled.
0097          */
0098         cond_resched();
0099     }
0100 
0101     *biop = bio;
0102     return 0;
0103 }
0104 EXPORT_SYMBOL(__blkdev_issue_discard);
0105 
0106 /**
0107  * blkdev_issue_discard - queue a discard
0108  * @bdev:   blockdev to issue discard for
0109  * @sector: start sector
0110  * @nr_sects:   number of sectors to discard
0111  * @gfp_mask:   memory allocation flags (for bio_alloc)
0112  * @flags:  BLKDEV_IFL_* flags to control behaviour
0113  *
0114  * Description:
0115  *    Issue a discard request for the sectors in question.
0116  */
0117 int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
0118         sector_t nr_sects, gfp_t gfp_mask, unsigned long flags)
0119 {
0120     struct bio *bio = NULL;
0121     struct blk_plug plug;
0122     int ret;
0123 
0124     blk_start_plug(&plug);
0125     ret = __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, flags,
0126             &bio);
0127     if (!ret && bio) {
0128         ret = submit_bio_wait(bio);
0129         if (ret == -EOPNOTSUPP && !(flags & BLKDEV_DISCARD_ZERO))
0130             ret = 0;
0131         bio_put(bio);
0132     }
0133     blk_finish_plug(&plug);
0134 
0135     return ret;
0136 }
0137 EXPORT_SYMBOL(blkdev_issue_discard);
0138 
0139 /**
0140  * __blkdev_issue_write_same - generate number of bios with same page
0141  * @bdev:   target blockdev
0142  * @sector: start sector
0143  * @nr_sects:   number of sectors to write
0144  * @gfp_mask:   memory allocation flags (for bio_alloc)
0145  * @page:   page containing data to write
0146  * @biop:   pointer to anchor bio
0147  *
0148  * Description:
0149  *  Generate and issue number of bios(REQ_OP_WRITE_SAME) with same page.
0150  */
0151 static int __blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
0152         sector_t nr_sects, gfp_t gfp_mask, struct page *page,
0153         struct bio **biop)
0154 {
0155     struct request_queue *q = bdev_get_queue(bdev);
0156     unsigned int max_write_same_sectors;
0157     struct bio *bio = *biop;
0158     sector_t bs_mask;
0159 
0160     if (!q)
0161         return -ENXIO;
0162 
0163     bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
0164     if ((sector | nr_sects) & bs_mask)
0165         return -EINVAL;
0166 
0167     if (!bdev_write_same(bdev))
0168         return -EOPNOTSUPP;
0169 
0170     /* Ensure that max_write_same_sectors doesn't overflow bi_size */
0171     max_write_same_sectors = UINT_MAX >> 9;
0172 
0173     while (nr_sects) {
0174         bio = next_bio(bio, 1, gfp_mask);
0175         bio->bi_iter.bi_sector = sector;
0176         bio->bi_bdev = bdev;
0177         bio->bi_vcnt = 1;
0178         bio->bi_io_vec->bv_page = page;
0179         bio->bi_io_vec->bv_offset = 0;
0180         bio->bi_io_vec->bv_len = bdev_logical_block_size(bdev);
0181         bio_set_op_attrs(bio, REQ_OP_WRITE_SAME, 0);
0182 
0183         if (nr_sects > max_write_same_sectors) {
0184             bio->bi_iter.bi_size = max_write_same_sectors << 9;
0185             nr_sects -= max_write_same_sectors;
0186             sector += max_write_same_sectors;
0187         } else {
0188             bio->bi_iter.bi_size = nr_sects << 9;
0189             nr_sects = 0;
0190         }
0191         cond_resched();
0192     }
0193 
0194     *biop = bio;
0195     return 0;
0196 }
0197 
0198 /**
0199  * blkdev_issue_write_same - queue a write same operation
0200  * @bdev:   target blockdev
0201  * @sector: start sector
0202  * @nr_sects:   number of sectors to write
0203  * @gfp_mask:   memory allocation flags (for bio_alloc)
0204  * @page:   page containing data
0205  *
0206  * Description:
0207  *    Issue a write same request for the sectors in question.
0208  */
0209 int blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
0210                 sector_t nr_sects, gfp_t gfp_mask,
0211                 struct page *page)
0212 {
0213     struct bio *bio = NULL;
0214     struct blk_plug plug;
0215     int ret;
0216 
0217     blk_start_plug(&plug);
0218     ret = __blkdev_issue_write_same(bdev, sector, nr_sects, gfp_mask, page,
0219             &bio);
0220     if (ret == 0 && bio) {
0221         ret = submit_bio_wait(bio);
0222         bio_put(bio);
0223     }
0224     blk_finish_plug(&plug);
0225     return ret;
0226 }
0227 EXPORT_SYMBOL(blkdev_issue_write_same);
0228 
0229 /**
0230  * __blkdev_issue_write_zeroes - generate number of bios with WRITE ZEROES
0231  * @bdev:   blockdev to issue
0232  * @sector: start sector
0233  * @nr_sects:   number of sectors to write
0234  * @gfp_mask:   memory allocation flags (for bio_alloc)
0235  * @biop:   pointer to anchor bio
0236  *
0237  * Description:
0238  *  Generate and issue number of bios(REQ_OP_WRITE_ZEROES) with zerofiled pages.
0239  */
0240 static int __blkdev_issue_write_zeroes(struct block_device *bdev,
0241         sector_t sector, sector_t nr_sects, gfp_t gfp_mask,
0242         struct bio **biop)
0243 {
0244     struct bio *bio = *biop;
0245     unsigned int max_write_zeroes_sectors;
0246     struct request_queue *q = bdev_get_queue(bdev);
0247 
0248     if (!q)
0249         return -ENXIO;
0250 
0251     /* Ensure that max_write_zeroes_sectors doesn't overflow bi_size */
0252     max_write_zeroes_sectors = bdev_write_zeroes_sectors(bdev);
0253 
0254     if (max_write_zeroes_sectors == 0)
0255         return -EOPNOTSUPP;
0256 
0257     while (nr_sects) {
0258         bio = next_bio(bio, 0, gfp_mask);
0259         bio->bi_iter.bi_sector = sector;
0260         bio->bi_bdev = bdev;
0261         bio_set_op_attrs(bio, REQ_OP_WRITE_ZEROES, 0);
0262 
0263         if (nr_sects > max_write_zeroes_sectors) {
0264             bio->bi_iter.bi_size = max_write_zeroes_sectors << 9;
0265             nr_sects -= max_write_zeroes_sectors;
0266             sector += max_write_zeroes_sectors;
0267         } else {
0268             bio->bi_iter.bi_size = nr_sects << 9;
0269             nr_sects = 0;
0270         }
0271         cond_resched();
0272     }
0273 
0274     *biop = bio;
0275     return 0;
0276 }
0277 
0278 /**
0279  * __blkdev_issue_zeroout - generate number of zero filed write bios
0280  * @bdev:   blockdev to issue
0281  * @sector: start sector
0282  * @nr_sects:   number of sectors to write
0283  * @gfp_mask:   memory allocation flags (for bio_alloc)
0284  * @biop:   pointer to anchor bio
0285  * @discard:    discard flag
0286  *
0287  * Description:
0288  *  Generate and issue number of bios with zerofiled pages.
0289  */
0290 int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
0291         sector_t nr_sects, gfp_t gfp_mask, struct bio **biop,
0292         bool discard)
0293 {
0294     int ret;
0295     int bi_size = 0;
0296     struct bio *bio = *biop;
0297     unsigned int sz;
0298     sector_t bs_mask;
0299 
0300     bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
0301     if ((sector | nr_sects) & bs_mask)
0302         return -EINVAL;
0303 
0304     ret = __blkdev_issue_write_zeroes(bdev, sector, nr_sects, gfp_mask,
0305             biop);
0306     if (ret == 0 || (ret && ret != -EOPNOTSUPP))
0307         goto out;
0308 
0309     ret = 0;
0310     while (nr_sects != 0) {
0311         bio = next_bio(bio, min(nr_sects, (sector_t)BIO_MAX_PAGES),
0312                 gfp_mask);
0313         bio->bi_iter.bi_sector = sector;
0314         bio->bi_bdev   = bdev;
0315         bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
0316 
0317         while (nr_sects != 0) {
0318             sz = min((sector_t) PAGE_SIZE >> 9 , nr_sects);
0319             bi_size = bio_add_page(bio, ZERO_PAGE(0), sz << 9, 0);
0320             nr_sects -= bi_size >> 9;
0321             sector += bi_size >> 9;
0322             if (bi_size < (sz << 9))
0323                 break;
0324         }
0325         cond_resched();
0326     }
0327 
0328     *biop = bio;
0329 out:
0330     return ret;
0331 }
0332 EXPORT_SYMBOL(__blkdev_issue_zeroout);
0333 
0334 /**
0335  * blkdev_issue_zeroout - zero-fill a block range
0336  * @bdev:   blockdev to write
0337  * @sector: start sector
0338  * @nr_sects:   number of sectors to write
0339  * @gfp_mask:   memory allocation flags (for bio_alloc)
0340  * @discard:    whether to discard the block range
0341  *
0342  * Description:
0343  *  Zero-fill a block range.  If the discard flag is set and the block
0344  *  device guarantees that subsequent READ operations to the block range
0345  *  in question will return zeroes, the blocks will be discarded. Should
0346  *  the discard request fail, if the discard flag is not set, or if
0347  *  discard_zeroes_data is not supported, this function will resort to
0348  *  zeroing the blocks manually, thus provisioning (allocating,
0349  *  anchoring) them. If the block device supports WRITE ZEROES or WRITE SAME
0350  *  command(s), blkdev_issue_zeroout() will use it to optimize the process of
0351  *  clearing the block range. Otherwise the zeroing will be performed
0352  *  using regular WRITE calls.
0353  */
0354 int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
0355              sector_t nr_sects, gfp_t gfp_mask, bool discard)
0356 {
0357     int ret;
0358     struct bio *bio = NULL;
0359     struct blk_plug plug;
0360 
0361     if (discard) {
0362         if (!blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask,
0363                 BLKDEV_DISCARD_ZERO))
0364             return 0;
0365     }
0366 
0367     if (!blkdev_issue_write_same(bdev, sector, nr_sects, gfp_mask,
0368             ZERO_PAGE(0)))
0369         return 0;
0370 
0371     blk_start_plug(&plug);
0372     ret = __blkdev_issue_zeroout(bdev, sector, nr_sects, gfp_mask,
0373             &bio, discard);
0374     if (ret == 0 && bio) {
0375         ret = submit_bio_wait(bio);
0376         bio_put(bio);
0377     }
0378     blk_finish_plug(&plug);
0379 
0380     return ret;
0381 }
0382 EXPORT_SYMBOL(blkdev_issue_zeroout);