the-tree/block/t10-pi.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * t10_pi.c - Functions for generating and verifying T10 Protection
0004  *        Information.
0005  */
0006
0007 #include <linux/t10-pi.h>
0008 #include <linux/blk-integrity.h>
0009 #include <linux/crc-t10dif.h>
0010 #include <linux/crc64.h>
0011 #include <linux/module.h>
0012 #include <net/checksum.h>
0013 #include <asm/unaligned.h>
0014
0015 typedef __be16 (csum_fn) (void *, unsigned int);
0016
0017 static __be16 t10_pi_crc_fn(void *data, unsigned int len)
0018 {
0019     return cpu_to_be16(crc_t10dif(data, len));
0020 }
0021
0022 static __be16 t10_pi_ip_fn(void *data, unsigned int len)
0023 {
0024     return (__force __be16)ip_compute_csum(data, len);
0025 }
0026
0027 /*
0028  * Type 1 and Type 2 protection use the same format: 16 bit guard tag,
0029  * 16 bit app tag, 32 bit reference tag. Type 3 does not define the ref
0030  * tag.
0031  */
0032 static blk_status_t t10_pi_generate(struct blk_integrity_iter *iter,
0033         csum_fn *fn, enum t10_dif_type type)
0034 {
0035     unsigned int i;
0036
0037     for (i = 0 ; i < iter->data_size ; i += iter->interval) {
0038         struct t10_pi_tuple *pi = iter->prot_buf;
0039
0040         pi->guard_tag = fn(iter->data_buf, iter->interval);
0041         pi->app_tag = 0;
0042
0043         if (type == T10_PI_TYPE1_PROTECTION)
0044             pi->ref_tag = cpu_to_be32(lower_32_bits(iter->seed));
0045         else
0046             pi->ref_tag = 0;
0047
0048         iter->data_buf += iter->interval;
0049         iter->prot_buf += iter->tuple_size;
0050         iter->seed++;
0051     }
0052
0053     return BLK_STS_OK;
0054 }
0055
0056 static blk_status_t t10_pi_verify(struct blk_integrity_iter *iter,
0057         csum_fn *fn, enum t10_dif_type type)
0058 {
0059     unsigned int i;
0060
0061     BUG_ON(type == T10_PI_TYPE0_PROTECTION);
0062
0063     for (i = 0 ; i < iter->data_size ; i += iter->interval) {
0064         struct t10_pi_tuple *pi = iter->prot_buf;
0065         __be16 csum;
0066
0067         if (type == T10_PI_TYPE1_PROTECTION ||
0068             type == T10_PI_TYPE2_PROTECTION) {
0069             if (pi->app_tag == T10_PI_APP_ESCAPE)
0070                 goto next;
0071
0072             if (be32_to_cpu(pi->ref_tag) !=
0073                 lower_32_bits(iter->seed)) {
0074                 pr_err("%s: ref tag error at location %llu " \
0075                        "(rcvd %u)\n", iter->disk_name,
0076                        (unsigned long long)
0077                        iter->seed, be32_to_cpu(pi->ref_tag));
0078                 return BLK_STS_PROTECTION;
0079             }
0080         } else if (type == T10_PI_TYPE3_PROTECTION) {
0081             if (pi->app_tag == T10_PI_APP_ESCAPE &&
0082                 pi->ref_tag == T10_PI_REF_ESCAPE)
0083                 goto next;
0084         }
0085
0086         csum = fn(iter->data_buf, iter->interval);
0087
0088         if (pi->guard_tag != csum) {
0089             pr_err("%s: guard tag error at sector %llu " \
0090                    "(rcvd %04x, want %04x)\n", iter->disk_name,
0091                    (unsigned long long)iter->seed,
0092                    be16_to_cpu(pi->guard_tag), be16_to_cpu(csum));
0093             return BLK_STS_PROTECTION;
0094         }
0095
0096 next:
0097         iter->data_buf += iter->interval;
0098         iter->prot_buf += iter->tuple_size;
0099         iter->seed++;
0100     }
0101
0102     return BLK_STS_OK;
0103 }
0104
0105 static blk_status_t t10_pi_type1_generate_crc(struct blk_integrity_iter *iter)
0106 {
0107     return t10_pi_generate(iter, t10_pi_crc_fn, T10_PI_TYPE1_PROTECTION);
0108 }
0109
0110 static blk_status_t t10_pi_type1_generate_ip(struct blk_integrity_iter *iter)
0111 {
0112     return t10_pi_generate(iter, t10_pi_ip_fn, T10_PI_TYPE1_PROTECTION);
0113 }
0114
0115 static blk_status_t t10_pi_type1_verify_crc(struct blk_integrity_iter *iter)
0116 {
0117     return t10_pi_verify(iter, t10_pi_crc_fn, T10_PI_TYPE1_PROTECTION);
0118 }
0119
0120 static blk_status_t t10_pi_type1_verify_ip(struct blk_integrity_iter *iter)
0121 {
0122     return t10_pi_verify(iter, t10_pi_ip_fn, T10_PI_TYPE1_PROTECTION);
0123 }
0124
0125 /**
0126  * t10_pi_type1_prepare - prepare PI prior submitting request to device
0127  * @rq:              request with PI that should be prepared
0128  *
0129  * For Type 1/Type 2, the virtual start sector is the one that was
0130  * originally submitted by the block layer for the ref_tag usage. Due to
0131  * partitioning, MD/DM cloning, etc. the actual physical start sector is
0132  * likely to be different. Remap protection information to match the
0133  * physical LBA.
0134  */
0135 static void t10_pi_type1_prepare(struct request *rq)
0136 {
0137     const int tuple_sz = rq->q->integrity.tuple_size;
0138     u32 ref_tag = t10_pi_ref_tag(rq);
0139     struct bio *bio;
0140
0141     __rq_for_each_bio(bio, rq) {
0142         struct bio_integrity_payload *bip = bio_integrity(bio);
0143         u32 virt = bip_get_seed(bip) & 0xffffffff;
0144         struct bio_vec iv;
0145         struct bvec_iter iter;
0146
0147         /* Already remapped? */
0148         if (bip->bip_flags & BIP_MAPPED_INTEGRITY)
0149             break;
0150
0151         bip_for_each_vec(iv, bip, iter) {
0152             unsigned int j;
0153             void *p;
0154
0155             p = bvec_kmap_local(&iv);
0156             for (j = 0; j < iv.bv_len; j += tuple_sz) {
0157                 struct t10_pi_tuple *pi = p;
0158
0159                 if (be32_to_cpu(pi->ref_tag) == virt)
0160                     pi->ref_tag = cpu_to_be32(ref_tag);
0161                 virt++;
0162                 ref_tag++;
0163                 p += tuple_sz;
0164             }
0165             kunmap_local(p);
0166         }
0167
0168         bip->bip_flags |= BIP_MAPPED_INTEGRITY;
0169     }
0170 }
0171
0172 /**
0173  * t10_pi_type1_complete - prepare PI prior returning request to the blk layer
0174  * @rq:              request with PI that should be prepared
0175  * @nr_bytes:        total bytes to prepare
0176  *
0177  * For Type 1/Type 2, the virtual start sector is the one that was
0178  * originally submitted by the block layer for the ref_tag usage. Due to
0179  * partitioning, MD/DM cloning, etc. the actual physical start sector is
0180  * likely to be different. Since the physical start sector was submitted
0181  * to the device, we should remap it back to virtual values expected by the
0182  * block layer.
0183  */
0184 static void t10_pi_type1_complete(struct request *rq, unsigned int nr_bytes)
0185 {
0186     unsigned intervals = nr_bytes >> rq->q->integrity.interval_exp;
0187     const int tuple_sz = rq->q->integrity.tuple_size;
0188     u32 ref_tag = t10_pi_ref_tag(rq);
0189     struct bio *bio;
0190
0191     __rq_for_each_bio(bio, rq) {
0192         struct bio_integrity_payload *bip = bio_integrity(bio);
0193         u32 virt = bip_get_seed(bip) & 0xffffffff;
0194         struct bio_vec iv;
0195         struct bvec_iter iter;
0196
0197         bip_for_each_vec(iv, bip, iter) {
0198             unsigned int j;
0199             void *p;
0200
0201             p = bvec_kmap_local(&iv);
0202             for (j = 0; j < iv.bv_len && intervals; j += tuple_sz) {
0203                 struct t10_pi_tuple *pi = p;
0204
0205                 if (be32_to_cpu(pi->ref_tag) == ref_tag)
0206                     pi->ref_tag = cpu_to_be32(virt);
0207                 virt++;
0208                 ref_tag++;
0209                 intervals--;
0210                 p += tuple_sz;
0211             }
0212             kunmap_local(p);
0213         }
0214     }
0215 }
0216
0217 static blk_status_t t10_pi_type3_generate_crc(struct blk_integrity_iter *iter)
0218 {
0219     return t10_pi_generate(iter, t10_pi_crc_fn, T10_PI_TYPE3_PROTECTION);
0220 }
0221
0222 static blk_status_t t10_pi_type3_generate_ip(struct blk_integrity_iter *iter)
0223 {
0224     return t10_pi_generate(iter, t10_pi_ip_fn, T10_PI_TYPE3_PROTECTION);
0225 }
0226
0227 static blk_status_t t10_pi_type3_verify_crc(struct blk_integrity_iter *iter)
0228 {
0229     return t10_pi_verify(iter, t10_pi_crc_fn, T10_PI_TYPE3_PROTECTION);
0230 }
0231
0232 static blk_status_t t10_pi_type3_verify_ip(struct blk_integrity_iter *iter)
0233 {
0234     return t10_pi_verify(iter, t10_pi_ip_fn, T10_PI_TYPE3_PROTECTION);
0235 }
0236
0237 /* Type 3 does not have a reference tag so no remapping is required. */
0238 static void t10_pi_type3_prepare(struct request *rq)
0239 {
0240 }
0241
0242 /* Type 3 does not have a reference tag so no remapping is required. */
0243 static void t10_pi_type3_complete(struct request *rq, unsigned int nr_bytes)
0244 {
0245 }
0246
0247 const struct blk_integrity_profile t10_pi_type1_crc = {
0248     .name           = "T10-DIF-TYPE1-CRC",
0249     .generate_fn        = t10_pi_type1_generate_crc,
0250     .verify_fn      = t10_pi_type1_verify_crc,
0251     .prepare_fn     = t10_pi_type1_prepare,
0252     .complete_fn        = t10_pi_type1_complete,
0253 };
0254 EXPORT_SYMBOL(t10_pi_type1_crc);
0255
0256 const struct blk_integrity_profile t10_pi_type1_ip = {
0257     .name           = "T10-DIF-TYPE1-IP",
0258     .generate_fn        = t10_pi_type1_generate_ip,
0259     .verify_fn      = t10_pi_type1_verify_ip,
0260     .prepare_fn     = t10_pi_type1_prepare,
0261     .complete_fn        = t10_pi_type1_complete,
0262 };
0263 EXPORT_SYMBOL(t10_pi_type1_ip);
0264
0265 const struct blk_integrity_profile t10_pi_type3_crc = {
0266     .name           = "T10-DIF-TYPE3-CRC",
0267     .generate_fn        = t10_pi_type3_generate_crc,
0268     .verify_fn      = t10_pi_type3_verify_crc,
0269     .prepare_fn     = t10_pi_type3_prepare,
0270     .complete_fn        = t10_pi_type3_complete,
0271 };
0272 EXPORT_SYMBOL(t10_pi_type3_crc);
0273
0274 const struct blk_integrity_profile t10_pi_type3_ip = {
0275     .name           = "T10-DIF-TYPE3-IP",
0276     .generate_fn        = t10_pi_type3_generate_ip,
0277     .verify_fn      = t10_pi_type3_verify_ip,
0278     .prepare_fn     = t10_pi_type3_prepare,
0279     .complete_fn        = t10_pi_type3_complete,
0280 };
0281 EXPORT_SYMBOL(t10_pi_type3_ip);
0282
0283 static __be64 ext_pi_crc64(void *data, unsigned int len)
0284 {
0285     return cpu_to_be64(crc64_rocksoft(data, len));
0286 }
0287
0288 static blk_status_t ext_pi_crc64_generate(struct blk_integrity_iter *iter,
0289                     enum t10_dif_type type)
0290 {
0291     unsigned int i;
0292
0293     for (i = 0 ; i < iter->data_size ; i += iter->interval) {
0294         struct crc64_pi_tuple *pi = iter->prot_buf;
0295
0296         pi->guard_tag = ext_pi_crc64(iter->data_buf, iter->interval);
0297         pi->app_tag = 0;
0298
0299         if (type == T10_PI_TYPE1_PROTECTION)
0300             put_unaligned_be48(iter->seed, pi->ref_tag);
0301         else
0302             put_unaligned_be48(0ULL, pi->ref_tag);
0303
0304         iter->data_buf += iter->interval;
0305         iter->prot_buf += iter->tuple_size;
0306         iter->seed++;
0307     }
0308
0309     return BLK_STS_OK;
0310 }
0311
0312 static bool ext_pi_ref_escape(u8 *ref_tag)
0313 {
0314     static u8 ref_escape[6] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
0315
0316     return memcmp(ref_tag, ref_escape, sizeof(ref_escape)) == 0;
0317 }
0318
0319 static blk_status_t ext_pi_crc64_verify(struct blk_integrity_iter *iter,
0320                       enum t10_dif_type type)
0321 {
0322     unsigned int i;
0323
0324     for (i = 0; i < iter->data_size; i += iter->interval) {
0325         struct crc64_pi_tuple *pi = iter->prot_buf;
0326         u64 ref, seed;
0327         __be64 csum;
0328
0329         if (type == T10_PI_TYPE1_PROTECTION) {
0330             if (pi->app_tag == T10_PI_APP_ESCAPE)
0331                 goto next;
0332
0333             ref = get_unaligned_be48(pi->ref_tag);
0334             seed = lower_48_bits(iter->seed);
0335             if (ref != seed) {
0336                 pr_err("%s: ref tag error at location %llu (rcvd %llu)\n",
0337                     iter->disk_name, seed, ref);
0338                 return BLK_STS_PROTECTION;
0339             }
0340         } else if (type == T10_PI_TYPE3_PROTECTION) {
0341             if (pi->app_tag == T10_PI_APP_ESCAPE &&
0342                 ext_pi_ref_escape(pi->ref_tag))
0343                 goto next;
0344         }
0345
0346         csum = ext_pi_crc64(iter->data_buf, iter->interval);
0347         if (pi->guard_tag != csum) {
0348             pr_err("%s: guard tag error at sector %llu " \
0349                    "(rcvd %016llx, want %016llx)\n",
0350                 iter->disk_name, (unsigned long long)iter->seed,
0351                 be64_to_cpu(pi->guard_tag), be64_to_cpu(csum));
0352             return BLK_STS_PROTECTION;
0353         }
0354
0355 next:
0356         iter->data_buf += iter->interval;
0357         iter->prot_buf += iter->tuple_size;
0358         iter->seed++;
0359     }
0360
0361     return BLK_STS_OK;
0362 }
0363
0364 static blk_status_t ext_pi_type1_verify_crc64(struct blk_integrity_iter *iter)
0365 {
0366     return ext_pi_crc64_verify(iter, T10_PI_TYPE1_PROTECTION);
0367 }
0368
0369 static blk_status_t ext_pi_type1_generate_crc64(struct blk_integrity_iter *iter)
0370 {
0371     return ext_pi_crc64_generate(iter, T10_PI_TYPE1_PROTECTION);
0372 }
0373
0374 static void ext_pi_type1_prepare(struct request *rq)
0375 {
0376     const int tuple_sz = rq->q->integrity.tuple_size;
0377     u64 ref_tag = ext_pi_ref_tag(rq);
0378     struct bio *bio;
0379
0380     __rq_for_each_bio(bio, rq) {
0381         struct bio_integrity_payload *bip = bio_integrity(bio);
0382         u64 virt = lower_48_bits(bip_get_seed(bip));
0383         struct bio_vec iv;
0384         struct bvec_iter iter;
0385
0386         /* Already remapped? */
0387         if (bip->bip_flags & BIP_MAPPED_INTEGRITY)
0388             break;
0389
0390         bip_for_each_vec(iv, bip, iter) {
0391             unsigned int j;
0392             void *p;
0393
0394             p = bvec_kmap_local(&iv);
0395             for (j = 0; j < iv.bv_len; j += tuple_sz) {
0396                 struct crc64_pi_tuple *pi = p;
0397                 u64 ref = get_unaligned_be48(pi->ref_tag);
0398
0399                 if (ref == virt)
0400                     put_unaligned_be48(ref_tag, pi->ref_tag);
0401                 virt++;
0402                 ref_tag++;
0403                 p += tuple_sz;
0404             }
0405             kunmap_local(p);
0406         }
0407
0408         bip->bip_flags |= BIP_MAPPED_INTEGRITY;
0409     }
0410 }
0411
0412 static void ext_pi_type1_complete(struct request *rq, unsigned int nr_bytes)
0413 {
0414     unsigned intervals = nr_bytes >> rq->q->integrity.interval_exp;
0415     const int tuple_sz = rq->q->integrity.tuple_size;
0416     u64 ref_tag = ext_pi_ref_tag(rq);
0417     struct bio *bio;
0418
0419     __rq_for_each_bio(bio, rq) {
0420         struct bio_integrity_payload *bip = bio_integrity(bio);
0421         u64 virt = lower_48_bits(bip_get_seed(bip));
0422         struct bio_vec iv;
0423         struct bvec_iter iter;
0424
0425         bip_for_each_vec(iv, bip, iter) {
0426             unsigned int j;
0427             void *p;
0428
0429             p = bvec_kmap_local(&iv);
0430             for (j = 0; j < iv.bv_len && intervals; j += tuple_sz) {
0431                 struct crc64_pi_tuple *pi = p;
0432                 u64 ref = get_unaligned_be48(pi->ref_tag);
0433
0434                 if (ref == ref_tag)
0435                     put_unaligned_be48(virt, pi->ref_tag);
0436                 virt++;
0437                 ref_tag++;
0438                 intervals--;
0439                 p += tuple_sz;
0440             }
0441             kunmap_local(p);
0442         }
0443     }
0444 }
0445
0446 static blk_status_t ext_pi_type3_verify_crc64(struct blk_integrity_iter *iter)
0447 {
0448     return ext_pi_crc64_verify(iter, T10_PI_TYPE3_PROTECTION);
0449 }
0450
0451 static blk_status_t ext_pi_type3_generate_crc64(struct blk_integrity_iter *iter)
0452 {
0453     return ext_pi_crc64_generate(iter, T10_PI_TYPE3_PROTECTION);
0454 }
0455
0456 const struct blk_integrity_profile ext_pi_type1_crc64 = {
0457     .name           = "EXT-DIF-TYPE1-CRC64",
0458     .generate_fn        = ext_pi_type1_generate_crc64,
0459     .verify_fn      = ext_pi_type1_verify_crc64,
0460     .prepare_fn     = ext_pi_type1_prepare,
0461     .complete_fn        = ext_pi_type1_complete,
0462 };
0463 EXPORT_SYMBOL_GPL(ext_pi_type1_crc64);
0464
0465 const struct blk_integrity_profile ext_pi_type3_crc64 = {
0466     .name           = "EXT-DIF-TYPE3-CRC64",
0467     .generate_fn        = ext_pi_type3_generate_crc64,
0468     .verify_fn      = ext_pi_type3_verify_crc64,
0469     .prepare_fn     = t10_pi_type3_prepare,
0470     .complete_fn        = t10_pi_type3_complete,
0471 };
0472 EXPORT_SYMBOL_GPL(ext_pi_type3_crc64);
0473
0474 MODULE_LICENSE("GPL");
0475 MODULE_LICENSE("GPL");