mtd/nand/ecc-sw-bch.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * This file provides ECC correction for more than 1 bit per block of data,
0004  * using binary BCH codes. It relies on the generic BCH library lib/bch.c.
0005  *
0006  * Copyright © 2011 Ivan Djelic <ivan.djelic@parrot.com>
0007  */
0008
0009 #include <linux/types.h>
0010 #include <linux/kernel.h>
0011 #include <linux/module.h>
0012 #include <linux/slab.h>
0013 #include <linux/bitops.h>
0014 #include <linux/mtd/nand.h>
0015 #include <linux/mtd/nand-ecc-sw-bch.h>
0016
0017 /**
0018  * nand_ecc_sw_bch_calculate - Calculate the ECC corresponding to a data block
0019  * @nand: NAND device
0020  * @buf: Input buffer with raw data
0021  * @code: Output buffer with ECC
0022  */
0023 int nand_ecc_sw_bch_calculate(struct nand_device *nand,
0024                   const unsigned char *buf, unsigned char *code)
0025 {
0026     struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv;
0027     unsigned int i;
0028
0029     memset(code, 0, engine_conf->code_size);
0030     bch_encode(engine_conf->bch, buf, nand->ecc.ctx.conf.step_size, code);
0031
0032     /* apply mask so that an erased page is a valid codeword */
0033     for (i = 0; i < engine_conf->code_size; i++)
0034         code[i] ^= engine_conf->eccmask[i];
0035
0036     return 0;
0037 }
0038 EXPORT_SYMBOL(nand_ecc_sw_bch_calculate);
0039
0040 /**
0041  * nand_ecc_sw_bch_correct - Detect, correct and report bit error(s)
0042  * @nand: NAND device
0043  * @buf: Raw data read from the chip
0044  * @read_ecc: ECC bytes from the chip
0045  * @calc_ecc: ECC calculated from the raw data
0046  *
0047  * Detect and correct bit errors for a data block.
0048  */
0049 int nand_ecc_sw_bch_correct(struct nand_device *nand, unsigned char *buf,
0050                 unsigned char *read_ecc, unsigned char *calc_ecc)
0051 {
0052     struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv;
0053     unsigned int step_size = nand->ecc.ctx.conf.step_size;
0054     unsigned int *errloc = engine_conf->errloc;
0055     int i, count;
0056
0057     count = bch_decode(engine_conf->bch, NULL, step_size, read_ecc,
0058                calc_ecc, NULL, errloc);
0059     if (count > 0) {
0060         for (i = 0; i < count; i++) {
0061             if (errloc[i] < (step_size * 8))
0062                 /* The error is in the data area: correct it */
0063                 buf[errloc[i] >> 3] ^= (1 << (errloc[i] & 7));
0064
0065             /* Otherwise the error is in the ECC area: nothing to do */
0066             pr_debug("%s: corrected bitflip %u\n", __func__,
0067                  errloc[i]);
0068         }
0069     } else if (count < 0) {
0070         pr_err("ECC unrecoverable error\n");
0071         count = -EBADMSG;
0072     }
0073
0074     return count;
0075 }
0076 EXPORT_SYMBOL(nand_ecc_sw_bch_correct);
0077
0078 /**
0079  * nand_ecc_sw_bch_cleanup - Cleanup software BCH ECC resources
0080  * @nand: NAND device
0081  */
0082 static void nand_ecc_sw_bch_cleanup(struct nand_device *nand)
0083 {
0084     struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv;
0085
0086     bch_free(engine_conf->bch);
0087     kfree(engine_conf->errloc);
0088     kfree(engine_conf->eccmask);
0089 }
0090
0091 /**
0092  * nand_ecc_sw_bch_init - Initialize software BCH ECC engine
0093  * @nand: NAND device
0094  *
0095  * Returns: a pointer to a new NAND BCH control structure, or NULL upon failure
0096  *
0097  * Initialize NAND BCH error correction. @nand.ecc parameters 'step_size' and
0098  * 'bytes' are used to compute the following BCH parameters:
0099  *     m, the Galois field order
0100  *     t, the error correction capability
0101  * 'bytes' should be equal to the number of bytes required to store m * t
0102  * bits, where m is such that 2^m - 1 > step_size * 8.
0103  *
0104  * Example: to configure 4 bit correction per 512 bytes, you should pass
0105  * step_size = 512 (thus, m = 13 is the smallest integer such that 2^m - 1 > 512 * 8)
0106  * bytes = 7 (7 bytes are required to store m * t = 13 * 4 = 52 bits)
0107  */
0108 static int nand_ecc_sw_bch_init(struct nand_device *nand)
0109 {
0110     struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv;
0111     unsigned int eccsize = nand->ecc.ctx.conf.step_size;
0112     unsigned int eccbytes = engine_conf->code_size;
0113     unsigned int m, t, i;
0114     unsigned char *erased_page;
0115     int ret;
0116
0117     m = fls(1 + (8 * eccsize));
0118     t = (eccbytes * 8) / m;
0119
0120     engine_conf->bch = bch_init(m, t, 0, false);
0121     if (!engine_conf->bch)
0122         return -EINVAL;
0123
0124     engine_conf->eccmask = kzalloc(eccbytes, GFP_KERNEL);
0125     engine_conf->errloc = kmalloc_array(t, sizeof(*engine_conf->errloc),
0126                         GFP_KERNEL);
0127     if (!engine_conf->eccmask || !engine_conf->errloc) {
0128         ret = -ENOMEM;
0129         goto cleanup;
0130     }
0131
0132     /* Compute and store the inverted ECC of an erased step */
0133     erased_page = kmalloc(eccsize, GFP_KERNEL);
0134     if (!erased_page) {
0135         ret = -ENOMEM;
0136         goto cleanup;
0137     }
0138
0139     memset(erased_page, 0xff, eccsize);
0140     bch_encode(engine_conf->bch, erased_page, eccsize,
0141            engine_conf->eccmask);
0142     kfree(erased_page);
0143
0144     for (i = 0; i < eccbytes; i++)
0145         engine_conf->eccmask[i] ^= 0xff;
0146
0147     /* Verify that the number of code bytes has the expected value */
0148     if (engine_conf->bch->ecc_bytes != eccbytes) {
0149         pr_err("Invalid number of ECC bytes: %u, expected: %u\n",
0150                eccbytes, engine_conf->bch->ecc_bytes);
0151         ret = -EINVAL;
0152         goto cleanup;
0153     }
0154
0155     /* Sanity checks */
0156     if (8 * (eccsize + eccbytes) >= (1 << m)) {
0157         pr_err("ECC step size is too large (%u)\n", eccsize);
0158         ret = -EINVAL;
0159         goto cleanup;
0160     }
0161
0162     return 0;
0163
0164 cleanup:
0165     nand_ecc_sw_bch_cleanup(nand);
0166
0167     return ret;
0168 }
0169
0170 int nand_ecc_sw_bch_init_ctx(struct nand_device *nand)
0171 {
0172     struct nand_ecc_props *conf = &nand->ecc.ctx.conf;
0173     struct mtd_info *mtd = nanddev_to_mtd(nand);
0174     struct nand_ecc_sw_bch_conf *engine_conf;
0175     unsigned int code_size = 0, nsteps;
0176     int ret;
0177
0178     /* Only large page NAND chips may use BCH */
0179     if (mtd->oobsize < 64) {
0180         pr_err("BCH cannot be used with small page NAND chips\n");
0181         return -EINVAL;
0182     }
0183
0184     if (!mtd->ooblayout)
0185         mtd_set_ooblayout(mtd, nand_get_large_page_ooblayout());
0186
0187     conf->engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
0188     conf->algo = NAND_ECC_ALGO_BCH;
0189     conf->step_size = nand->ecc.user_conf.step_size;
0190     conf->strength = nand->ecc.user_conf.strength;
0191
0192     /*
0193      * Board driver should supply ECC size and ECC strength
0194      * values to select how many bits are correctable.
0195      * Otherwise, default to 512 bytes for large page devices and 256 for
0196      * small page devices.
0197      */
0198     if (!conf->step_size) {
0199         if (mtd->oobsize >= 64)
0200             conf->step_size = 512;
0201         else
0202             conf->step_size = 256;
0203
0204         conf->strength = 4;
0205     }
0206
0207     nsteps = mtd->writesize / conf->step_size;
0208
0209     /* Maximize */
0210     if (nand->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH) {
0211         conf->step_size = 1024;
0212         nsteps = mtd->writesize / conf->step_size;
0213         /* Reserve 2 bytes for the BBM */
0214         code_size = (mtd->oobsize - 2) / nsteps;
0215         conf->strength = code_size * 8 / fls(8 * conf->step_size);
0216     }
0217
0218     if (!code_size)
0219         code_size = DIV_ROUND_UP(conf->strength *
0220                      fls(8 * conf->step_size), 8);
0221
0222     if (!conf->strength)
0223         conf->strength = (code_size * 8) / fls(8 * conf->step_size);
0224
0225     if (!code_size && !conf->strength) {
0226         pr_err("Missing ECC parameters\n");
0227         return -EINVAL;
0228     }
0229
0230     engine_conf = kzalloc(sizeof(*engine_conf), GFP_KERNEL);
0231     if (!engine_conf)
0232         return -ENOMEM;
0233
0234     ret = nand_ecc_init_req_tweaking(&engine_conf->req_ctx, nand);
0235     if (ret)
0236         goto free_engine_conf;
0237
0238     engine_conf->code_size = code_size;
0239     engine_conf->calc_buf = kzalloc(mtd->oobsize, GFP_KERNEL);
0240     engine_conf->code_buf = kzalloc(mtd->oobsize, GFP_KERNEL);
0241     if (!engine_conf->calc_buf || !engine_conf->code_buf) {
0242         ret = -ENOMEM;
0243         goto free_bufs;
0244     }
0245
0246     nand->ecc.ctx.priv = engine_conf;
0247     nand->ecc.ctx.nsteps = nsteps;
0248     nand->ecc.ctx.total = nsteps * code_size;
0249
0250     ret = nand_ecc_sw_bch_init(nand);
0251     if (ret)
0252         goto free_bufs;
0253
0254     /* Verify the layout validity */
0255     if (mtd_ooblayout_count_eccbytes(mtd) !=
0256         nand->ecc.ctx.nsteps * engine_conf->code_size) {
0257         pr_err("Invalid ECC layout\n");
0258         ret = -EINVAL;
0259         goto cleanup_bch_ctx;
0260     }
0261
0262     return 0;
0263
0264 cleanup_bch_ctx:
0265     nand_ecc_sw_bch_cleanup(nand);
0266 free_bufs:
0267     nand_ecc_cleanup_req_tweaking(&engine_conf->req_ctx);
0268     kfree(engine_conf->calc_buf);
0269     kfree(engine_conf->code_buf);
0270 free_engine_conf:
0271     kfree(engine_conf);
0272
0273     return ret;
0274 }
0275 EXPORT_SYMBOL(nand_ecc_sw_bch_init_ctx);
0276
0277 void nand_ecc_sw_bch_cleanup_ctx(struct nand_device *nand)
0278 {
0279     struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv;
0280
0281     if (engine_conf) {
0282         nand_ecc_sw_bch_cleanup(nand);
0283         nand_ecc_cleanup_req_tweaking(&engine_conf->req_ctx);
0284         kfree(engine_conf->calc_buf);
0285         kfree(engine_conf->code_buf);
0286         kfree(engine_conf);
0287     }
0288 }
0289 EXPORT_SYMBOL(nand_ecc_sw_bch_cleanup_ctx);
0290
0291 static int nand_ecc_sw_bch_prepare_io_req(struct nand_device *nand,
0292                       struct nand_page_io_req *req)
0293 {
0294     struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv;
0295     struct mtd_info *mtd = nanddev_to_mtd(nand);
0296     int eccsize = nand->ecc.ctx.conf.step_size;
0297     int eccbytes = engine_conf->code_size;
0298     int eccsteps = nand->ecc.ctx.nsteps;
0299     int total = nand->ecc.ctx.total;
0300     u8 *ecccalc = engine_conf->calc_buf;
0301     const u8 *data;
0302     int i;
0303
0304     /* Nothing to do for a raw operation */
0305     if (req->mode == MTD_OPS_RAW)
0306         return 0;
0307
0308     /* This engine does not provide BBM/free OOB bytes protection */
0309     if (!req->datalen)
0310         return 0;
0311
0312     nand_ecc_tweak_req(&engine_conf->req_ctx, req);
0313
0314     /* No more preparation for page read */
0315     if (req->type == NAND_PAGE_READ)
0316         return 0;
0317
0318     /* Preparation for page write: derive the ECC bytes and place them */
0319     for (i = 0, data = req->databuf.out;
0320          eccsteps;
0321          eccsteps--, i += eccbytes, data += eccsize)
0322         nand_ecc_sw_bch_calculate(nand, data, &ecccalc[i]);
0323
0324     return mtd_ooblayout_set_eccbytes(mtd, ecccalc, (void *)req->oobbuf.out,
0325                       0, total);
0326 }
0327
0328 static int nand_ecc_sw_bch_finish_io_req(struct nand_device *nand,
0329                      struct nand_page_io_req *req)
0330 {
0331     struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv;
0332     struct mtd_info *mtd = nanddev_to_mtd(nand);
0333     int eccsize = nand->ecc.ctx.conf.step_size;
0334     int total = nand->ecc.ctx.total;
0335     int eccbytes = engine_conf->code_size;
0336     int eccsteps = nand->ecc.ctx.nsteps;
0337     u8 *ecccalc = engine_conf->calc_buf;
0338     u8 *ecccode = engine_conf->code_buf;
0339     unsigned int max_bitflips = 0;
0340     u8 *data = req->databuf.in;
0341     int i, ret;
0342
0343     /* Nothing to do for a raw operation */
0344     if (req->mode == MTD_OPS_RAW)
0345         return 0;
0346
0347     /* This engine does not provide BBM/free OOB bytes protection */
0348     if (!req->datalen)
0349         return 0;
0350
0351     /* No more preparation for page write */
0352     if (req->type == NAND_PAGE_WRITE) {
0353         nand_ecc_restore_req(&engine_conf->req_ctx, req);
0354         return 0;
0355     }
0356
0357     /* Finish a page read: retrieve the (raw) ECC bytes*/
0358     ret = mtd_ooblayout_get_eccbytes(mtd, ecccode, req->oobbuf.in, 0,
0359                      total);
0360     if (ret)
0361         return ret;
0362
0363     /* Calculate the ECC bytes */
0364     for (i = 0; eccsteps; eccsteps--, i += eccbytes, data += eccsize)
0365         nand_ecc_sw_bch_calculate(nand, data, &ecccalc[i]);
0366
0367     /* Finish a page read: compare and correct */
0368     for (eccsteps = nand->ecc.ctx.nsteps, i = 0, data = req->databuf.in;
0369          eccsteps;
0370          eccsteps--, i += eccbytes, data += eccsize) {
0371         int stat =  nand_ecc_sw_bch_correct(nand, data,
0372                             &ecccode[i],
0373                             &ecccalc[i]);
0374         if (stat < 0) {
0375             mtd->ecc_stats.failed++;
0376         } else {
0377             mtd->ecc_stats.corrected += stat;
0378             max_bitflips = max_t(unsigned int, max_bitflips, stat);
0379         }
0380     }
0381
0382     nand_ecc_restore_req(&engine_conf->req_ctx, req);
0383
0384     return max_bitflips;
0385 }
0386
0387 static struct nand_ecc_engine_ops nand_ecc_sw_bch_engine_ops = {
0388     .init_ctx = nand_ecc_sw_bch_init_ctx,
0389     .cleanup_ctx = nand_ecc_sw_bch_cleanup_ctx,
0390     .prepare_io_req = nand_ecc_sw_bch_prepare_io_req,
0391     .finish_io_req = nand_ecc_sw_bch_finish_io_req,
0392 };
0393
0394 static struct nand_ecc_engine nand_ecc_sw_bch_engine = {
0395     .ops = &nand_ecc_sw_bch_engine_ops,
0396 };
0397
0398 struct nand_ecc_engine *nand_ecc_sw_bch_get_engine(void)
0399 {
0400     return &nand_ecc_sw_bch_engine;
0401 }
0402 EXPORT_SYMBOL(nand_ecc_sw_bch_get_engine);
0403
0404 MODULE_LICENSE("GPL");
0405 MODULE_AUTHOR("Ivan Djelic <ivan.djelic@parrot.com>");
0406 MODULE_DESCRIPTION("NAND software BCH ECC support");