Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0+
0002 /*
0003  * Shared descriptors for aead, skcipher algorithms
0004  *
0005  * Copyright 2016-2019 NXP
0006  */
0007 
0008 #include "compat.h"
0009 #include "desc_constr.h"
0010 #include "caamalg_desc.h"
0011 
0012 /*
0013  * For aead functions, read payload and write payload,
0014  * both of which are specified in req->src and req->dst
0015  */
0016 static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
0017 {
0018     append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
0019     append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
0020                  KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
0021 }
0022 
0023 /* Set DK bit in class 1 operation if shared */
0024 static inline void append_dec_op1(u32 *desc, u32 type)
0025 {
0026     u32 *jump_cmd, *uncond_jump_cmd;
0027 
0028     /* DK bit is valid only for AES */
0029     if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) {
0030         append_operation(desc, type | OP_ALG_AS_INITFINAL |
0031                  OP_ALG_DECRYPT);
0032         return;
0033     }
0034 
0035     jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
0036     append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT);
0037     uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
0038     set_jump_tgt_here(desc, jump_cmd);
0039     append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT |
0040              OP_ALG_AAI_DK);
0041     set_jump_tgt_here(desc, uncond_jump_cmd);
0042 }
0043 
0044 /**
0045  * cnstr_shdsc_aead_null_encap - IPSec ESP encapsulation shared descriptor
0046  *                               (non-protocol) with no (null) encryption.
0047  * @desc: pointer to buffer used for descriptor construction
0048  * @adata: pointer to authentication transform definitions.
0049  *         A split key is required for SEC Era < 6; the size of the split key
0050  *         is specified in this case. Valid algorithm values - one of
0051  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
0052  *         with OP_ALG_AAI_HMAC_PRECOMP.
0053  * @icvsize: integrity check value (ICV) size (truncated or full)
0054  * @era: SEC Era
0055  */
0056 void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
0057                  unsigned int icvsize, int era)
0058 {
0059     u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
0060 
0061     init_sh_desc(desc, HDR_SHARE_SERIAL);
0062 
0063     /* Skip if already shared */
0064     key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
0065                    JUMP_COND_SHRD);
0066     if (era < 6) {
0067         if (adata->key_inline)
0068             append_key_as_imm(desc, adata->key_virt,
0069                       adata->keylen_pad, adata->keylen,
0070                       CLASS_2 | KEY_DEST_MDHA_SPLIT |
0071                       KEY_ENC);
0072         else
0073             append_key(desc, adata->key_dma, adata->keylen,
0074                    CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
0075     } else {
0076         append_proto_dkp(desc, adata);
0077     }
0078     set_jump_tgt_here(desc, key_jump_cmd);
0079 
0080     /* assoclen + cryptlen = seqinlen */
0081     append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
0082 
0083     /* Prepare to read and write cryptlen + assoclen bytes */
0084     append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
0085     append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
0086 
0087     /*
0088      * MOVE_LEN opcode is not available in all SEC HW revisions,
0089      * thus need to do some magic, i.e. self-patch the descriptor
0090      * buffer.
0091      */
0092     read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
0093                     MOVE_DEST_MATH3 |
0094                     (0x6 << MOVE_LEN_SHIFT));
0095     write_move_cmd = append_move(desc, MOVE_SRC_MATH3 |
0096                      MOVE_DEST_DESCBUF |
0097                      MOVE_WAITCOMP |
0098                      (0x8 << MOVE_LEN_SHIFT));
0099 
0100     /* Class 2 operation */
0101     append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
0102              OP_ALG_ENCRYPT);
0103 
0104     /* Read and write cryptlen bytes */
0105     aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
0106 
0107     set_move_tgt_here(desc, read_move_cmd);
0108     set_move_tgt_here(desc, write_move_cmd);
0109     append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
0110     append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
0111             MOVE_AUX_LS);
0112 
0113     /* Write ICV */
0114     append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
0115              LDST_SRCDST_BYTE_CONTEXT);
0116 
0117     print_hex_dump_debug("aead null enc shdesc@" __stringify(__LINE__)": ",
0118                  DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
0119                  1);
0120 }
0121 EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap);
0122 
0123 /**
0124  * cnstr_shdsc_aead_null_decap - IPSec ESP decapsulation shared descriptor
0125  *                               (non-protocol) with no (null) decryption.
0126  * @desc: pointer to buffer used for descriptor construction
0127  * @adata: pointer to authentication transform definitions.
0128  *         A split key is required for SEC Era < 6; the size of the split key
0129  *         is specified in this case. Valid algorithm values - one of
0130  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
0131  *         with OP_ALG_AAI_HMAC_PRECOMP.
0132  * @icvsize: integrity check value (ICV) size (truncated or full)
0133  * @era: SEC Era
0134  */
0135 void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata,
0136                  unsigned int icvsize, int era)
0137 {
0138     u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd, *jump_cmd;
0139 
0140     init_sh_desc(desc, HDR_SHARE_SERIAL);
0141 
0142     /* Skip if already shared */
0143     key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
0144                    JUMP_COND_SHRD);
0145     if (era < 6) {
0146         if (adata->key_inline)
0147             append_key_as_imm(desc, adata->key_virt,
0148                       adata->keylen_pad, adata->keylen,
0149                       CLASS_2 | KEY_DEST_MDHA_SPLIT |
0150                       KEY_ENC);
0151         else
0152             append_key(desc, adata->key_dma, adata->keylen,
0153                    CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
0154     } else {
0155         append_proto_dkp(desc, adata);
0156     }
0157     set_jump_tgt_here(desc, key_jump_cmd);
0158 
0159     /* Class 2 operation */
0160     append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
0161              OP_ALG_DECRYPT | OP_ALG_ICV_ON);
0162 
0163     /* assoclen + cryptlen = seqoutlen */
0164     append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
0165 
0166     /* Prepare to read and write cryptlen + assoclen bytes */
0167     append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
0168     append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
0169 
0170     /*
0171      * MOVE_LEN opcode is not available in all SEC HW revisions,
0172      * thus need to do some magic, i.e. self-patch the descriptor
0173      * buffer.
0174      */
0175     read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
0176                     MOVE_DEST_MATH2 |
0177                     (0x6 << MOVE_LEN_SHIFT));
0178     write_move_cmd = append_move(desc, MOVE_SRC_MATH2 |
0179                      MOVE_DEST_DESCBUF |
0180                      MOVE_WAITCOMP |
0181                      (0x8 << MOVE_LEN_SHIFT));
0182 
0183     /* Read and write cryptlen bytes */
0184     aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
0185 
0186     /*
0187      * Insert a NOP here, since we need at least 4 instructions between
0188      * code patching the descriptor buffer and the location being patched.
0189      */
0190     jump_cmd = append_jump(desc, JUMP_TEST_ALL);
0191     set_jump_tgt_here(desc, jump_cmd);
0192 
0193     set_move_tgt_here(desc, read_move_cmd);
0194     set_move_tgt_here(desc, write_move_cmd);
0195     append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
0196     append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
0197             MOVE_AUX_LS);
0198     append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
0199 
0200     /* Load ICV */
0201     append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
0202                  FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
0203 
0204     print_hex_dump_debug("aead null dec shdesc@" __stringify(__LINE__)": ",
0205                  DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
0206                  1);
0207 }
0208 EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap);
0209 
0210 static void init_sh_desc_key_aead(u32 * const desc,
0211                   struct alginfo * const cdata,
0212                   struct alginfo * const adata,
0213                   const bool is_rfc3686, u32 *nonce, int era)
0214 {
0215     u32 *key_jump_cmd;
0216     unsigned int enckeylen = cdata->keylen;
0217 
0218     /* Note: Context registers are saved. */
0219     init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
0220 
0221     /* Skip if already shared */
0222     key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
0223                    JUMP_COND_SHRD);
0224 
0225     /*
0226      * RFC3686 specific:
0227      *  | key = {AUTH_KEY, ENC_KEY, NONCE}
0228      *  | enckeylen = encryption key size + nonce size
0229      */
0230     if (is_rfc3686)
0231         enckeylen -= CTR_RFC3686_NONCE_SIZE;
0232 
0233     if (era < 6) {
0234         if (adata->key_inline)
0235             append_key_as_imm(desc, adata->key_virt,
0236                       adata->keylen_pad, adata->keylen,
0237                       CLASS_2 | KEY_DEST_MDHA_SPLIT |
0238                       KEY_ENC);
0239         else
0240             append_key(desc, adata->key_dma, adata->keylen,
0241                    CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
0242     } else {
0243         append_proto_dkp(desc, adata);
0244     }
0245 
0246     if (cdata->key_inline)
0247         append_key_as_imm(desc, cdata->key_virt, enckeylen,
0248                   enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
0249     else
0250         append_key(desc, cdata->key_dma, enckeylen, CLASS_1 |
0251                KEY_DEST_CLASS_REG);
0252 
0253     /* Load Counter into CONTEXT1 reg */
0254     if (is_rfc3686) {
0255         append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
0256                    LDST_CLASS_IND_CCB |
0257                    LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
0258         append_move(desc,
0259                 MOVE_SRC_OUTFIFO |
0260                 MOVE_DEST_CLASS1CTX |
0261                 (16 << MOVE_OFFSET_SHIFT) |
0262                 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
0263     }
0264 
0265     set_jump_tgt_here(desc, key_jump_cmd);
0266 }
0267 
0268 /**
0269  * cnstr_shdsc_aead_encap - IPSec ESP encapsulation shared descriptor
0270  *                          (non-protocol).
0271  * @desc: pointer to buffer used for descriptor construction
0272  * @cdata: pointer to block cipher transform definitions
0273  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
0274  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
0275  * @adata: pointer to authentication transform definitions.
0276  *         A split key is required for SEC Era < 6; the size of the split key
0277  *         is specified in this case. Valid algorithm values - one of
0278  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
0279  *         with OP_ALG_AAI_HMAC_PRECOMP.
0280  * @ivsize: initialization vector size
0281  * @icvsize: integrity check value (ICV) size (truncated or full)
0282  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
0283  * @nonce: pointer to rfc3686 nonce
0284  * @ctx1_iv_off: IV offset in CONTEXT1 register
0285  * @is_qi: true when called from caam/qi
0286  * @era: SEC Era
0287  */
0288 void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata,
0289                 struct alginfo *adata, unsigned int ivsize,
0290                 unsigned int icvsize, const bool is_rfc3686,
0291                 u32 *nonce, const u32 ctx1_iv_off, const bool is_qi,
0292                 int era)
0293 {
0294     /* Note: Context registers are saved. */
0295     init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
0296 
0297     /* Class 2 operation */
0298     append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
0299              OP_ALG_ENCRYPT);
0300 
0301     if (is_qi) {
0302         u32 *wait_load_cmd;
0303 
0304         /* REG3 = assoclen */
0305         append_seq_load(desc, 4, LDST_CLASS_DECO |
0306                 LDST_SRCDST_WORD_DECO_MATH3 |
0307                 (4 << LDST_OFFSET_SHIFT));
0308 
0309         wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
0310                         JUMP_COND_CALM | JUMP_COND_NCP |
0311                         JUMP_COND_NOP | JUMP_COND_NIP |
0312                         JUMP_COND_NIFP);
0313         set_jump_tgt_here(desc, wait_load_cmd);
0314 
0315         append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
0316                 LDST_SRCDST_BYTE_CONTEXT |
0317                 (ctx1_iv_off << LDST_OFFSET_SHIFT));
0318     }
0319 
0320     /* Read and write assoclen bytes */
0321     if (is_qi || era < 3) {
0322         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
0323         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
0324     } else {
0325         append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
0326         append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
0327     }
0328 
0329     /* Skip assoc data */
0330     append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
0331 
0332     /* read assoc before reading payload */
0333     append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
0334                       FIFOLDST_VLF);
0335 
0336     /* Load Counter into CONTEXT1 reg */
0337     if (is_rfc3686)
0338         append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
0339                      LDST_SRCDST_BYTE_CONTEXT |
0340                      ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
0341                       LDST_OFFSET_SHIFT));
0342 
0343     /* Class 1 operation */
0344     append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
0345              OP_ALG_ENCRYPT);
0346 
0347     /* Read and write cryptlen bytes */
0348     append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
0349     append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
0350     aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
0351 
0352     /* Write ICV */
0353     append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
0354              LDST_SRCDST_BYTE_CONTEXT);
0355 
0356     print_hex_dump_debug("aead enc shdesc@" __stringify(__LINE__)": ",
0357                  DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
0358                  1);
0359 }
0360 EXPORT_SYMBOL(cnstr_shdsc_aead_encap);
0361 
0362 /**
0363  * cnstr_shdsc_aead_decap - IPSec ESP decapsulation shared descriptor
0364  *                          (non-protocol).
0365  * @desc: pointer to buffer used for descriptor construction
0366  * @cdata: pointer to block cipher transform definitions
0367  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
0368  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
0369  * @adata: pointer to authentication transform definitions.
0370  *         A split key is required for SEC Era < 6; the size of the split key
0371  *         is specified in this case. Valid algorithm values - one of
0372  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
0373  *         with OP_ALG_AAI_HMAC_PRECOMP.
0374  * @ivsize: initialization vector size
0375  * @icvsize: integrity check value (ICV) size (truncated or full)
0376  * @geniv: whether to generate Encrypted Chain IV
0377  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
0378  * @nonce: pointer to rfc3686 nonce
0379  * @ctx1_iv_off: IV offset in CONTEXT1 register
0380  * @is_qi: true when called from caam/qi
0381  * @era: SEC Era
0382  */
0383 void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata,
0384                 struct alginfo *adata, unsigned int ivsize,
0385                 unsigned int icvsize, const bool geniv,
0386                 const bool is_rfc3686, u32 *nonce,
0387                 const u32 ctx1_iv_off, const bool is_qi, int era)
0388 {
0389     /* Note: Context registers are saved. */
0390     init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
0391 
0392     /* Class 2 operation */
0393     append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
0394              OP_ALG_DECRYPT | OP_ALG_ICV_ON);
0395 
0396     if (is_qi) {
0397         u32 *wait_load_cmd;
0398 
0399         /* REG3 = assoclen */
0400         append_seq_load(desc, 4, LDST_CLASS_DECO |
0401                 LDST_SRCDST_WORD_DECO_MATH3 |
0402                 (4 << LDST_OFFSET_SHIFT));
0403 
0404         wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
0405                         JUMP_COND_CALM | JUMP_COND_NCP |
0406                         JUMP_COND_NOP | JUMP_COND_NIP |
0407                         JUMP_COND_NIFP);
0408         set_jump_tgt_here(desc, wait_load_cmd);
0409 
0410         if (!geniv)
0411             append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
0412                     LDST_SRCDST_BYTE_CONTEXT |
0413                     (ctx1_iv_off << LDST_OFFSET_SHIFT));
0414     }
0415 
0416     /* Read and write assoclen bytes */
0417     if (is_qi || era < 3) {
0418         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
0419         if (geniv)
0420             append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM,
0421                         ivsize);
0422         else
0423             append_math_add(desc, VARSEQOUTLEN, ZERO, REG3,
0424                     CAAM_CMD_SZ);
0425     } else {
0426         append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
0427         if (geniv)
0428             append_math_add_imm_u32(desc, VARSEQOUTLEN, DPOVRD, IMM,
0429                         ivsize);
0430         else
0431             append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD,
0432                     CAAM_CMD_SZ);
0433     }
0434 
0435     /* Skip assoc data */
0436     append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
0437 
0438     /* read assoc before reading payload */
0439     append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
0440                  KEY_VLF);
0441 
0442     if (geniv) {
0443         append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
0444                 LDST_SRCDST_BYTE_CONTEXT |
0445                 (ctx1_iv_off << LDST_OFFSET_SHIFT));
0446         append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO |
0447                 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize);
0448     }
0449 
0450     /* Load Counter into CONTEXT1 reg */
0451     if (is_rfc3686)
0452         append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
0453                      LDST_SRCDST_BYTE_CONTEXT |
0454                      ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
0455                       LDST_OFFSET_SHIFT));
0456 
0457     /* Choose operation */
0458     if (ctx1_iv_off)
0459         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
0460                  OP_ALG_DECRYPT);
0461     else
0462         append_dec_op1(desc, cdata->algtype);
0463 
0464     /* Read and write cryptlen bytes */
0465     append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
0466     append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
0467     aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
0468 
0469     /* Load ICV */
0470     append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
0471                  FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
0472 
0473     print_hex_dump_debug("aead dec shdesc@" __stringify(__LINE__)": ",
0474                  DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
0475                  1);
0476 }
0477 EXPORT_SYMBOL(cnstr_shdsc_aead_decap);
0478 
0479 /**
0480  * cnstr_shdsc_aead_givencap - IPSec ESP encapsulation shared descriptor
0481  *                             (non-protocol) with HW-generated initialization
0482  *                             vector.
0483  * @desc: pointer to buffer used for descriptor construction
0484  * @cdata: pointer to block cipher transform definitions
0485  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
0486  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
0487  * @adata: pointer to authentication transform definitions.
0488  *         A split key is required for SEC Era < 6; the size of the split key
0489  *         is specified in this case. Valid algorithm values - one of
0490  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
0491  *         with OP_ALG_AAI_HMAC_PRECOMP.
0492  * @ivsize: initialization vector size
0493  * @icvsize: integrity check value (ICV) size (truncated or full)
0494  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
0495  * @nonce: pointer to rfc3686 nonce
0496  * @ctx1_iv_off: IV offset in CONTEXT1 register
0497  * @is_qi: true when called from caam/qi
0498  * @era: SEC Era
0499  */
0500 void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata,
0501                    struct alginfo *adata, unsigned int ivsize,
0502                    unsigned int icvsize, const bool is_rfc3686,
0503                    u32 *nonce, const u32 ctx1_iv_off,
0504                    const bool is_qi, int era)
0505 {
0506     u32 geniv, moveiv;
0507     u32 *wait_cmd;
0508 
0509     /* Note: Context registers are saved. */
0510     init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
0511 
0512     if (is_qi) {
0513         u32 *wait_load_cmd;
0514 
0515         /* REG3 = assoclen */
0516         append_seq_load(desc, 4, LDST_CLASS_DECO |
0517                 LDST_SRCDST_WORD_DECO_MATH3 |
0518                 (4 << LDST_OFFSET_SHIFT));
0519 
0520         wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
0521                         JUMP_COND_CALM | JUMP_COND_NCP |
0522                         JUMP_COND_NOP | JUMP_COND_NIP |
0523                         JUMP_COND_NIFP);
0524         set_jump_tgt_here(desc, wait_load_cmd);
0525     }
0526 
0527     if (is_rfc3686) {
0528         if (is_qi)
0529             append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
0530                     LDST_SRCDST_BYTE_CONTEXT |
0531                     (ctx1_iv_off << LDST_OFFSET_SHIFT));
0532 
0533         goto copy_iv;
0534     }
0535 
0536     /* Generate IV */
0537     geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
0538         NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
0539         NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT);
0540     append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
0541                 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
0542     append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
0543     append_move(desc, MOVE_WAITCOMP |
0544             MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX |
0545             (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
0546             (ivsize << MOVE_LEN_SHIFT));
0547     append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
0548 
0549 copy_iv:
0550     /* Copy IV to class 1 context */
0551     append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO |
0552             (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
0553             (ivsize << MOVE_LEN_SHIFT));
0554 
0555     /* Return to encryption */
0556     append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
0557              OP_ALG_ENCRYPT);
0558 
0559     /* Read and write assoclen bytes */
0560     if (is_qi || era < 3) {
0561         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
0562         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
0563     } else {
0564         append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
0565         append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
0566     }
0567 
0568     /* Skip assoc data */
0569     append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
0570 
0571     /* read assoc before reading payload */
0572     append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
0573                  KEY_VLF);
0574 
0575     /* Copy iv from outfifo to class 2 fifo */
0576     moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
0577          NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT);
0578     append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
0579                 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
0580     append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB |
0581                 LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
0582 
0583     /* Load Counter into CONTEXT1 reg */
0584     if (is_rfc3686)
0585         append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
0586                      LDST_SRCDST_BYTE_CONTEXT |
0587                      ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
0588                       LDST_OFFSET_SHIFT));
0589 
0590     /* Class 1 operation */
0591     append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
0592              OP_ALG_ENCRYPT);
0593 
0594     /* Will write ivsize + cryptlen */
0595     append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
0596 
0597     /* Not need to reload iv */
0598     append_seq_fifo_load(desc, ivsize,
0599                  FIFOLD_CLASS_SKIP);
0600 
0601     /* Will read cryptlen */
0602     append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
0603 
0604     /*
0605      * Wait for IV transfer (ofifo -> class2) to finish before starting
0606      * ciphertext transfer (ofifo -> external memory).
0607      */
0608     wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NIFP);
0609     set_jump_tgt_here(desc, wait_cmd);
0610 
0611     append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF |
0612                  FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH);
0613     append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
0614 
0615     /* Write ICV */
0616     append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
0617              LDST_SRCDST_BYTE_CONTEXT);
0618 
0619     print_hex_dump_debug("aead givenc shdesc@" __stringify(__LINE__)": ",
0620                  DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
0621                  1);
0622 }
0623 EXPORT_SYMBOL(cnstr_shdsc_aead_givencap);
0624 
0625 /**
0626  * cnstr_shdsc_gcm_encap - gcm encapsulation shared descriptor
0627  * @desc: pointer to buffer used for descriptor construction
0628  * @cdata: pointer to block cipher transform definitions
0629  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
0630  * @ivsize: initialization vector size
0631  * @icvsize: integrity check value (ICV) size (truncated or full)
0632  * @is_qi: true when called from caam/qi
0633  */
0634 void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata,
0635                unsigned int ivsize, unsigned int icvsize,
0636                const bool is_qi)
0637 {
0638     u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1,
0639         *zero_assoc_jump_cmd2;
0640 
0641     init_sh_desc(desc, HDR_SHARE_SERIAL);
0642 
0643     /* skip key loading if they are loaded due to sharing */
0644     key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
0645                    JUMP_COND_SHRD);
0646     if (cdata->key_inline)
0647         append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
0648                   cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
0649     else
0650         append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
0651                KEY_DEST_CLASS_REG);
0652     set_jump_tgt_here(desc, key_jump_cmd);
0653 
0654     /* class 1 operation */
0655     append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
0656              OP_ALG_ENCRYPT);
0657 
0658     if (is_qi) {
0659         u32 *wait_load_cmd;
0660 
0661         /* REG3 = assoclen */
0662         append_seq_load(desc, 4, LDST_CLASS_DECO |
0663                 LDST_SRCDST_WORD_DECO_MATH3 |
0664                 (4 << LDST_OFFSET_SHIFT));
0665 
0666         wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
0667                         JUMP_COND_CALM | JUMP_COND_NCP |
0668                         JUMP_COND_NOP | JUMP_COND_NIP |
0669                         JUMP_COND_NIFP);
0670         set_jump_tgt_here(desc, wait_load_cmd);
0671 
0672         append_math_sub_imm_u32(desc, VARSEQOUTLEN, SEQINLEN, IMM,
0673                     ivsize);
0674     } else {
0675         append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0,
0676                 CAAM_CMD_SZ);
0677     }
0678 
0679     /* if assoclen + cryptlen is ZERO, skip to ICV write */
0680     zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
0681                          JUMP_COND_MATH_Z);
0682 
0683     if (is_qi)
0684         append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
0685                      FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
0686 
0687     /* if assoclen is ZERO, skip reading the assoc data */
0688     append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
0689     zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
0690                        JUMP_COND_MATH_Z);
0691 
0692     append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
0693 
0694     /* skip assoc data */
0695     append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
0696 
0697     /* cryptlen = seqinlen - assoclen */
0698     append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
0699 
0700     /* if cryptlen is ZERO jump to zero-payload commands */
0701     zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
0702                         JUMP_COND_MATH_Z);
0703 
0704     /* read assoc data */
0705     append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
0706                  FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
0707     set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
0708 
0709     append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
0710 
0711     /* write encrypted data */
0712     append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
0713 
0714     /* read payload data */
0715     append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
0716                  FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
0717 
0718     /* jump to ICV writing */
0719     if (is_qi)
0720         append_jump(desc, JUMP_TEST_ALL | 4);
0721     else
0722         append_jump(desc, JUMP_TEST_ALL | 2);
0723 
0724     /* zero-payload commands */
0725     set_jump_tgt_here(desc, zero_payload_jump_cmd);
0726 
0727     /* read assoc data */
0728     append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
0729                  FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
0730     if (is_qi)
0731         /* jump to ICV writing */
0732         append_jump(desc, JUMP_TEST_ALL | 2);
0733 
0734     /* There is no input data */
0735     set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
0736 
0737     if (is_qi)
0738         append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
0739                      FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 |
0740                      FIFOLD_TYPE_LAST1);
0741 
0742     /* write ICV */
0743     append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
0744              LDST_SRCDST_BYTE_CONTEXT);
0745 
0746     print_hex_dump_debug("gcm enc shdesc@" __stringify(__LINE__)": ",
0747                  DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
0748                  1);
0749 }
0750 EXPORT_SYMBOL(cnstr_shdsc_gcm_encap);
0751 
0752 /**
0753  * cnstr_shdsc_gcm_decap - gcm decapsulation shared descriptor
0754  * @desc: pointer to buffer used for descriptor construction
0755  * @cdata: pointer to block cipher transform definitions
0756  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
0757  * @ivsize: initialization vector size
0758  * @icvsize: integrity check value (ICV) size (truncated or full)
0759  * @is_qi: true when called from caam/qi
0760  */
0761 void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata,
0762                unsigned int ivsize, unsigned int icvsize,
0763                const bool is_qi)
0764 {
0765     u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1;
0766 
0767     init_sh_desc(desc, HDR_SHARE_SERIAL);
0768 
0769     /* skip key loading if they are loaded due to sharing */
0770     key_jump_cmd = append_jump(desc, JUMP_JSL |
0771                    JUMP_TEST_ALL | JUMP_COND_SHRD);
0772     if (cdata->key_inline)
0773         append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
0774                   cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
0775     else
0776         append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
0777                KEY_DEST_CLASS_REG);
0778     set_jump_tgt_here(desc, key_jump_cmd);
0779 
0780     /* class 1 operation */
0781     append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
0782              OP_ALG_DECRYPT | OP_ALG_ICV_ON);
0783 
0784     if (is_qi) {
0785         u32 *wait_load_cmd;
0786 
0787         /* REG3 = assoclen */
0788         append_seq_load(desc, 4, LDST_CLASS_DECO |
0789                 LDST_SRCDST_WORD_DECO_MATH3 |
0790                 (4 << LDST_OFFSET_SHIFT));
0791 
0792         wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
0793                         JUMP_COND_CALM | JUMP_COND_NCP |
0794                         JUMP_COND_NOP | JUMP_COND_NIP |
0795                         JUMP_COND_NIFP);
0796         set_jump_tgt_here(desc, wait_load_cmd);
0797 
0798         append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
0799                      FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
0800     }
0801 
0802     /* if assoclen is ZERO, skip reading the assoc data */
0803     append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
0804     zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
0805                          JUMP_COND_MATH_Z);
0806 
0807     append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
0808 
0809     /* skip assoc data */
0810     append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
0811 
0812     /* read assoc data */
0813     append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
0814                  FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
0815 
0816     set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
0817 
0818     /* cryptlen = seqoutlen - assoclen */
0819     append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
0820 
0821     /* jump to zero-payload command if cryptlen is zero */
0822     zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
0823                         JUMP_COND_MATH_Z);
0824 
0825     append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
0826 
0827     /* store encrypted data */
0828     append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
0829 
0830     /* read payload data */
0831     append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
0832                  FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
0833 
0834     /* zero-payload command */
0835     set_jump_tgt_here(desc, zero_payload_jump_cmd);
0836 
0837     /* read ICV */
0838     append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
0839                  FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
0840 
0841     print_hex_dump_debug("gcm dec shdesc@" __stringify(__LINE__)": ",
0842                  DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
0843                  1);
0844 }
0845 EXPORT_SYMBOL(cnstr_shdsc_gcm_decap);
0846 
0847 /**
0848  * cnstr_shdsc_rfc4106_encap - IPSec ESP gcm encapsulation shared descriptor
0849  *                             (non-protocol).
0850  * @desc: pointer to buffer used for descriptor construction
0851  * @cdata: pointer to block cipher transform definitions
0852  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
0853  * @ivsize: initialization vector size
0854  * @icvsize: integrity check value (ICV) size (truncated or full)
0855  * @is_qi: true when called from caam/qi
0856  *
0857  * Input sequence: AAD | PTXT
0858  * Output sequence: AAD | CTXT | ICV
0859  * AAD length (assoclen), which includes the IV length, is available in Math3.
0860  */
0861 void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata,
0862                    unsigned int ivsize, unsigned int icvsize,
0863                    const bool is_qi)
0864 {
0865     u32 *key_jump_cmd, *zero_cryptlen_jump_cmd, *skip_instructions;
0866     init_sh_desc(desc, HDR_SHARE_SERIAL);
0867 
0868     /* Skip key loading if it is loaded due to sharing */
0869     key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
0870                    JUMP_COND_SHRD);
0871     if (cdata->key_inline)
0872         append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
0873                   cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
0874     else
0875         append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
0876                KEY_DEST_CLASS_REG);
0877     set_jump_tgt_here(desc, key_jump_cmd);
0878 
0879     /* Class 1 operation */
0880     append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
0881              OP_ALG_ENCRYPT);
0882 
0883     if (is_qi) {
0884         u32 *wait_load_cmd;
0885 
0886         /* REG3 = assoclen */
0887         append_seq_load(desc, 4, LDST_CLASS_DECO |
0888                 LDST_SRCDST_WORD_DECO_MATH3 |
0889                 (4 << LDST_OFFSET_SHIFT));
0890 
0891         wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
0892                         JUMP_COND_CALM | JUMP_COND_NCP |
0893                         JUMP_COND_NOP | JUMP_COND_NIP |
0894                         JUMP_COND_NIFP);
0895         set_jump_tgt_here(desc, wait_load_cmd);
0896 
0897         /* Read salt and IV */
0898         append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
0899                     cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
0900                     FIFOLD_TYPE_IV);
0901         append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
0902                      FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
0903     }
0904 
0905     append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
0906     append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
0907 
0908     /* Skip AAD */
0909     append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
0910 
0911     /* Read cryptlen and set this value into VARSEQOUTLEN */
0912     append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
0913 
0914     /* If cryptlen is ZERO jump to AAD command */
0915     zero_cryptlen_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
0916                         JUMP_COND_MATH_Z);
0917 
0918     /* Read AAD data */
0919     append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
0920                  FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
0921 
0922     /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
0923     append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA);
0924 
0925     /* Skip IV */
0926     append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
0927     append_math_add(desc, VARSEQINLEN, VARSEQOUTLEN, REG0, CAAM_CMD_SZ);
0928 
0929     /* Write encrypted data */
0930     append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
0931 
0932     /* Read payload data */
0933     append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
0934                  FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
0935 
0936     /* Jump instructions to avoid double reading of AAD */
0937     skip_instructions = append_jump(desc, JUMP_TEST_ALL);
0938 
0939     /* There is no input data, cryptlen = 0 */
0940     set_jump_tgt_here(desc, zero_cryptlen_jump_cmd);
0941 
0942     /* Read AAD */
0943     append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
0944                  FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
0945 
0946     set_jump_tgt_here(desc, skip_instructions);
0947 
0948     /* Write ICV */
0949     append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
0950              LDST_SRCDST_BYTE_CONTEXT);
0951 
0952     print_hex_dump_debug("rfc4106 enc shdesc@" __stringify(__LINE__)": ",
0953                  DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
0954                  1);
0955 }
0956 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap);
0957 
0958 /**
0959  * cnstr_shdsc_rfc4106_decap - IPSec ESP gcm decapsulation shared descriptor
0960  *                             (non-protocol).
0961  * @desc: pointer to buffer used for descriptor construction
0962  * @cdata: pointer to block cipher transform definitions
0963  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
0964  * @ivsize: initialization vector size
0965  * @icvsize: integrity check value (ICV) size (truncated or full)
0966  * @is_qi: true when called from caam/qi
0967  */
0968 void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata,
0969                    unsigned int ivsize, unsigned int icvsize,
0970                    const bool is_qi)
0971 {
0972     u32 *key_jump_cmd;
0973 
0974     init_sh_desc(desc, HDR_SHARE_SERIAL);
0975 
0976     /* Skip key loading if it is loaded due to sharing */
0977     key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
0978                    JUMP_COND_SHRD);
0979     if (cdata->key_inline)
0980         append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
0981                   cdata->keylen, CLASS_1 |
0982                   KEY_DEST_CLASS_REG);
0983     else
0984         append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
0985                KEY_DEST_CLASS_REG);
0986     set_jump_tgt_here(desc, key_jump_cmd);
0987 
0988     /* Class 1 operation */
0989     append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
0990              OP_ALG_DECRYPT | OP_ALG_ICV_ON);
0991 
0992     if (is_qi) {
0993         u32 *wait_load_cmd;
0994 
0995         /* REG3 = assoclen */
0996         append_seq_load(desc, 4, LDST_CLASS_DECO |
0997                 LDST_SRCDST_WORD_DECO_MATH3 |
0998                 (4 << LDST_OFFSET_SHIFT));
0999 
1000         wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1001                         JUMP_COND_CALM | JUMP_COND_NCP |
1002                         JUMP_COND_NOP | JUMP_COND_NIP |
1003                         JUMP_COND_NIFP);
1004         set_jump_tgt_here(desc, wait_load_cmd);
1005 
1006         /* Read salt and IV */
1007         append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1008                     cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1009                     FIFOLD_TYPE_IV);
1010         append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1011                      FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1012     }
1013 
1014     append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
1015     append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1016 
1017     /* Read assoc data */
1018     append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1019                  FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
1020 
1021     /* Skip IV */
1022     append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
1023 
1024     /* Will read cryptlen bytes */
1025     append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ);
1026 
1027     /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
1028     append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
1029 
1030     /* Skip assoc data */
1031     append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
1032 
1033     /* Will write cryptlen bytes */
1034     append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1035 
1036     /* Store payload data */
1037     append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1038 
1039     /* Read encrypted data */
1040     append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1041                  FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
1042 
1043     /* Read ICV */
1044     append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
1045                  FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1046 
1047     print_hex_dump_debug("rfc4106 dec shdesc@" __stringify(__LINE__)": ",
1048                  DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1049                  1);
1050 }
1051 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap);
1052 
1053 /**
1054  * cnstr_shdsc_rfc4543_encap - IPSec ESP gmac encapsulation shared descriptor
1055  *                             (non-protocol).
1056  * @desc: pointer to buffer used for descriptor construction
1057  * @cdata: pointer to block cipher transform definitions
1058  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
1059  * @ivsize: initialization vector size
1060  * @icvsize: integrity check value (ICV) size (truncated or full)
1061  * @is_qi: true when called from caam/qi
1062  */
1063 void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata,
1064                    unsigned int ivsize, unsigned int icvsize,
1065                    const bool is_qi)
1066 {
1067     u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
1068 
1069     init_sh_desc(desc, HDR_SHARE_SERIAL);
1070 
1071     /* Skip key loading if it is loaded due to sharing */
1072     key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1073                    JUMP_COND_SHRD);
1074     if (cdata->key_inline)
1075         append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1076                   cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1077     else
1078         append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
1079                KEY_DEST_CLASS_REG);
1080     set_jump_tgt_here(desc, key_jump_cmd);
1081 
1082     /* Class 1 operation */
1083     append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1084              OP_ALG_ENCRYPT);
1085 
1086     if (is_qi) {
1087         /* assoclen is not needed, skip it */
1088         append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP);
1089 
1090         /* Read salt and IV */
1091         append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1092                     cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1093                     FIFOLD_TYPE_IV);
1094         append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1095                      FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1096     }
1097 
1098     /* assoclen + cryptlen = seqinlen */
1099     append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
1100 
1101     /*
1102      * MOVE_LEN opcode is not available in all SEC HW revisions,
1103      * thus need to do some magic, i.e. self-patch the descriptor
1104      * buffer.
1105      */
1106     read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1107                     (0x6 << MOVE_LEN_SHIFT));
1108     write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1109                      (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
1110 
1111     /* Will read assoclen + cryptlen bytes */
1112     append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1113 
1114     /* Will write assoclen + cryptlen bytes */
1115     append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1116 
1117     /* Read and write assoclen + cryptlen bytes */
1118     aead_append_src_dst(desc, FIFOLD_TYPE_AAD);
1119 
1120     set_move_tgt_here(desc, read_move_cmd);
1121     set_move_tgt_here(desc, write_move_cmd);
1122     append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1123     /* Move payload data to OFIFO */
1124     append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1125 
1126     /* Write ICV */
1127     append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
1128              LDST_SRCDST_BYTE_CONTEXT);
1129 
1130     print_hex_dump_debug("rfc4543 enc shdesc@" __stringify(__LINE__)": ",
1131                  DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1132                  1);
1133 }
1134 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap);
1135 
1136 /**
1137  * cnstr_shdsc_rfc4543_decap - IPSec ESP gmac decapsulation shared descriptor
1138  *                             (non-protocol).
1139  * @desc: pointer to buffer used for descriptor construction
1140  * @cdata: pointer to block cipher transform definitions
1141  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
1142  * @ivsize: initialization vector size
1143  * @icvsize: integrity check value (ICV) size (truncated or full)
1144  * @is_qi: true when called from caam/qi
1145  */
1146 void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata,
1147                    unsigned int ivsize, unsigned int icvsize,
1148                    const bool is_qi)
1149 {
1150     u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
1151 
1152     init_sh_desc(desc, HDR_SHARE_SERIAL);
1153 
1154     /* Skip key loading if it is loaded due to sharing */
1155     key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1156                    JUMP_COND_SHRD);
1157     if (cdata->key_inline)
1158         append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1159                   cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1160     else
1161         append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
1162                KEY_DEST_CLASS_REG);
1163     set_jump_tgt_here(desc, key_jump_cmd);
1164 
1165     /* Class 1 operation */
1166     append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1167              OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1168 
1169     if (is_qi) {
1170         /* assoclen is not needed, skip it */
1171         append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP);
1172 
1173         /* Read salt and IV */
1174         append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1175                     cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1176                     FIFOLD_TYPE_IV);
1177         append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1178                      FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1179     }
1180 
1181     /* assoclen + cryptlen = seqoutlen */
1182     append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1183 
1184     /*
1185      * MOVE_LEN opcode is not available in all SEC HW revisions,
1186      * thus need to do some magic, i.e. self-patch the descriptor
1187      * buffer.
1188      */
1189     read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1190                     (0x6 << MOVE_LEN_SHIFT));
1191     write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1192                      (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
1193 
1194     /* Will read assoclen + cryptlen bytes */
1195     append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1196 
1197     /* Will write assoclen + cryptlen bytes */
1198     append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1199 
1200     /* Store payload data */
1201     append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1202 
1203     /* In-snoop assoclen + cryptlen data */
1204     append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF |
1205                  FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1);
1206 
1207     set_move_tgt_here(desc, read_move_cmd);
1208     set_move_tgt_here(desc, write_move_cmd);
1209     append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1210     /* Move payload data to OFIFO */
1211     append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1212     append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1213 
1214     /* Read ICV */
1215     append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
1216                  FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1217 
1218     print_hex_dump_debug("rfc4543 dec shdesc@" __stringify(__LINE__)": ",
1219                  DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1220                  1);
1221 }
1222 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap);
1223 
1224 /**
1225  * cnstr_shdsc_chachapoly - Chacha20 + Poly1305 generic AEAD (rfc7539) and
1226  *                          IPsec ESP (rfc7634, a.k.a. rfc7539esp) shared
1227  *                          descriptor (non-protocol).
1228  * @desc: pointer to buffer used for descriptor construction
1229  * @cdata: pointer to block cipher transform definitions
1230  *         Valid algorithm values - OP_ALG_ALGSEL_CHACHA20 ANDed with
1231  *         OP_ALG_AAI_AEAD.
1232  * @adata: pointer to authentication transform definitions
1233  *         Valid algorithm values - OP_ALG_ALGSEL_POLY1305 ANDed with
1234  *         OP_ALG_AAI_AEAD.
1235  * @ivsize: initialization vector size
1236  * @icvsize: integrity check value (ICV) size (truncated or full)
1237  * @encap: true if encapsulation, false if decapsulation
1238  * @is_qi: true when called from caam/qi
1239  */
1240 void cnstr_shdsc_chachapoly(u32 * const desc, struct alginfo *cdata,
1241                 struct alginfo *adata, unsigned int ivsize,
1242                 unsigned int icvsize, const bool encap,
1243                 const bool is_qi)
1244 {
1245     u32 *key_jump_cmd, *wait_cmd;
1246     u32 nfifo;
1247     const bool is_ipsec = (ivsize != CHACHAPOLY_IV_SIZE);
1248 
1249     /* Note: Context registers are saved. */
1250     init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1251 
1252     /* skip key loading if they are loaded due to sharing */
1253     key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1254                    JUMP_COND_SHRD);
1255 
1256     append_key_as_imm(desc, cdata->key_virt, cdata->keylen, cdata->keylen,
1257               CLASS_1 | KEY_DEST_CLASS_REG);
1258 
1259     /* For IPsec load the salt from keymat in the context register */
1260     if (is_ipsec)
1261         append_load_as_imm(desc, cdata->key_virt + cdata->keylen, 4,
1262                    LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT |
1263                    4 << LDST_OFFSET_SHIFT);
1264 
1265     set_jump_tgt_here(desc, key_jump_cmd);
1266 
1267     /* Class 2 and 1 operations: Poly & ChaCha */
1268     if (encap) {
1269         append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
1270                  OP_ALG_ENCRYPT);
1271         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1272                  OP_ALG_ENCRYPT);
1273     } else {
1274         append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
1275                  OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1276         append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1277                  OP_ALG_DECRYPT);
1278     }
1279 
1280     if (is_qi) {
1281         u32 *wait_load_cmd;
1282         u32 ctx1_iv_off = is_ipsec ? 8 : 4;
1283 
1284         /* REG3 = assoclen */
1285         append_seq_load(desc, 4, LDST_CLASS_DECO |
1286                 LDST_SRCDST_WORD_DECO_MATH3 |
1287                 4 << LDST_OFFSET_SHIFT);
1288 
1289         wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1290                         JUMP_COND_CALM | JUMP_COND_NCP |
1291                         JUMP_COND_NOP | JUMP_COND_NIP |
1292                         JUMP_COND_NIFP);
1293         set_jump_tgt_here(desc, wait_load_cmd);
1294 
1295         append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
1296                 LDST_SRCDST_BYTE_CONTEXT |
1297                 ctx1_iv_off << LDST_OFFSET_SHIFT);
1298     }
1299 
1300     /*
1301      * MAGIC with NFIFO
1302      * Read associated data from the input and send them to class1 and
1303      * class2 alignment blocks. From class1 send data to output fifo and
1304      * then write it to memory since we don't need to encrypt AD.
1305      */
1306     nfifo = NFIFOENTRY_DEST_BOTH | NFIFOENTRY_FC1 | NFIFOENTRY_FC2 |
1307         NFIFOENTRY_DTYPE_POLY | NFIFOENTRY_BND;
1308     append_load_imm_u32(desc, nfifo, LDST_CLASS_IND_CCB |
1309                 LDST_SRCDST_WORD_INFO_FIFO_SM | LDLEN_MATH3);
1310 
1311     append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
1312     append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1313     append_seq_fifo_load(desc, 0, FIFOLD_TYPE_NOINFOFIFO |
1314                  FIFOLD_CLASS_CLASS1 | LDST_VLF);
1315     append_move_len(desc, MOVE_AUX_LS | MOVE_SRC_AUX_ABLK |
1316             MOVE_DEST_OUTFIFO | MOVELEN_MRSEL_MATH3);
1317     append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | LDST_VLF);
1318 
1319     /* IPsec - copy IV at the output */
1320     if (is_ipsec)
1321         append_seq_fifo_store(desc, ivsize, FIFOST_TYPE_METADATA |
1322                       0x2 << 25);
1323 
1324     wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TYPE_LOCAL |
1325                    JUMP_COND_NOP | JUMP_TEST_ALL);
1326     set_jump_tgt_here(desc, wait_cmd);
1327 
1328     if (encap) {
1329         /* Read and write cryptlen bytes */
1330         append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1331         append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0,
1332                 CAAM_CMD_SZ);
1333         aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
1334 
1335         /* Write ICV */
1336         append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
1337                  LDST_SRCDST_BYTE_CONTEXT);
1338     } else {
1339         /* Read and write cryptlen bytes */
1340         append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0,
1341                 CAAM_CMD_SZ);
1342         append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0,
1343                 CAAM_CMD_SZ);
1344         aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
1345 
1346         /* Load ICV for verification */
1347         append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
1348                      FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
1349     }
1350 
1351     print_hex_dump_debug("chachapoly shdesc@" __stringify(__LINE__)": ",
1352                  DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1353                  1);
1354 }
1355 EXPORT_SYMBOL(cnstr_shdsc_chachapoly);
1356 
1357 /* For skcipher encrypt and decrypt, read from req->src and write to req->dst */
1358 static inline void skcipher_append_src_dst(u32 *desc)
1359 {
1360     append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1361     append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1362     append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 |
1363                  KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
1364     append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
1365 }
1366 
1367 /**
1368  * cnstr_shdsc_skcipher_encap - skcipher encapsulation shared descriptor
1369  * @desc: pointer to buffer used for descriptor construction
1370  * @cdata: pointer to block cipher transform definitions
1371  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1372  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128
1373  *                                - OP_ALG_ALGSEL_CHACHA20
1374  * @ivsize: initialization vector size
1375  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1376  * @ctx1_iv_off: IV offset in CONTEXT1 register
1377  */
1378 void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata,
1379                 unsigned int ivsize, const bool is_rfc3686,
1380                 const u32 ctx1_iv_off)
1381 {
1382     u32 *key_jump_cmd;
1383     u32 options = cdata->algtype | OP_ALG_AS_INIT | OP_ALG_ENCRYPT;
1384     bool is_chacha20 = ((cdata->algtype & OP_ALG_ALGSEL_MASK) ==
1385                 OP_ALG_ALGSEL_CHACHA20);
1386 
1387     init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1388     /* Skip if already shared */
1389     key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1390                    JUMP_COND_SHRD);
1391 
1392     /* Load class1 key only */
1393     append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1394               cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1395 
1396     /* Load nonce into CONTEXT1 reg */
1397     if (is_rfc3686) {
1398         const u8 *nonce = cdata->key_virt + cdata->keylen;
1399 
1400         append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1401                    LDST_CLASS_IND_CCB |
1402                    LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1403         append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1404                 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1405                 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1406     }
1407 
1408     set_jump_tgt_here(desc, key_jump_cmd);
1409 
1410     /* Load IV, if there is one */
1411     if (ivsize)
1412         append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1413                 LDST_CLASS_1_CCB | (ctx1_iv_off <<
1414                 LDST_OFFSET_SHIFT));
1415 
1416     /* Load counter into CONTEXT1 reg */
1417     if (is_rfc3686)
1418         append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1419                      LDST_SRCDST_BYTE_CONTEXT |
1420                      ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1421                       LDST_OFFSET_SHIFT));
1422 
1423     /* Load operation */
1424     if (is_chacha20)
1425         options |= OP_ALG_AS_FINALIZE;
1426     append_operation(desc, options);
1427 
1428     /* Perform operation */
1429     skcipher_append_src_dst(desc);
1430 
1431     /* Store IV */
1432     if (!is_chacha20 && ivsize)
1433         append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1434                  LDST_CLASS_1_CCB | (ctx1_iv_off <<
1435                  LDST_OFFSET_SHIFT));
1436 
1437     print_hex_dump_debug("skcipher enc shdesc@" __stringify(__LINE__)": ",
1438                  DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1439                  1);
1440 }
1441 EXPORT_SYMBOL(cnstr_shdsc_skcipher_encap);
1442 
1443 /**
1444  * cnstr_shdsc_skcipher_decap - skcipher decapsulation shared descriptor
1445  * @desc: pointer to buffer used for descriptor construction
1446  * @cdata: pointer to block cipher transform definitions
1447  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1448  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128
1449  *                                - OP_ALG_ALGSEL_CHACHA20
1450  * @ivsize: initialization vector size
1451  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1452  * @ctx1_iv_off: IV offset in CONTEXT1 register
1453  */
1454 void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata,
1455                 unsigned int ivsize, const bool is_rfc3686,
1456                 const u32 ctx1_iv_off)
1457 {
1458     u32 *key_jump_cmd;
1459     bool is_chacha20 = ((cdata->algtype & OP_ALG_ALGSEL_MASK) ==
1460                 OP_ALG_ALGSEL_CHACHA20);
1461 
1462     init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1463     /* Skip if already shared */
1464     key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1465                    JUMP_COND_SHRD);
1466 
1467     /* Load class1 key only */
1468     append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1469               cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1470 
1471     /* Load nonce into CONTEXT1 reg */
1472     if (is_rfc3686) {
1473         const u8 *nonce = cdata->key_virt + cdata->keylen;
1474 
1475         append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1476                    LDST_CLASS_IND_CCB |
1477                    LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1478         append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1479                 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1480                 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1481     }
1482 
1483     set_jump_tgt_here(desc, key_jump_cmd);
1484 
1485     /* Load IV, if there is one */
1486     if (ivsize)
1487         append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1488                 LDST_CLASS_1_CCB | (ctx1_iv_off <<
1489                 LDST_OFFSET_SHIFT));
1490 
1491     /* Load counter into CONTEXT1 reg */
1492     if (is_rfc3686)
1493         append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1494                      LDST_SRCDST_BYTE_CONTEXT |
1495                      ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1496                       LDST_OFFSET_SHIFT));
1497 
1498     /* Choose operation */
1499     if (ctx1_iv_off)
1500         append_operation(desc, cdata->algtype | OP_ALG_AS_INIT |
1501                  OP_ALG_DECRYPT);
1502     else
1503         append_dec_op1(desc, cdata->algtype);
1504 
1505     /* Perform operation */
1506     skcipher_append_src_dst(desc);
1507 
1508     /* Store IV */
1509     if (!is_chacha20 && ivsize)
1510         append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1511                  LDST_CLASS_1_CCB | (ctx1_iv_off <<
1512                  LDST_OFFSET_SHIFT));
1513 
1514     print_hex_dump_debug("skcipher dec shdesc@" __stringify(__LINE__)": ",
1515                  DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1516                  1);
1517 }
1518 EXPORT_SYMBOL(cnstr_shdsc_skcipher_decap);
1519 
1520 /**
1521  * cnstr_shdsc_xts_skcipher_encap - xts skcipher encapsulation shared descriptor
1522  * @desc: pointer to buffer used for descriptor construction
1523  * @cdata: pointer to block cipher transform definitions
1524  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
1525  */
1526 void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata)
1527 {
1528     /*
1529      * Set sector size to a big value, practically disabling
1530      * sector size segmentation in xts implementation. We cannot
1531      * take full advantage of this HW feature with existing
1532      * crypto API / dm-crypt SW architecture.
1533      */
1534     __be64 sector_size = cpu_to_be64(BIT(15));
1535     u32 *key_jump_cmd;
1536 
1537     init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1538     /* Skip if already shared */
1539     key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1540                    JUMP_COND_SHRD);
1541 
1542     /* Load class1 keys only */
1543     append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1544               cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1545 
1546     /* Load sector size with index 40 bytes (0x28) */
1547     append_load_as_imm(desc, (void *)&sector_size, 8, LDST_CLASS_1_CCB |
1548                LDST_SRCDST_BYTE_CONTEXT |
1549                (0x28 << LDST_OFFSET_SHIFT));
1550 
1551     set_jump_tgt_here(desc, key_jump_cmd);
1552 
1553     /*
1554      * create sequence for loading the sector index / 16B tweak value
1555      * Lower 8B of IV - sector index / tweak lower half
1556      * Upper 8B of IV - upper half of 16B tweak
1557      */
1558     append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1559             (0x20 << LDST_OFFSET_SHIFT));
1560     append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1561             (0x30 << LDST_OFFSET_SHIFT));
1562 
1563     /* Load operation */
1564     append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1565              OP_ALG_ENCRYPT);
1566 
1567     /* Perform operation */
1568     skcipher_append_src_dst(desc);
1569 
1570     /* Store lower 8B and upper 8B of IV */
1571     append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1572              (0x20 << LDST_OFFSET_SHIFT));
1573     append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1574              (0x30 << LDST_OFFSET_SHIFT));
1575 
1576     print_hex_dump_debug("xts skcipher enc shdesc@" __stringify(__LINE__)
1577                  ": ", DUMP_PREFIX_ADDRESS, 16, 4,
1578                  desc, desc_bytes(desc), 1);
1579 }
1580 EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_encap);
1581 
1582 /**
1583  * cnstr_shdsc_xts_skcipher_decap - xts skcipher decapsulation shared descriptor
1584  * @desc: pointer to buffer used for descriptor construction
1585  * @cdata: pointer to block cipher transform definitions
1586  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
1587  */
1588 void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata)
1589 {
1590     /*
1591      * Set sector size to a big value, practically disabling
1592      * sector size segmentation in xts implementation. We cannot
1593      * take full advantage of this HW feature with existing
1594      * crypto API / dm-crypt SW architecture.
1595      */
1596     __be64 sector_size = cpu_to_be64(BIT(15));
1597     u32 *key_jump_cmd;
1598 
1599     init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1600     /* Skip if already shared */
1601     key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1602                    JUMP_COND_SHRD);
1603 
1604     /* Load class1 key only */
1605     append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1606               cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1607 
1608     /* Load sector size with index 40 bytes (0x28) */
1609     append_load_as_imm(desc, (void *)&sector_size, 8, LDST_CLASS_1_CCB |
1610                LDST_SRCDST_BYTE_CONTEXT |
1611                (0x28 << LDST_OFFSET_SHIFT));
1612 
1613     set_jump_tgt_here(desc, key_jump_cmd);
1614 
1615     /*
1616      * create sequence for loading the sector index / 16B tweak value
1617      * Lower 8B of IV - sector index / tweak lower half
1618      * Upper 8B of IV - upper half of 16B tweak
1619      */
1620     append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1621             (0x20 << LDST_OFFSET_SHIFT));
1622     append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1623             (0x30 << LDST_OFFSET_SHIFT));
1624     /* Load operation */
1625     append_dec_op1(desc, cdata->algtype);
1626 
1627     /* Perform operation */
1628     skcipher_append_src_dst(desc);
1629 
1630     /* Store lower 8B and upper 8B of IV */
1631     append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1632              (0x20 << LDST_OFFSET_SHIFT));
1633     append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1634              (0x30 << LDST_OFFSET_SHIFT));
1635 
1636     print_hex_dump_debug("xts skcipher dec shdesc@" __stringify(__LINE__)
1637                  ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
1638                  desc_bytes(desc), 1);
1639 }
1640 EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_decap);
1641 
1642 MODULE_LICENSE("GPL");
1643 MODULE_DESCRIPTION("FSL CAAM descriptor support");
1644 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!