0001
0002 #ifndef __NITROX_REQ_H
0003 #define __NITROX_REQ_H
0004
0005 #include <linux/dma-mapping.h>
0006 #include <crypto/aes.h>
0007
0008 #include "nitrox_dev.h"
0009
0010 #define PENDING_SIG 0xFFFFFFFFFFFFFFFFUL
0011 #define PRIO 4001
0012
0013 typedef void (*sereq_completion_t)(void *req, int err);
0014
0015
0016
0017
0018
0019
0020
0021
0022
0023
0024 struct gphdr {
0025 __be16 param0;
0026 __be16 param1;
0027 __be16 param2;
0028 __be16 param3;
0029 };
0030
0031
0032
0033
0034
0035
0036
0037
0038
0039
0040 union se_req_ctrl {
0041 u64 value;
0042 struct {
0043 u64 raz : 22;
0044 u64 arg : 8;
0045 u64 ctxc : 2;
0046 u64 unca : 1;
0047 u64 info : 3;
0048 u64 unc : 8;
0049 u64 ctxl : 12;
0050 u64 uddl : 8;
0051 } s;
0052 };
0053
0054 #define MAX_IV_LEN 16
0055
0056
0057
0058
0059
0060
0061
0062
0063
0064
0065
0066
0067
0068 struct se_crypto_request {
0069 u8 opcode;
0070 gfp_t gfp;
0071 u32 flags;
0072 u64 ctx_handle;
0073
0074 struct gphdr gph;
0075 union se_req_ctrl ctrl;
0076 u64 *orh;
0077 u64 *comp;
0078
0079 struct scatterlist *src;
0080 struct scatterlist *dst;
0081 };
0082
0083
0084 #define FLEXI_CRYPTO_ENCRYPT_HMAC 0x33
0085 #define ENCRYPT 0
0086 #define DECRYPT 1
0087
0088
0089 #define IV_FROM_CTX 0
0090
0091 #define IV_FROM_DPTR 1
0092
0093
0094
0095
0096 enum flexi_cipher {
0097 CIPHER_NULL = 0,
0098 CIPHER_3DES_CBC,
0099 CIPHER_3DES_ECB,
0100 CIPHER_AES_CBC,
0101 CIPHER_AES_ECB,
0102 CIPHER_AES_CFB,
0103 CIPHER_AES_CTR,
0104 CIPHER_AES_GCM,
0105 CIPHER_AES_XTS,
0106 CIPHER_AES_CCM,
0107 CIPHER_AES_CBC_CTS,
0108 CIPHER_AES_ECB_CTS,
0109 CIPHER_INVALID
0110 };
0111
0112 enum flexi_auth {
0113 AUTH_NULL = 0,
0114 AUTH_MD5,
0115 AUTH_SHA1,
0116 AUTH_SHA2_SHA224,
0117 AUTH_SHA2_SHA256,
0118 AUTH_SHA2_SHA384,
0119 AUTH_SHA2_SHA512,
0120 AUTH_GMAC,
0121 AUTH_INVALID
0122 };
0123
0124
0125
0126
0127
0128
0129 struct crypto_keys {
0130 union {
0131 u8 key[AES_MAX_KEY_SIZE];
0132 u8 key1[AES_MAX_KEY_SIZE];
0133 } u;
0134 u8 iv[AES_BLOCK_SIZE];
0135 };
0136
0137
0138
0139
0140
0141
0142 struct auth_keys {
0143 union {
0144 u8 ipad[64];
0145 u8 key2[64];
0146 } u;
0147 u8 opad[64];
0148 };
0149
0150 union fc_ctx_flags {
0151 __be64 f;
0152 u64 fu;
0153 struct {
0154 #if defined(__BIG_ENDIAN_BITFIELD)
0155 u64 cipher_type : 4;
0156 u64 reserved_59 : 1;
0157 u64 aes_keylen : 2;
0158 u64 iv_source : 1;
0159 u64 hash_type : 4;
0160 u64 reserved_49_51 : 3;
0161 u64 auth_input_type: 1;
0162 u64 mac_len : 8;
0163 u64 reserved_0_39 : 40;
0164 #else
0165 u64 reserved_0_39 : 40;
0166 u64 mac_len : 8;
0167 u64 auth_input_type: 1;
0168 u64 reserved_49_51 : 3;
0169 u64 hash_type : 4;
0170 u64 iv_source : 1;
0171 u64 aes_keylen : 2;
0172 u64 reserved_59 : 1;
0173 u64 cipher_type : 4;
0174 #endif
0175 } w0;
0176 };
0177
0178
0179
0180
0181
0182
0183
0184
0185
0186
0187
0188
0189
0190 struct flexi_crypto_context {
0191 union fc_ctx_flags flags;
0192 struct crypto_keys crypto;
0193 struct auth_keys auth;
0194 };
0195
0196 struct crypto_ctx_hdr {
0197 struct dma_pool *pool;
0198 dma_addr_t dma;
0199 void *vaddr;
0200 };
0201
0202 struct nitrox_crypto_ctx {
0203 struct nitrox_device *ndev;
0204 union {
0205 u64 ctx_handle;
0206 struct flexi_crypto_context *fctx;
0207 } u;
0208 struct crypto_ctx_hdr *chdr;
0209 sereq_completion_t callback;
0210 };
0211
0212 struct nitrox_kcrypt_request {
0213 struct se_crypto_request creq;
0214 u8 *src;
0215 u8 *dst;
0216 u8 *iv_out;
0217 };
0218
0219
0220
0221
0222
0223
0224
0225
0226
0227
0228
0229
0230
0231
0232
0233
0234 struct nitrox_aead_rctx {
0235 struct nitrox_kcrypt_request nkreq;
0236 unsigned int cryptlen;
0237 unsigned int assoclen;
0238 unsigned int srclen;
0239 unsigned int dstlen;
0240 u8 *iv;
0241 int ivsize;
0242 u32 flags;
0243 u64 ctx_handle;
0244 struct scatterlist *src;
0245 struct scatterlist *dst;
0246 u8 ctrl_arg;
0247 };
0248
0249
0250
0251
0252
0253
0254
0255
0256 struct nitrox_rfc4106_rctx {
0257 struct nitrox_aead_rctx base;
0258 struct scatterlist src[3];
0259 struct scatterlist dst[3];
0260 u8 assoc[20];
0261 };
0262
0263
0264
0265
0266
0267
0268
0269
0270
0271
0272
0273
0274
0275
0276
0277
0278
0279
0280
0281
0282
0283 union pkt_instr_hdr {
0284 __be64 bev;
0285 u64 value;
0286 struct {
0287 #if defined(__BIG_ENDIAN_BITFIELD)
0288 u64 raz_48_63 : 16;
0289 u64 g : 1;
0290 u64 gsz : 7;
0291 u64 ihi : 1;
0292 u64 ssz : 7;
0293 u64 raz_30_31 : 2;
0294 u64 fsz : 6;
0295 u64 raz_16_23 : 8;
0296 u64 tlen : 16;
0297 #else
0298 u64 tlen : 16;
0299 u64 raz_16_23 : 8;
0300 u64 fsz : 6;
0301 u64 raz_30_31 : 2;
0302 u64 ssz : 7;
0303 u64 ihi : 1;
0304 u64 gsz : 7;
0305 u64 g : 1;
0306 u64 raz_48_63 : 16;
0307 #endif
0308 } s;
0309 };
0310
0311
0312
0313
0314
0315
0316
0317
0318
0319
0320
0321
0322
0323
0324
0325
0326
0327
0328 union pkt_hdr {
0329 __be64 bev[2];
0330 u64 value[2];
0331 struct {
0332 #if defined(__BIG_ENDIAN_BITFIELD)
0333 u64 opcode : 8;
0334 u64 arg : 8;
0335 u64 ctxc : 2;
0336 u64 unca : 1;
0337 u64 raz_44 : 1;
0338 u64 info : 3;
0339 u64 destport : 9;
0340 u64 unc : 8;
0341 u64 raz_19_23 : 5;
0342 u64 grp : 3;
0343 u64 raz_15 : 1;
0344 u64 ctxl : 7;
0345 u64 uddl : 8;
0346 #else
0347 u64 uddl : 8;
0348 u64 ctxl : 7;
0349 u64 raz_15 : 1;
0350 u64 grp : 3;
0351 u64 raz_19_23 : 5;
0352 u64 unc : 8;
0353 u64 destport : 9;
0354 u64 info : 3;
0355 u64 raz_44 : 1;
0356 u64 unca : 1;
0357 u64 ctxc : 2;
0358 u64 arg : 8;
0359 u64 opcode : 8;
0360 #endif
0361 __be64 ctxp;
0362 } s;
0363 };
0364
0365
0366
0367
0368
0369
0370
0371
0372
0373
0374
0375 union slc_store_info {
0376 __be64 bev[2];
0377 u64 value[2];
0378 struct {
0379 #if defined(__BIG_ENDIAN_BITFIELD)
0380 u64 raz_39_63 : 25;
0381 u64 ssz : 7;
0382 u64 raz_0_31 : 32;
0383 #else
0384 u64 raz_0_31 : 32;
0385 u64 ssz : 7;
0386 u64 raz_39_63 : 25;
0387 #endif
0388 __be64 rptr;
0389 } s;
0390 };
0391
0392
0393
0394
0395
0396
0397
0398
0399
0400
0401
0402 struct nps_pkt_instr {
0403 __be64 dptr0;
0404 union pkt_instr_hdr ih;
0405 union pkt_hdr irh;
0406 union slc_store_info slc;
0407 u64 fdata[2];
0408 };
0409
0410
0411
0412
0413
0414
0415
0416
0417
0418
0419
0420
0421 struct aqmq_command_s {
0422 __be16 opcode;
0423 __be16 param1;
0424 __be16 param2;
0425 __be16 dlen;
0426 __be64 dptr;
0427 __be64 rptr;
0428 union {
0429 __be64 word3;
0430 #if defined(__BIG_ENDIAN_BITFIELD)
0431 u64 grp : 3;
0432 u64 cptr : 61;
0433 #else
0434 u64 cptr : 61;
0435 u64 grp : 3;
0436 #endif
0437 };
0438 };
0439
0440
0441
0442
0443
0444
0445
0446 struct ctx_hdr {
0447 struct dma_pool *pool;
0448 dma_addr_t dma;
0449 dma_addr_t ctx_dma;
0450 };
0451
0452
0453
0454
0455
0456
0457
0458
0459
0460
0461
0462
0463 struct nitrox_sgcomp {
0464 __be16 len[4];
0465 __be64 dma[4];
0466 };
0467
0468
0469
0470
0471
0472
0473
0474
0475
0476
0477 struct nitrox_sgtable {
0478 u8 sgmap_cnt;
0479 u16 total_bytes;
0480 u32 sgcomp_len;
0481 dma_addr_t sgcomp_dma;
0482 struct scatterlist *sg;
0483 struct nitrox_sgcomp *sgcomp;
0484 };
0485
0486
0487 #define ORH_HLEN 8
0488
0489 #define COMP_HLEN 8
0490
0491 struct resp_hdr {
0492 u64 *orh;
0493 u64 *completion;
0494 };
0495
0496 typedef void (*completion_t)(void *arg, int err);
0497
0498
0499
0500
0501
0502
0503
0504
0505
0506
0507
0508
0509
0510
0511
0512 struct nitrox_softreq {
0513 struct list_head response;
0514 struct list_head backlog;
0515
0516 u32 flags;
0517 gfp_t gfp;
0518 atomic_t status;
0519
0520 struct nitrox_device *ndev;
0521 struct nitrox_cmdq *cmdq;
0522
0523 struct nps_pkt_instr instr;
0524 struct resp_hdr resp;
0525 struct nitrox_sgtable in;
0526 struct nitrox_sgtable out;
0527
0528 unsigned long tstamp;
0529
0530 completion_t callback;
0531 void *cb_arg;
0532 };
0533
0534 static inline int flexi_aes_keylen(int keylen)
0535 {
0536 int aes_keylen;
0537
0538 switch (keylen) {
0539 case AES_KEYSIZE_128:
0540 aes_keylen = 1;
0541 break;
0542 case AES_KEYSIZE_192:
0543 aes_keylen = 2;
0544 break;
0545 case AES_KEYSIZE_256:
0546 aes_keylen = 3;
0547 break;
0548 default:
0549 aes_keylen = -EINVAL;
0550 break;
0551 }
0552 return aes_keylen;
0553 }
0554
0555 static inline void *alloc_req_buf(int nents, int extralen, gfp_t gfp)
0556 {
0557 size_t size;
0558
0559 size = sizeof(struct scatterlist) * nents;
0560 size += extralen;
0561
0562 return kzalloc(size, gfp);
0563 }
0564
0565
0566
0567
0568
0569
0570
0571
0572
0573 static inline struct scatterlist *create_single_sg(struct scatterlist *sg,
0574 void *buf, int buflen)
0575 {
0576 sg_set_buf(sg, buf, buflen);
0577 sg++;
0578 return sg;
0579 }
0580
0581
0582
0583
0584
0585
0586
0587
0588
0589
0590 static inline struct scatterlist *create_multi_sg(struct scatterlist *to_sg,
0591 struct scatterlist *from_sg,
0592 int buflen)
0593 {
0594 struct scatterlist *sg = to_sg;
0595 unsigned int sglen;
0596
0597 for (; buflen && from_sg; buflen -= sglen) {
0598 sglen = from_sg->length;
0599 if (sglen > buflen)
0600 sglen = buflen;
0601
0602 sg_set_buf(sg, sg_virt(from_sg), sglen);
0603 from_sg = sg_next(from_sg);
0604 sg++;
0605 }
0606
0607 return sg;
0608 }
0609
0610 static inline void set_orh_value(u64 *orh)
0611 {
0612 WRITE_ONCE(*orh, PENDING_SIG);
0613 }
0614
0615 static inline void set_comp_value(u64 *comp)
0616 {
0617 WRITE_ONCE(*comp, PENDING_SIG);
0618 }
0619
0620 static inline int alloc_src_req_buf(struct nitrox_kcrypt_request *nkreq,
0621 int nents, int ivsize)
0622 {
0623 struct se_crypto_request *creq = &nkreq->creq;
0624
0625 nkreq->src = alloc_req_buf(nents, ivsize, creq->gfp);
0626 if (!nkreq->src)
0627 return -ENOMEM;
0628
0629 return 0;
0630 }
0631
0632 static inline void nitrox_creq_copy_iv(char *dst, char *src, int size)
0633 {
0634 memcpy(dst, src, size);
0635 }
0636
0637 static inline struct scatterlist *nitrox_creq_src_sg(char *iv, int ivsize)
0638 {
0639 return (struct scatterlist *)(iv + ivsize);
0640 }
0641
0642 static inline void nitrox_creq_set_src_sg(struct nitrox_kcrypt_request *nkreq,
0643 int nents, int ivsize,
0644 struct scatterlist *src, int buflen)
0645 {
0646 char *iv = nkreq->src;
0647 struct scatterlist *sg;
0648 struct se_crypto_request *creq = &nkreq->creq;
0649
0650 creq->src = nitrox_creq_src_sg(iv, ivsize);
0651 sg = creq->src;
0652 sg_init_table(sg, nents);
0653
0654
0655
0656
0657
0658
0659
0660
0661 sg = create_single_sg(sg, iv, ivsize);
0662
0663 create_multi_sg(sg, src, buflen);
0664 }
0665
0666 static inline int alloc_dst_req_buf(struct nitrox_kcrypt_request *nkreq,
0667 int nents)
0668 {
0669 int extralen = ORH_HLEN + COMP_HLEN;
0670 struct se_crypto_request *creq = &nkreq->creq;
0671
0672 nkreq->dst = alloc_req_buf(nents, extralen, creq->gfp);
0673 if (!nkreq->dst)
0674 return -ENOMEM;
0675
0676 return 0;
0677 }
0678
0679 static inline void nitrox_creq_set_orh(struct nitrox_kcrypt_request *nkreq)
0680 {
0681 struct se_crypto_request *creq = &nkreq->creq;
0682
0683 creq->orh = (u64 *)(nkreq->dst);
0684 set_orh_value(creq->orh);
0685 }
0686
0687 static inline void nitrox_creq_set_comp(struct nitrox_kcrypt_request *nkreq)
0688 {
0689 struct se_crypto_request *creq = &nkreq->creq;
0690
0691 creq->comp = (u64 *)(nkreq->dst + ORH_HLEN);
0692 set_comp_value(creq->comp);
0693 }
0694
0695 static inline struct scatterlist *nitrox_creq_dst_sg(char *dst)
0696 {
0697 return (struct scatterlist *)(dst + ORH_HLEN + COMP_HLEN);
0698 }
0699
0700 static inline void nitrox_creq_set_dst_sg(struct nitrox_kcrypt_request *nkreq,
0701 int nents, int ivsize,
0702 struct scatterlist *dst, int buflen)
0703 {
0704 struct se_crypto_request *creq = &nkreq->creq;
0705 struct scatterlist *sg;
0706 char *iv = nkreq->src;
0707
0708 creq->dst = nitrox_creq_dst_sg(nkreq->dst);
0709 sg = creq->dst;
0710 sg_init_table(sg, nents);
0711
0712
0713
0714
0715
0716
0717
0718
0719 sg = create_single_sg(sg, creq->orh, ORH_HLEN);
0720
0721 sg = create_single_sg(sg, iv, ivsize);
0722
0723 sg = create_multi_sg(sg, dst, buflen);
0724
0725 create_single_sg(sg, creq->comp, COMP_HLEN);
0726 }
0727
0728 #endif