crypto/bcm/cipher.h

0001
0002 /* SPDX-License-Identifier: GPL-2.0-only */
0003 /*
0004  * Copyright 2016 Broadcom
0005  */
0006
0007 #ifndef _CIPHER_H
0008 #define _CIPHER_H
0009
0010 #include <linux/atomic.h>
0011 #include <linux/mailbox/brcm-message.h>
0012 #include <linux/mailbox_client.h>
0013 #include <crypto/aes.h>
0014 #include <crypto/internal/hash.h>
0015 #include <crypto/internal/skcipher.h>
0016 #include <crypto/aead.h>
0017 #include <crypto/arc4.h>
0018 #include <crypto/gcm.h>
0019 #include <crypto/sha1.h>
0020 #include <crypto/sha2.h>
0021 #include <crypto/sha3.h>
0022
0023 #include "spu.h"
0024 #include "spum.h"
0025 #include "spu2.h"
0026
0027 /* Driver supports up to MAX_SPUS SPU blocks */
0028 #define MAX_SPUS 16
0029
0030 #define ARC4_STATE_SIZE     4
0031
0032 #define CCM_AES_IV_SIZE    16
0033 #define CCM_ESP_IV_SIZE     8
0034 #define RFC4543_ICV_SIZE   16
0035
0036 #define MAX_KEY_SIZE    ARC4_MAX_KEY_SIZE
0037 #define MAX_IV_SIZE AES_BLOCK_SIZE
0038 #define MAX_DIGEST_SIZE SHA3_512_DIGEST_SIZE
0039 #define MAX_ASSOC_SIZE  512
0040
0041 /* size of salt value for AES-GCM-ESP and AES-CCM-ESP */
0042 #define GCM_ESP_SALT_SIZE   4
0043 #define CCM_ESP_SALT_SIZE   3
0044 #define MAX_SALT_SIZE       GCM_ESP_SALT_SIZE
0045 #define GCM_ESP_SALT_OFFSET 0
0046 #define CCM_ESP_SALT_OFFSET 1
0047
0048 #define GCM_ESP_DIGESTSIZE 16
0049
0050 #define MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE
0051
0052 /*
0053  * Maximum number of bytes from a non-final hash request that can be deferred
0054  * until more data is available. With new crypto API framework, this
0055  * can be no more than one block of data.
0056  */
0057 #define HASH_CARRY_MAX  MAX_HASH_BLOCK_SIZE
0058
0059 /* Force at least 4-byte alignment of all SPU message fields */
0060 #define SPU_MSG_ALIGN  4
0061
0062 /* Number of times to resend mailbox message if mb queue is full */
0063 #define SPU_MB_RETRY_MAX  1000
0064
0065 /* op_counts[] indexes */
0066 enum op_type {
0067     SPU_OP_CIPHER,
0068     SPU_OP_HASH,
0069     SPU_OP_HMAC,
0070     SPU_OP_AEAD,
0071     SPU_OP_NUM
0072 };
0073
0074 enum spu_spu_type {
0075     SPU_TYPE_SPUM,
0076     SPU_TYPE_SPU2,
0077 };
0078
0079 /*
0080  * SPUM_NS2 and SPUM_NSP are the SPU-M block on Northstar 2 and Northstar Plus,
0081  * respectively.
0082  */
0083 enum spu_spu_subtype {
0084     SPU_SUBTYPE_SPUM_NS2,
0085     SPU_SUBTYPE_SPUM_NSP,
0086     SPU_SUBTYPE_SPU2_V1,
0087     SPU_SUBTYPE_SPU2_V2
0088 };
0089
0090 struct spu_type_subtype {
0091     enum spu_spu_type type;
0092     enum spu_spu_subtype subtype;
0093 };
0094
0095 struct cipher_op {
0096     enum spu_cipher_alg alg;
0097     enum spu_cipher_mode mode;
0098 };
0099
0100 struct auth_op {
0101     enum hash_alg alg;
0102     enum hash_mode mode;
0103 };
0104
0105 struct iproc_alg_s {
0106     u32 type;
0107     union {
0108         struct skcipher_alg skcipher;
0109         struct ahash_alg hash;
0110         struct aead_alg aead;
0111     } alg;
0112     struct cipher_op cipher_info;
0113     struct auth_op auth_info;
0114     bool auth_first;
0115     bool registered;
0116 };
0117
0118 /*
0119  * Buffers for a SPU request/reply message pair. All part of one structure to
0120  * allow a single alloc per request.
0121  */
0122 struct spu_msg_buf {
0123     /* Request message fragments */
0124
0125     /*
0126      * SPU request message header. For SPU-M, holds MH, EMH, SCTX, BDESC,
0127      * and BD header. For SPU2, holds FMD, OMD.
0128      */
0129     u8 bcm_spu_req_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)];
0130
0131     /* IV or counter. Size to include salt. Also used for XTS tweek. */
0132     u8 iv_ctr[ALIGN(2 * AES_BLOCK_SIZE, SPU_MSG_ALIGN)];
0133
0134     /* Hash digest. request and response. */
0135     u8 digest[ALIGN(MAX_DIGEST_SIZE, SPU_MSG_ALIGN)];
0136
0137     /* SPU request message padding */
0138     u8 spu_req_pad[ALIGN(SPU_PAD_LEN_MAX, SPU_MSG_ALIGN)];
0139
0140     /* SPU-M request message STATUS field */
0141     u8 tx_stat[ALIGN(SPU_TX_STATUS_LEN, SPU_MSG_ALIGN)];
0142
0143     /* Response message fragments */
0144
0145     /* SPU response message header */
0146     u8 spu_resp_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)];
0147
0148     /* SPU response message STATUS field padding */
0149     u8 rx_stat_pad[ALIGN(SPU_STAT_PAD_MAX, SPU_MSG_ALIGN)];
0150
0151     /* SPU response message STATUS field */
0152     u8 rx_stat[ALIGN(SPU_RX_STATUS_LEN, SPU_MSG_ALIGN)];
0153
0154     union {
0155         /* Buffers only used for skcipher */
0156         struct {
0157             /*
0158              * Field used for either SUPDT when RC4 is used
0159              * -OR- tweak value when XTS/AES is used
0160              */
0161             u8 supdt_tweak[ALIGN(SPU_SUPDT_LEN, SPU_MSG_ALIGN)];
0162         } c;
0163
0164         /* Buffers only used for aead */
0165         struct {
0166             /* SPU response pad for GCM data */
0167             u8 gcmpad[ALIGN(AES_BLOCK_SIZE, SPU_MSG_ALIGN)];
0168
0169             /* SPU request msg padding for GCM AAD */
0170             u8 req_aad_pad[ALIGN(SPU_PAD_LEN_MAX, SPU_MSG_ALIGN)];
0171
0172             /* SPU response data to be discarded */
0173             u8 resp_aad[ALIGN(MAX_ASSOC_SIZE + MAX_IV_SIZE,
0174                       SPU_MSG_ALIGN)];
0175         } a;
0176     };
0177 };
0178
0179 struct iproc_ctx_s {
0180     u8 enckey[MAX_KEY_SIZE + ARC4_STATE_SIZE];
0181     unsigned int enckeylen;
0182
0183     u8 authkey[MAX_KEY_SIZE + ARC4_STATE_SIZE];
0184     unsigned int authkeylen;
0185
0186     u8 salt[MAX_SALT_SIZE];
0187     unsigned int salt_len;
0188     unsigned int salt_offset;
0189     u8 iv[MAX_IV_SIZE];
0190
0191     unsigned int digestsize;
0192
0193     struct iproc_alg_s *alg;
0194     bool is_esp;
0195
0196     struct cipher_op cipher;
0197     enum spu_cipher_type cipher_type;
0198
0199     struct auth_op auth;
0200     bool auth_first;
0201
0202     /*
0203      * The maximum length in bytes of the payload in a SPU message for this
0204      * context. For SPU-M, the payload is the combination of AAD and data.
0205      * For SPU2, the payload is just data. A value of SPU_MAX_PAYLOAD_INF
0206      * indicates that there is no limit to the length of the SPU message
0207      * payload.
0208      */
0209     unsigned int max_payload;
0210
0211     struct crypto_aead *fallback_cipher;
0212
0213     /* auth_type is determined during processing of request */
0214
0215     u8 ipad[MAX_HASH_BLOCK_SIZE];
0216     u8 opad[MAX_HASH_BLOCK_SIZE];
0217
0218     /*
0219      * Buffer to hold SPU message header template. Template is created at
0220      * setkey time for skcipher requests, since most of the fields in the
0221      * header are known at that time. At request time, just fill in a few
0222      * missing pieces related to length of data in the request and IVs, etc.
0223      */
0224     u8 bcm_spu_req_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)];
0225
0226     /* Length of SPU request header */
0227     u16 spu_req_hdr_len;
0228
0229     /* Expected length of SPU response header */
0230     u16 spu_resp_hdr_len;
0231
0232     /*
0233      * shash descriptor - needed to perform incremental hashing in
0234      * in software, when hw doesn't support it.
0235      */
0236     struct shash_desc *shash;
0237
0238     bool is_rfc4543;    /* RFC 4543 style of GMAC */
0239 };
0240
0241 /* state from iproc_reqctx_s necessary for hash state export/import */
0242 struct spu_hash_export_s {
0243     unsigned int total_todo;
0244     unsigned int total_sent;
0245     u8 hash_carry[HASH_CARRY_MAX];
0246     unsigned int hash_carry_len;
0247     u8 incr_hash[MAX_DIGEST_SIZE];
0248     bool is_sw_hmac;
0249 };
0250
0251 struct iproc_reqctx_s {
0252     /* general context */
0253     struct crypto_async_request *parent;
0254
0255     /* only valid after enqueue() */
0256     struct iproc_ctx_s *ctx;
0257
0258     u8 chan_idx;   /* Mailbox channel to be used to submit this request */
0259
0260     /* total todo, rx'd, and sent for this request */
0261     unsigned int total_todo;
0262     unsigned int total_received;    /* only valid for skcipher */
0263     unsigned int total_sent;
0264
0265     /*
0266      * num bytes sent to hw from the src sg in this request. This can differ
0267      * from total_sent for incremental hashing. total_sent includes previous
0268      * init() and update() data. src_sent does not.
0269      */
0270     unsigned int src_sent;
0271
0272     /*
0273      * For AEAD requests, start of associated data. This will typically
0274      * point to the beginning of the src scatterlist from the request,
0275      * since assoc data is at the beginning of the src scatterlist rather
0276      * than in its own sg.
0277      */
0278     struct scatterlist *assoc;
0279
0280     /*
0281      * scatterlist entry and offset to start of data for next chunk. Crypto
0282      * API src scatterlist for AEAD starts with AAD, if present. For first
0283      * chunk, src_sg is sg entry at beginning of input data (after AAD).
0284      * src_skip begins at the offset in that sg entry where data begins.
0285      */
0286     struct scatterlist *src_sg;
0287     int src_nents;      /* Number of src entries with data */
0288     u32 src_skip;       /* bytes of current sg entry already used */
0289
0290     /*
0291      * Same for destination. For AEAD, if there is AAD, output data must
0292      * be written at offset following AAD.
0293      */
0294     struct scatterlist *dst_sg;
0295     int dst_nents;      /* Number of dst entries with data */
0296     u32 dst_skip;       /* bytes of current sg entry already written */
0297
0298     /* Mailbox message used to send this request to PDC driver */
0299     struct brcm_message mb_mssg;
0300
0301     bool bd_suppress;   /* suppress BD field in SPU response? */
0302
0303     /* cipher context */
0304     bool is_encrypt;
0305
0306     /*
0307      * CBC mode: IV.  CTR mode: counter.  Else empty. Used as a DMA
0308      * buffer for AEAD requests. So allocate as DMAable memory. If IV
0309      * concatenated with salt, includes the salt.
0310      */
0311     u8 *iv_ctr;
0312     /* Length of IV or counter, in bytes */
0313     unsigned int iv_ctr_len;
0314
0315     /*
0316      * Hash requests can be of any size, whether initial, update, or final.
0317      * A non-final request must be submitted to the SPU as an integral
0318      * number of blocks. This may leave data at the end of the request
0319      * that is not a full block. Since the request is non-final, it cannot
0320      * be padded. So, we write the remainder to this hash_carry buffer and
0321      * hold it until the next request arrives. The carry data is then
0322      * submitted at the beginning of the data in the next SPU msg.
0323      * hash_carry_len is the number of bytes currently in hash_carry. These
0324      * fields are only used for ahash requests.
0325      */
0326     u8 hash_carry[HASH_CARRY_MAX];
0327     unsigned int hash_carry_len;
0328     unsigned int is_final;  /* is this the final for the hash op? */
0329
0330     /*
0331      * Digest from incremental hash is saved here to include in next hash
0332      * operation. Cannot be stored in req->result for truncated hashes,
0333      * since result may be sized for final digest. Cannot be saved in
0334      * msg_buf because that gets deleted between incremental hash ops
0335      * and is not saved as part of export().
0336      */
0337     u8 incr_hash[MAX_DIGEST_SIZE];
0338
0339     /* hmac context */
0340     bool is_sw_hmac;
0341
0342     /* aead context */
0343     struct crypto_tfm *old_tfm;
0344     crypto_completion_t old_complete;
0345     void *old_data;
0346
0347     gfp_t gfp;
0348
0349     /* Buffers used to build SPU request and response messages */
0350     struct spu_msg_buf msg_buf;
0351 };
0352
0353 /*
0354  * Structure encapsulates a set of function pointers specific to the type of
0355  * SPU hardware running. These functions handling creation and parsing of
0356  * SPU request messages and SPU response messages. Includes hardware-specific
0357  * values read from device tree.
0358  */
0359 struct spu_hw {
0360     void (*spu_dump_msg_hdr)(u8 *buf, unsigned int buf_len);
0361     u32 (*spu_ctx_max_payload)(enum spu_cipher_alg cipher_alg,
0362                    enum spu_cipher_mode cipher_mode,
0363                    unsigned int blocksize);
0364     u32 (*spu_payload_length)(u8 *spu_hdr);
0365     u16 (*spu_response_hdr_len)(u16 auth_key_len, u16 enc_key_len,
0366                     bool is_hash);
0367     u16 (*spu_hash_pad_len)(enum hash_alg hash_alg,
0368                 enum hash_mode hash_mode, u32 chunksize,
0369                 u16 hash_block_size);
0370     u32 (*spu_gcm_ccm_pad_len)(enum spu_cipher_mode cipher_mode,
0371                    unsigned int data_size);
0372     u32 (*spu_assoc_resp_len)(enum spu_cipher_mode cipher_mode,
0373                   unsigned int assoc_len,
0374                   unsigned int iv_len, bool is_encrypt);
0375     u8 (*spu_aead_ivlen)(enum spu_cipher_mode cipher_mode,
0376                  u16 iv_len);
0377     enum hash_type (*spu_hash_type)(u32 src_sent);
0378     u32 (*spu_digest_size)(u32 digest_size, enum hash_alg alg,
0379                    enum hash_type);
0380     u32 (*spu_create_request)(u8 *spu_hdr,
0381                   struct spu_request_opts *req_opts,
0382                   struct spu_cipher_parms *cipher_parms,
0383                   struct spu_hash_parms *hash_parms,
0384                   struct spu_aead_parms *aead_parms,
0385                   unsigned int data_size);
0386     u16 (*spu_cipher_req_init)(u8 *spu_hdr,
0387                    struct spu_cipher_parms *cipher_parms);
0388     void (*spu_cipher_req_finish)(u8 *spu_hdr,
0389                       u16 spu_req_hdr_len,
0390                       unsigned int is_inbound,
0391                       struct spu_cipher_parms *cipher_parms,
0392                       unsigned int data_size);
0393     void (*spu_request_pad)(u8 *pad_start, u32 gcm_padding,
0394                 u32 hash_pad_len, enum hash_alg auth_alg,
0395                 enum hash_mode auth_mode,
0396                 unsigned int total_sent, u32 status_padding);
0397     u8 (*spu_xts_tweak_in_payload)(void);
0398     u8 (*spu_tx_status_len)(void);
0399     u8 (*spu_rx_status_len)(void);
0400     int (*spu_status_process)(u8 *statp);
0401     void (*spu_ccm_update_iv)(unsigned int digestsize,
0402                   struct spu_cipher_parms *cipher_parms,
0403                   unsigned int assoclen, unsigned int chunksize,
0404                   bool is_encrypt, bool is_esp);
0405     u32 (*spu_wordalign_padlen)(u32 data_size);
0406
0407     /* The base virtual address of the SPU hw registers */
0408     void __iomem *reg_vbase[MAX_SPUS];
0409
0410     /* Version of the SPU hardware */
0411     enum spu_spu_type spu_type;
0412
0413     /* Sub-version of the SPU hardware */
0414     enum spu_spu_subtype spu_subtype;
0415
0416     /* The number of SPUs on this platform */
0417     u32 num_spu;
0418
0419     /* The number of SPU channels on this platform */
0420     u32 num_chan;
0421 };
0422
0423 struct bcm_device_private {
0424     struct platform_device *pdev;
0425
0426     struct spu_hw spu;
0427
0428     atomic_t session_count; /* number of streams active */
0429     atomic_t stream_count;  /* monotonic counter for streamID's */
0430
0431     /* Length of BCM header. Set to 0 when hw does not expect BCM HEADER. */
0432     u8 bcm_hdr_len;
0433
0434     /* The index of the channel to use for the next crypto request */
0435     atomic_t next_chan;
0436
0437     struct dentry *debugfs_dir;
0438     struct dentry *debugfs_stats;
0439
0440     /* Number of request bytes processed and result bytes returned */
0441     atomic64_t bytes_in;
0442     atomic64_t bytes_out;
0443
0444     /* Number of operations of each type */
0445     atomic_t op_counts[SPU_OP_NUM];
0446
0447     atomic_t cipher_cnt[CIPHER_ALG_LAST][CIPHER_MODE_LAST];
0448     atomic_t hash_cnt[HASH_ALG_LAST];
0449     atomic_t hmac_cnt[HASH_ALG_LAST];
0450     atomic_t aead_cnt[AEAD_TYPE_LAST];
0451
0452     /* Number of calls to setkey() for each operation type */
0453     atomic_t setkey_cnt[SPU_OP_NUM];
0454
0455     /* Number of times request was resubmitted because mb was full */
0456     atomic_t mb_no_spc;
0457
0458     /* Number of mailbox send failures */
0459     atomic_t mb_send_fail;
0460
0461     /* Number of ICV check failures for AEAD messages */
0462     atomic_t bad_icv;
0463
0464     struct mbox_client mcl;
0465
0466     /* Array of mailbox channel pointers, one for each channel */
0467     struct mbox_chan **mbox;
0468 };
0469
0470 extern struct bcm_device_private iproc_priv;
0471
0472 #endif