[LEDE-DEV] [RFC 4/6 v2] Generic: Crypto - aesni - add ccm(aes) algorithm implementation
Chris Blake
chrisrblake93 at gmail.com
Sat Oct 22 10:39:26 PDT 2016
This patch was provided by Ben at http://lists.infradead.org/pipermail
/lede-dev/2016-October/003332.html and adds ccm(aes) support to the AES-NI
acceleration. Currently it is not submitted upstream, but has been confirmed
working on the PC Engines APU2 using openssl.
Cc: Ben Greear <greearb at candelatech.com>
Signed-off-by: Chris Blake <chrisrblake93 at gmail.com>
---
...esni-add-ccm-aes-algorithm-implementation.patch | 552 +++++++++++++++++++++
1 file changed, 552 insertions(+)
create mode 100644 target/linux/generic/patches-4.4/891-0001-crypto-aesni-add-ccm-aes-algorithm-implementation.patch
diff --git a/target/linux/generic/patches-4.4/891-0001-crypto-aesni-add-ccm-aes-algorithm-implementation.patch b/target/linux/generic/patches-4.4/891-0001-crypto-aesni-add-ccm-aes-algorithm-implementation.patch
new file mode 100644
index 0000000..3973adb
--- /dev/null
+++ b/target/linux/generic/patches-4.4/891-0001-crypto-aesni-add-ccm-aes-algorithm-implementation.patch
@@ -0,0 +1,552 @@
+From abd4e4982a674ab469916f0a50e01e69b739ce71 Mon Sep 17 00:00:00 2001
+From: Yauhen Kharuzhy <jekhor at gmail.com>
+Date: Fri, 23 Oct 2015 00:22:02 +0300
+Subject: [PATCH] crypto: aesni - add ccm(aes) algorithm implementation
+
+Add ccm(aes) implementation from linux-wireless mailing list (see
+http://permalink.gmane.org/gmane.linux.kernel.wireless.general/126679).
+
+This eliminates FPU context store/restore overhead existing in more
+general ccm_base(ctr(aes-aesni),aes-aesni) case in MAC calculation.
+
+Convert this patch to new AEAD API.
+
+Signed-off-by: Yauhen Kharuzhy <jekhor at gmail.com>
+---
+ arch/x86/crypto/aesni-intel_glue.c | 491 ++++++++++++++++++++++++++++++++++++-
+ crypto/testmgr.c | 4 +
+ 2 files changed, 494 insertions(+), 1 deletion(-)
+
+diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
+index 5b7fa14..087a952 100644
+--- a/arch/x86/crypto/aesni-intel_glue.c
++++ b/arch/x86/crypto/aesni-intel_glue.c
+@@ -36,6 +36,7 @@
+ #include <asm/crypto/aes.h>
+ #include <crypto/ablk_helper.h>
+ #include <crypto/scatterwalk.h>
++#include <crypto/aead.h>
+ #include <crypto/internal/aead.h>
+ #include <linux/workqueue.h>
+ #include <linux/spinlock.h>
+@@ -529,6 +530,456 @@ static int ctr_crypt(struct blkcipher_desc *desc,
+
+ return err;
+ }
++
++static int __ccm_setkey(struct crypto_aead *tfm, const u8 *in_key,
++ unsigned int key_len)
++{
++ struct crypto_aes_ctx *ctx = crypto_aead_ctx(tfm);
++
++ return aes_set_key_common(crypto_aead_tfm(tfm), ctx, in_key, key_len);
++}
++
++static int __ccm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
++{
++ if ((authsize & 1) || authsize < 4)
++ return -EINVAL;
++ return 0;
++}
++
++static int set_msg_len(u8 *block, unsigned int msglen, int csize)
++{
++ __be32 data;
++
++ memset(block, 0, csize);
++ block += csize;
++
++ if (csize >= 4)
++ csize = 4;
++ else if (msglen > (1 << (8 * csize)))
++ return -EOVERFLOW;
++
++ data = cpu_to_be32(msglen);
++ memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
++
++ return 0;
++}
++
++static int ccm_init_mac(struct aead_request *req, u8 maciv[], u32 msglen)
++{
++ struct crypto_aead *aead = crypto_aead_reqtfm(req);
++ __be32 *n = (__be32 *)&maciv[AES_BLOCK_SIZE - 8];
++ u32 l = req->iv[0] + 1;
++
++ /* verify that CCM dimension 'L' is set correctly in the IV */
++ if (l < 2 || l > 8)
++ return -EINVAL;
++
++ /* verify that msglen can in fact be represented in L bytes */
++ if (l < 4 && msglen >> (8 * l))
++ return -EOVERFLOW;
++
++ /*
++ * Even if the CCM spec allows L values of up to 8, the Linux cryptoapi
++ * uses a u32 type to represent msglen so the top 4 bytes are always 0.
++ */
++ n[0] = 0;
++ n[1] = cpu_to_be32(msglen);
++
++ memcpy(maciv, req->iv, AES_BLOCK_SIZE - l);
++
++ /*
++ * Meaning of byte 0 according to CCM spec (RFC 3610/NIST 800-38C)
++ * - bits 0..2 : max # of bytes required to represent msglen, minus 1
++ * (already set by caller)
++ * - bits 3..5 : size of auth tag (1 => 4 bytes, 2 => 6 bytes, etc)
++ * - bit 6 : indicates presence of authenticate-only data
++ */
++ maciv[0] |= (crypto_aead_authsize(aead) - 2) << 2;
++ if (req->assoclen)
++ maciv[0] |= 0x40;
++
++ memset(&req->iv[AES_BLOCK_SIZE - l], 0, l);
++ return set_msg_len(maciv + AES_BLOCK_SIZE - l, msglen, l);
++}
++
++static int compute_mac(struct crypto_aes_ctx *ctx, u8 mac[], u8 *data, int n,
++ unsigned int ilen, u8 *idata)
++{
++ unsigned int bs = AES_BLOCK_SIZE;
++ u8 *odata = mac;
++ int datalen, getlen;
++
++ datalen = n;
++
++ /* first time in here, block may be partially filled. */
++ getlen = bs - ilen;
++ if (datalen >= getlen) {
++ memcpy(idata + ilen, data, getlen);
++ crypto_xor(odata, idata, bs);
++
++ aesni_enc(ctx, odata, odata);
++ datalen -= getlen;
++ data += getlen;
++ ilen = 0;
++ }
++
++ /* now encrypt rest of data */
++ while (datalen >= bs) {
++ crypto_xor(odata, data, bs);
++
++ aesni_enc(ctx, odata, odata);
++
++ datalen -= bs;
++ data += bs;
++ }
++
++ /* check and see if there's leftover data that wasn't
++ * enough to fill a block.
++ */
++ if (datalen) {
++ memcpy(idata + ilen, data, datalen);
++ ilen += datalen;
++ }
++ return ilen;
++}
++
++static unsigned int get_data_to_compute(struct crypto_aes_ctx *ctx, u8 mac[],
++ u8 *idata, struct scatterlist *sg,
++ unsigned int len, unsigned int ilen)
++{
++ struct scatter_walk walk;
++ u8 *data_src;
++ int n;
++
++ scatterwalk_start(&walk, sg);
++
++ while (len) {
++ n = scatterwalk_clamp(&walk, len);
++ if (!n) {
++ scatterwalk_start(&walk, sg_next(walk.sg));
++ n = scatterwalk_clamp(&walk, len);
++ }
++ data_src = scatterwalk_map(&walk);
++
++ ilen = compute_mac(ctx, mac, data_src, n, ilen, idata);
++ len -= n;
++
++ scatterwalk_unmap(data_src);
++ scatterwalk_advance(&walk, n);
++ scatterwalk_done(&walk, 0, len);
++ }
++
++ /* any leftover needs padding and then encrypted */
++ if (ilen) {
++ int padlen;
++ u8 *odata = mac;
++
++ padlen = AES_BLOCK_SIZE - ilen;
++ memset(idata + ilen, 0, padlen);
++ crypto_xor(odata, idata, AES_BLOCK_SIZE);
++
++ aesni_enc(ctx, odata, odata);
++ ilen = 0;
++ }
++ return ilen;
++}
++
++static void ccm_calculate_auth_mac(struct aead_request *req,
++ struct crypto_aes_ctx *ctx, u8 mac[],
++ struct scatterlist *src,
++ unsigned int cryptlen)
++{
++ unsigned int ilen;
++ u8 idata[AES_BLOCK_SIZE];
++ u32 len = req->assoclen;
++
++ aesni_enc(ctx, mac, mac);
++
++ if (len) {
++ struct __packed {
++ __be16 l;
++ __be32 h;
++ } *ltag = (void *)idata;
++
++ /* prepend the AAD with a length tag */
++ if (len < 0xff00) {
++ ltag->l = cpu_to_be16(len);
++ ilen = 2;
++ } else {
++ ltag->l = cpu_to_be16(0xfffe);
++ ltag->h = cpu_to_be32(len);
++ ilen = 6;
++ }
++
++ ilen = get_data_to_compute(ctx, mac, idata,
++ req->src, req->assoclen,
++ ilen);
++ } else {
++ ilen = 0;
++ }
++
++ /* compute plaintext into mac */
++ if (cryptlen) {
++ ilen = get_data_to_compute(ctx, mac, idata,
++ src, cryptlen, ilen);
++ }
++}
++
++static int __ccm_encrypt(struct aead_request *req)
++{
++ struct crypto_aead *aead = crypto_aead_reqtfm(req);
++ struct crypto_aes_ctx *ctx = aes_ctx(crypto_aead_ctx(aead));
++ struct blkcipher_desc desc = { .info = req->iv };
++ struct blkcipher_walk walk;
++ struct scatterlist src[3], dst[3], *pdst, *sg;
++ u8 __aligned(8) mac[AES_BLOCK_SIZE];
++ u32 len = req->cryptlen;
++ int err;
++
++ err = ccm_init_mac(req, mac, len);
++ if (err)
++ return err;
++
++ sg_init_table(src, 3);
++ sg_set_buf(src, mac, sizeof(mac));
++ sg = scatterwalk_ffwd(src + 1, req->src, req->assoclen);
++ if (sg != src + 1)
++ sg_chain(src, 2, sg);
++
++ pdst = src;
++ if (req->src != req->dst) {
++ sg_init_table(dst, 3);
++ sg_set_buf(dst, mac, sizeof(mac));
++ sg = scatterwalk_ffwd(dst + 1, req->dst, req->assoclen);
++ if (sg != dst + 1)
++ sg_chain(dst, 2, sg);
++ pdst = dst;
++ }
++
++ ccm_calculate_auth_mac(req, ctx, mac, sg_next(src), len);
++
++ len += sizeof(mac);
++ blkcipher_walk_init(&walk, pdst, src, len);
++ err = blkcipher_aead_walk_virt_block(&desc, &walk, aead,
++ AES_BLOCK_SIZE);
++
++ while ((len = walk.nbytes) >= AES_BLOCK_SIZE) {
++ aesni_ctr_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
++ len & AES_BLOCK_MASK, walk.iv);
++ len &= AES_BLOCK_SIZE - 1;
++ err = blkcipher_walk_done(&desc, &walk, len);
++ }
++ if (walk.nbytes) {
++ ctr_crypt_final(ctx, &walk);
++ err = blkcipher_walk_done(&desc, &walk, 0);
++ }
++
++ if (err)
++ return err;
++
++ /* copy authtag to end of dst */
++ scatterwalk_map_and_copy(mac, sg_next(pdst), req->cryptlen,
++ crypto_aead_authsize(aead), 1);
++ return 0;
++}
++
++static int __ccm_decrypt(struct aead_request *req)
++{
++ struct crypto_aead *aead = crypto_aead_reqtfm(req);
++ struct crypto_aes_ctx *ctx = aes_ctx(crypto_aead_ctx(aead));
++ unsigned int authsize = crypto_aead_authsize(aead);
++ struct blkcipher_desc desc = { .info = req->iv };
++ struct blkcipher_walk walk;
++ struct scatterlist src[3], dst[3], *pdst, *sg;
++ u8 __aligned(8) authtag[AES_BLOCK_SIZE], mac[AES_BLOCK_SIZE];
++ u32 len;
++ int err;
++
++ if (req->cryptlen < authsize)
++ return -EINVAL;
++
++ err = ccm_init_mac(req, mac, req->cryptlen - authsize);
++ if (err)
++ return err;
++
++ sg_init_table(src, 3);
++ sg_set_buf(src, authtag, sizeof(authtag));
++ sg = scatterwalk_ffwd(src + 1, req->src, req->assoclen);
++ if (sg != src + 1)
++ sg_chain(src, 2, sg);
++
++ pdst = src;
++ if (req->src != req->dst) {
++ sg_init_table(dst, 3);
++ sg_set_buf(dst, authtag, sizeof(authtag));
++ sg = scatterwalk_ffwd(dst + 1, req->dst, req->assoclen);
++ if (sg != dst + 1)
++ sg_chain(dst, 2, sg);
++
++ pdst = dst;
++ }
++
++ scatterwalk_map_and_copy(authtag, sg_next(src),
++ req->cryptlen - authsize, authsize, 0);
++
++ blkcipher_walk_init(&walk, pdst, src,
++ req->cryptlen - authsize + sizeof(mac));
++ err = blkcipher_aead_walk_virt_block(&desc, &walk, aead,
++ AES_BLOCK_SIZE);
++
++ while ((len = walk.nbytes) >= AES_BLOCK_SIZE) {
++ aesni_ctr_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
++ len & AES_BLOCK_MASK, walk.iv);
++ len &= AES_BLOCK_SIZE - 1;
++ err = blkcipher_walk_done(&desc, &walk, len);
++ }
++ if (walk.nbytes) {
++ ctr_crypt_final(ctx, &walk);
++ err = blkcipher_walk_done(&desc, &walk, 0);
++ }
++
++ ccm_calculate_auth_mac(req, ctx, mac, sg_next(pdst),
++ req->cryptlen - authsize);
++ if (err)
++ return err;
++
++ /* compare calculated auth tag with the stored one */
++ if (crypto_memneq(mac, authtag, authsize))
++ return -EBADMSG;
++ return 0;
++}
++
++struct ccm_async_ctx {
++ struct crypto_aes_ctx ctx;
++ struct crypto_aead *fallback;
++};
++
++static inline struct
++ccm_async_ctx *get_ccm_ctx(struct crypto_aead *aead)
++{
++ return (struct ccm_async_ctx *)
++ PTR_ALIGN((u8 *)
++ crypto_aead_ctx(aead), AESNI_ALIGN);
++}
++
++static int ccm_init(struct crypto_aead *tfm)
++{
++ struct crypto_aead *crypto_tfm;
++ struct ccm_async_ctx *ctx = (struct ccm_async_ctx *)
++ PTR_ALIGN((u8 *)crypto_aead_ctx(tfm), AESNI_ALIGN);
++
++ crypto_tfm = crypto_alloc_aead("ccm(aes)", 0,
++ CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
++ if (IS_ERR(crypto_tfm))
++ return PTR_ERR(crypto_tfm);
++
++ ctx->fallback = crypto_tfm;
++ return 0;
++}
++
++static void ccm_exit(struct crypto_aead *tfm)
++{
++ struct ccm_async_ctx *ctx = (struct ccm_async_ctx *)
++ PTR_ALIGN((u8 *)crypto_aead_ctx(tfm), AESNI_ALIGN);
++
++ if (!IS_ERR_OR_NULL(ctx->fallback))
++ crypto_free_aead(ctx->fallback);
++}
++
++static int ccm_setkey(struct crypto_aead *aead, const u8 *in_key,
++ unsigned int key_len)
++{
++ struct crypto_tfm *tfm = crypto_aead_tfm(aead);
++ struct ccm_async_ctx *ctx = (struct ccm_async_ctx *)
++ PTR_ALIGN((u8 *)crypto_aead_ctx(aead), AESNI_ALIGN);
++ int err;
++
++ err = __ccm_setkey(aead, in_key, key_len);
++ if (err)
++ return err;
++
++ /*
++ * Set the fallback transform to use the same request flags as
++ * the hardware transform.
++ */
++ ctx->fallback->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
++ ctx->fallback->base.crt_flags |=
++ tfm->crt_flags & CRYPTO_TFM_REQ_MASK;
++ return crypto_aead_setkey(ctx->fallback, in_key, key_len);
++}
++
++static int ccm_setauthsize(struct crypto_aead *aead, unsigned int authsize)
++{
++ struct ccm_async_ctx *ctx = (struct ccm_async_ctx *)
++ PTR_ALIGN((u8 *)crypto_aead_ctx(aead), AESNI_ALIGN);
++ int err;
++
++ err = __ccm_setauthsize(aead, authsize);
++ if (err)
++ return err;
++
++ return crypto_aead_setauthsize(ctx->fallback, authsize);
++}
++
++static int ccm_encrypt(struct aead_request *req)
++{
++ int ret;
++
++ if (!irq_fpu_usable()) {
++ struct crypto_aead *aead = crypto_aead_reqtfm(req);
++ struct ccm_async_ctx *ctx = get_ccm_ctx(aead);
++ struct crypto_aead *fallback = ctx->fallback;
++
++ char aead_req_data[sizeof(struct aead_request) +
++ crypto_aead_reqsize(fallback)]
++ __aligned(__alignof__(struct aead_request));
++ struct aead_request *aead_req = (void *) aead_req_data;
++
++ memset(aead_req, 0, sizeof(aead_req_data));
++ aead_request_set_tfm(aead_req, fallback);
++ aead_request_set_ad(aead_req, req->assoclen);
++ aead_request_set_crypt(aead_req, req->src, req->dst,
++ req->cryptlen, req->iv);
++ aead_request_set_callback(aead_req, req->base.flags,
++ req->base.complete, req->base.data);
++ ret = crypto_aead_encrypt(aead_req);
++ } else {
++ kernel_fpu_begin();
++ ret = __ccm_encrypt(req);
++ kernel_fpu_end();
++ }
++ return ret;
++}
++
++static int ccm_decrypt(struct aead_request *req)
++{
++ int ret;
++
++ if (!irq_fpu_usable()) {
++ struct crypto_aead *aead = crypto_aead_reqtfm(req);
++ struct ccm_async_ctx *ctx = get_ccm_ctx(aead);
++ struct crypto_aead *fallback = ctx->fallback;
++
++ char aead_req_data[sizeof(struct aead_request) +
++ crypto_aead_reqsize(fallback)]
++ __aligned(__alignof__(struct aead_request));
++ struct aead_request *aead_req = (void *) aead_req_data;
++
++ memset(aead_req, 0, sizeof(aead_req_data));
++ aead_request_set_tfm(aead_req, fallback);
++ aead_request_set_ad(aead_req, req->assoclen);
++ aead_request_set_crypt(aead_req, req->src, req->dst,
++ req->cryptlen, req->iv);
++ aead_request_set_callback(aead_req, req->base.flags,
++ req->base.complete, req->base.data);
++ ret = crypto_aead_decrypt(aead_req);
++ } else {
++ kernel_fpu_begin();
++ ret = __ccm_decrypt(req);
++ kernel_fpu_end();
++ }
++ return ret;
++}
+ #endif
+
+ static int ablk_ecb_init(struct crypto_tfm *tfm)
+@@ -1437,7 +1888,45 @@ static struct aead_alg aesni_aead_algs[] = { {
+ .cra_ctxsize = sizeof(struct cryptd_aead *),
+ .cra_module = THIS_MODULE,
+ },
+-} };
++}, {
++ .ivsize = AES_BLOCK_SIZE,
++ .maxauthsize = AES_BLOCK_SIZE,
++ .setkey = __ccm_setkey,
++ .setauthsize = __ccm_setauthsize,
++ .encrypt = __ccm_encrypt,
++ .decrypt = __ccm_decrypt,
++ .base = {
++ .cra_name = "__ccm-aes-aesni",
++ .cra_driver_name = "__driver-ccm-aes-aesni",
++ .cra_priority = 0,
++ .cra_flags = CRYPTO_ALG_INTERNAL,
++ .cra_blocksize = 1,
++ .cra_ctxsize = sizeof(struct crypto_aes_ctx) +
++ AESNI_ALIGN - 1,
++ .cra_alignmask = 0,
++ .cra_module = THIS_MODULE,
++ },
++}, {
++ .base = {
++ .cra_name = "ccm(aes)",
++ .cra_driver_name = "ccm-aes-aesni",
++ .cra_priority = 700,
++ .cra_flags = CRYPTO_ALG_NEED_FALLBACK,
++ .cra_blocksize = 1,
++ .cra_ctxsize = AESNI_ALIGN - 1 +
++ sizeof(struct ccm_async_ctx),
++ .cra_alignmask = 0,
++ .cra_module = THIS_MODULE,
++ },
++ .init = ccm_init,
++ .exit = ccm_exit,
++ .ivsize = AES_BLOCK_SIZE,
++ .maxauthsize = AES_BLOCK_SIZE,
++ .setkey = ccm_setkey,
++ .setauthsize = ccm_setauthsize,
++ .encrypt = ccm_encrypt,
++ .decrypt = ccm_decrypt,
++}};
+ #else
+ static struct aead_alg aesni_aead_algs[0];
+ #endif
+diff --git a/crypto/testmgr.c b/crypto/testmgr.c
+index c727fb0..2a765ca 100644
+--- a/crypto/testmgr.c
++++ b/crypto/testmgr.c
+@@ -1991,6 +1991,10 @@ static const struct alg_test_desc alg_test_descs[] = {
+ .alg = "__driver-cbc-twofish-avx",
+ .test = alg_test_null,
+ }, {
++ .alg = "__driver-ccm-aes-aesni",
++ .test = alg_test_null,
++ .fips_allowed = 1,
++ }, {
+ .alg = "__driver-ecb-aes-aesni",
+ .test = alg_test_null,
+ .fips_allowed = 1,
+--
+1.8.3.1
--
2.7.4
More information about the Lede-dev
mailing list