Path: blob/master/drivers/crypto/rockchip/rk3288_crypto_skcipher.c
51764 views
// SPDX-License-Identifier: GPL-2.0-only1/*2* Crypto acceleration support for Rockchip RK32883*4* Copyright (c) 2015, Fuzhou Rockchip Electronics Co., Ltd5*6* Author: Zain Wang <[email protected]>7*8* Some ideas are from marvell-cesa.c and s5p-sss.c driver.9*/1011#include <crypto/engine.h>12#include <crypto/internal/skcipher.h>13#include <crypto/scatterwalk.h>14#include <linux/device.h>15#include <linux/err.h>16#include <linux/kernel.h>17#include <linux/string.h>18#include "rk3288_crypto.h"1920#define RK_CRYPTO_DEC BIT(0)2122static int rk_cipher_need_fallback(struct skcipher_request *req)23{24struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);25struct skcipher_alg *alg = crypto_skcipher_alg(tfm);26struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.skcipher.base);27struct scatterlist *sgs, *sgd;28unsigned int stodo, dtodo, len;29unsigned int bs = crypto_skcipher_blocksize(tfm);3031if (!req->cryptlen)32return true;3334len = req->cryptlen;35sgs = req->src;36sgd = req->dst;37while (sgs && sgd) {38if (!IS_ALIGNED(sgs->offset, sizeof(u32))) {39algt->stat_fb_align++;40return true;41}42if (!IS_ALIGNED(sgd->offset, sizeof(u32))) {43algt->stat_fb_align++;44return true;45}46stodo = min(len, sgs->length);47if (stodo % bs) {48algt->stat_fb_len++;49return true;50}51dtodo = min(len, sgd->length);52if (dtodo % bs) {53algt->stat_fb_len++;54return true;55}56if (stodo != dtodo) {57algt->stat_fb_sgdiff++;58return true;59}60len -= stodo;61sgs = sg_next(sgs);62sgd = sg_next(sgd);63}64return false;65}6667static int rk_cipher_fallback(struct skcipher_request *areq)68{69struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);70struct rk_cipher_ctx *op = crypto_skcipher_ctx(tfm);71struct rk_cipher_rctx *rctx = skcipher_request_ctx(areq);72struct skcipher_alg *alg = crypto_skcipher_alg(tfm);73struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.skcipher.base);74int err;7576algt->stat_fb++;7778skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm);79skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags,80areq->base.complete, areq->base.data);81skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst,82areq->cryptlen, areq->iv);83if (rctx->mode & RK_CRYPTO_DEC)84err = crypto_skcipher_decrypt(&rctx->fallback_req);85else86err = crypto_skcipher_encrypt(&rctx->fallback_req);87return err;88}8990static int rk_cipher_handle_req(struct skcipher_request *req)91{92struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);93struct rk_crypto_info *rkc;94struct crypto_engine *engine;9596if (rk_cipher_need_fallback(req))97return rk_cipher_fallback(req);9899rkc = get_rk_crypto();100101engine = rkc->engine;102rctx->dev = rkc;103104return crypto_transfer_skcipher_request_to_engine(engine, req);105}106107static int rk_aes_setkey(struct crypto_skcipher *cipher,108const u8 *key, unsigned int keylen)109{110struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher);111struct rk_cipher_ctx *ctx = crypto_tfm_ctx(tfm);112113if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&114keylen != AES_KEYSIZE_256)115return -EINVAL;116ctx->keylen = keylen;117memcpy(ctx->key, key, keylen);118119return crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen);120}121122static int rk_des_setkey(struct crypto_skcipher *cipher,123const u8 *key, unsigned int keylen)124{125struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher);126int err;127128err = verify_skcipher_des_key(cipher, key);129if (err)130return err;131132ctx->keylen = keylen;133memcpy(ctx->key, key, keylen);134135return crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen);136}137138static int rk_tdes_setkey(struct crypto_skcipher *cipher,139const u8 *key, unsigned int keylen)140{141struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher);142int err;143144err = verify_skcipher_des3_key(cipher, key);145if (err)146return err;147148ctx->keylen = keylen;149memcpy(ctx->key, key, keylen);150151return crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen);152}153154static int rk_aes_ecb_encrypt(struct skcipher_request *req)155{156struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);157158rctx->mode = RK_CRYPTO_AES_ECB_MODE;159return rk_cipher_handle_req(req);160}161162static int rk_aes_ecb_decrypt(struct skcipher_request *req)163{164struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);165166rctx->mode = RK_CRYPTO_AES_ECB_MODE | RK_CRYPTO_DEC;167return rk_cipher_handle_req(req);168}169170static int rk_aes_cbc_encrypt(struct skcipher_request *req)171{172struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);173174rctx->mode = RK_CRYPTO_AES_CBC_MODE;175return rk_cipher_handle_req(req);176}177178static int rk_aes_cbc_decrypt(struct skcipher_request *req)179{180struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);181182rctx->mode = RK_CRYPTO_AES_CBC_MODE | RK_CRYPTO_DEC;183return rk_cipher_handle_req(req);184}185186static int rk_des_ecb_encrypt(struct skcipher_request *req)187{188struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);189190rctx->mode = 0;191return rk_cipher_handle_req(req);192}193194static int rk_des_ecb_decrypt(struct skcipher_request *req)195{196struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);197198rctx->mode = RK_CRYPTO_DEC;199return rk_cipher_handle_req(req);200}201202static int rk_des_cbc_encrypt(struct skcipher_request *req)203{204struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);205206rctx->mode = RK_CRYPTO_TDES_CHAINMODE_CBC;207return rk_cipher_handle_req(req);208}209210static int rk_des_cbc_decrypt(struct skcipher_request *req)211{212struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);213214rctx->mode = RK_CRYPTO_TDES_CHAINMODE_CBC | RK_CRYPTO_DEC;215return rk_cipher_handle_req(req);216}217218static int rk_des3_ede_ecb_encrypt(struct skcipher_request *req)219{220struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);221222rctx->mode = RK_CRYPTO_TDES_SELECT;223return rk_cipher_handle_req(req);224}225226static int rk_des3_ede_ecb_decrypt(struct skcipher_request *req)227{228struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);229230rctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_DEC;231return rk_cipher_handle_req(req);232}233234static int rk_des3_ede_cbc_encrypt(struct skcipher_request *req)235{236struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);237238rctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC;239return rk_cipher_handle_req(req);240}241242static int rk_des3_ede_cbc_decrypt(struct skcipher_request *req)243{244struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);245246rctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC |247RK_CRYPTO_DEC;248return rk_cipher_handle_req(req);249}250251static void rk_cipher_hw_init(struct rk_crypto_info *dev, struct skcipher_request *req)252{253struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);254struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher);255struct rk_cipher_rctx *rctx = skcipher_request_ctx(req);256struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher);257u32 block, conf_reg = 0;258259block = crypto_tfm_alg_blocksize(tfm);260261if (block == DES_BLOCK_SIZE) {262rctx->mode |= RK_CRYPTO_TDES_FIFO_MODE |263RK_CRYPTO_TDES_BYTESWAP_KEY |264RK_CRYPTO_TDES_BYTESWAP_IV;265CRYPTO_WRITE(dev, RK_CRYPTO_TDES_CTRL, rctx->mode);266memcpy_toio(dev->reg + RK_CRYPTO_TDES_KEY1_0, ctx->key, ctx->keylen);267conf_reg = RK_CRYPTO_DESSEL;268} else {269rctx->mode |= RK_CRYPTO_AES_FIFO_MODE |270RK_CRYPTO_AES_KEY_CHANGE |271RK_CRYPTO_AES_BYTESWAP_KEY |272RK_CRYPTO_AES_BYTESWAP_IV;273if (ctx->keylen == AES_KEYSIZE_192)274rctx->mode |= RK_CRYPTO_AES_192BIT_key;275else if (ctx->keylen == AES_KEYSIZE_256)276rctx->mode |= RK_CRYPTO_AES_256BIT_key;277CRYPTO_WRITE(dev, RK_CRYPTO_AES_CTRL, rctx->mode);278memcpy_toio(dev->reg + RK_CRYPTO_AES_KEY_0, ctx->key, ctx->keylen);279}280conf_reg |= RK_CRYPTO_BYTESWAP_BTFIFO |281RK_CRYPTO_BYTESWAP_BRFIFO;282CRYPTO_WRITE(dev, RK_CRYPTO_CONF, conf_reg);283CRYPTO_WRITE(dev, RK_CRYPTO_INTENA,284RK_CRYPTO_BCDMA_ERR_ENA | RK_CRYPTO_BCDMA_DONE_ENA);285}286287static void crypto_dma_start(struct rk_crypto_info *dev,288struct scatterlist *sgs,289struct scatterlist *sgd, unsigned int todo)290{291CRYPTO_WRITE(dev, RK_CRYPTO_BRDMAS, sg_dma_address(sgs));292CRYPTO_WRITE(dev, RK_CRYPTO_BRDMAL, todo);293CRYPTO_WRITE(dev, RK_CRYPTO_BTDMAS, sg_dma_address(sgd));294CRYPTO_WRITE(dev, RK_CRYPTO_CTRL, RK_CRYPTO_BLOCK_START |295_SBF(RK_CRYPTO_BLOCK_START, 16));296}297298static int rk_cipher_run(struct crypto_engine *engine, void *async_req)299{300struct skcipher_request *areq = container_of(async_req, struct skcipher_request, base);301struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);302struct rk_cipher_rctx *rctx = skcipher_request_ctx(areq);303struct scatterlist *sgs, *sgd;304int err = 0;305int ivsize = crypto_skcipher_ivsize(tfm);306int offset;307u8 iv[AES_BLOCK_SIZE];308u8 biv[AES_BLOCK_SIZE];309u8 *ivtouse = areq->iv;310unsigned int len = areq->cryptlen;311unsigned int todo;312struct skcipher_alg *alg = crypto_skcipher_alg(tfm);313struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.skcipher.base);314struct rk_crypto_info *rkc = rctx->dev;315316err = pm_runtime_resume_and_get(rkc->dev);317if (err)318return err;319320algt->stat_req++;321rkc->nreq++;322323if (areq->iv && ivsize > 0) {324if (rctx->mode & RK_CRYPTO_DEC) {325offset = areq->cryptlen - ivsize;326scatterwalk_map_and_copy(rctx->backup_iv, areq->src,327offset, ivsize, 0);328}329}330331sgs = areq->src;332sgd = areq->dst;333334while (sgs && sgd && len) {335if (!sgs->length) {336sgs = sg_next(sgs);337sgd = sg_next(sgd);338continue;339}340if (rctx->mode & RK_CRYPTO_DEC) {341/* we backup last block of source to be used as IV at next step */342offset = sgs->length - ivsize;343scatterwalk_map_and_copy(biv, sgs, offset, ivsize, 0);344}345if (sgs == sgd) {346err = dma_map_sg(rkc->dev, sgs, 1, DMA_BIDIRECTIONAL);347if (err <= 0) {348err = -EINVAL;349goto theend_iv;350}351} else {352err = dma_map_sg(rkc->dev, sgs, 1, DMA_TO_DEVICE);353if (err <= 0) {354err = -EINVAL;355goto theend_iv;356}357err = dma_map_sg(rkc->dev, sgd, 1, DMA_FROM_DEVICE);358if (err <= 0) {359err = -EINVAL;360goto theend_sgs;361}362}363err = 0;364rk_cipher_hw_init(rkc, areq);365if (ivsize) {366if (ivsize == DES_BLOCK_SIZE)367memcpy_toio(rkc->reg + RK_CRYPTO_TDES_IV_0, ivtouse, ivsize);368else369memcpy_toio(rkc->reg + RK_CRYPTO_AES_IV_0, ivtouse, ivsize);370}371reinit_completion(&rkc->complete);372rkc->status = 0;373374todo = min(sg_dma_len(sgs), len);375len -= todo;376crypto_dma_start(rkc, sgs, sgd, todo / 4);377wait_for_completion_interruptible_timeout(&rkc->complete,378msecs_to_jiffies(2000));379if (!rkc->status) {380dev_err(rkc->dev, "DMA timeout\n");381err = -EFAULT;382goto theend;383}384if (sgs == sgd) {385dma_unmap_sg(rkc->dev, sgs, 1, DMA_BIDIRECTIONAL);386} else {387dma_unmap_sg(rkc->dev, sgs, 1, DMA_TO_DEVICE);388dma_unmap_sg(rkc->dev, sgd, 1, DMA_FROM_DEVICE);389}390if (rctx->mode & RK_CRYPTO_DEC) {391memcpy(iv, biv, ivsize);392ivtouse = iv;393} else {394offset = sgd->length - ivsize;395scatterwalk_map_and_copy(iv, sgd, offset, ivsize, 0);396ivtouse = iv;397}398sgs = sg_next(sgs);399sgd = sg_next(sgd);400}401402if (areq->iv && ivsize > 0) {403offset = areq->cryptlen - ivsize;404if (rctx->mode & RK_CRYPTO_DEC) {405memcpy(areq->iv, rctx->backup_iv, ivsize);406memzero_explicit(rctx->backup_iv, ivsize);407} else {408scatterwalk_map_and_copy(areq->iv, areq->dst, offset,409ivsize, 0);410}411}412413theend:414pm_runtime_put_autosuspend(rkc->dev);415416local_bh_disable();417crypto_finalize_skcipher_request(engine, areq, err);418local_bh_enable();419return 0;420421theend_sgs:422if (sgs == sgd) {423dma_unmap_sg(rkc->dev, sgs, 1, DMA_BIDIRECTIONAL);424} else {425dma_unmap_sg(rkc->dev, sgs, 1, DMA_TO_DEVICE);426dma_unmap_sg(rkc->dev, sgd, 1, DMA_FROM_DEVICE);427}428theend_iv:429return err;430}431432static int rk_cipher_tfm_init(struct crypto_skcipher *tfm)433{434struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(tfm);435const char *name = crypto_tfm_alg_name(&tfm->base);436struct skcipher_alg *alg = crypto_skcipher_alg(tfm);437struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.skcipher.base);438439ctx->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);440if (IS_ERR(ctx->fallback_tfm)) {441dev_err(algt->dev->dev, "ERROR: Cannot allocate fallback for %s %ld\n",442name, PTR_ERR(ctx->fallback_tfm));443return PTR_ERR(ctx->fallback_tfm);444}445446crypto_skcipher_set_reqsize(tfm, sizeof(struct rk_cipher_rctx) +447crypto_skcipher_reqsize(ctx->fallback_tfm));448449return 0;450}451452static void rk_cipher_tfm_exit(struct crypto_skcipher *tfm)453{454struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(tfm);455456memzero_explicit(ctx->key, ctx->keylen);457crypto_free_skcipher(ctx->fallback_tfm);458}459460struct rk_crypto_tmp rk_ecb_aes_alg = {461.type = CRYPTO_ALG_TYPE_SKCIPHER,462.alg.skcipher.base = {463.base.cra_name = "ecb(aes)",464.base.cra_driver_name = "ecb-aes-rk",465.base.cra_priority = 300,466.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,467.base.cra_blocksize = AES_BLOCK_SIZE,468.base.cra_ctxsize = sizeof(struct rk_cipher_ctx),469.base.cra_alignmask = 0x0f,470.base.cra_module = THIS_MODULE,471472.init = rk_cipher_tfm_init,473.exit = rk_cipher_tfm_exit,474.min_keysize = AES_MIN_KEY_SIZE,475.max_keysize = AES_MAX_KEY_SIZE,476.setkey = rk_aes_setkey,477.encrypt = rk_aes_ecb_encrypt,478.decrypt = rk_aes_ecb_decrypt,479},480.alg.skcipher.op = {481.do_one_request = rk_cipher_run,482},483};484485struct rk_crypto_tmp rk_cbc_aes_alg = {486.type = CRYPTO_ALG_TYPE_SKCIPHER,487.alg.skcipher.base = {488.base.cra_name = "cbc(aes)",489.base.cra_driver_name = "cbc-aes-rk",490.base.cra_priority = 300,491.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,492.base.cra_blocksize = AES_BLOCK_SIZE,493.base.cra_ctxsize = sizeof(struct rk_cipher_ctx),494.base.cra_alignmask = 0x0f,495.base.cra_module = THIS_MODULE,496497.init = rk_cipher_tfm_init,498.exit = rk_cipher_tfm_exit,499.min_keysize = AES_MIN_KEY_SIZE,500.max_keysize = AES_MAX_KEY_SIZE,501.ivsize = AES_BLOCK_SIZE,502.setkey = rk_aes_setkey,503.encrypt = rk_aes_cbc_encrypt,504.decrypt = rk_aes_cbc_decrypt,505},506.alg.skcipher.op = {507.do_one_request = rk_cipher_run,508},509};510511struct rk_crypto_tmp rk_ecb_des_alg = {512.type = CRYPTO_ALG_TYPE_SKCIPHER,513.alg.skcipher.base = {514.base.cra_name = "ecb(des)",515.base.cra_driver_name = "ecb-des-rk",516.base.cra_priority = 300,517.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,518.base.cra_blocksize = DES_BLOCK_SIZE,519.base.cra_ctxsize = sizeof(struct rk_cipher_ctx),520.base.cra_alignmask = 0x07,521.base.cra_module = THIS_MODULE,522523.init = rk_cipher_tfm_init,524.exit = rk_cipher_tfm_exit,525.min_keysize = DES_KEY_SIZE,526.max_keysize = DES_KEY_SIZE,527.setkey = rk_des_setkey,528.encrypt = rk_des_ecb_encrypt,529.decrypt = rk_des_ecb_decrypt,530},531.alg.skcipher.op = {532.do_one_request = rk_cipher_run,533},534};535536struct rk_crypto_tmp rk_cbc_des_alg = {537.type = CRYPTO_ALG_TYPE_SKCIPHER,538.alg.skcipher.base = {539.base.cra_name = "cbc(des)",540.base.cra_driver_name = "cbc-des-rk",541.base.cra_priority = 300,542.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,543.base.cra_blocksize = DES_BLOCK_SIZE,544.base.cra_ctxsize = sizeof(struct rk_cipher_ctx),545.base.cra_alignmask = 0x07,546.base.cra_module = THIS_MODULE,547548.init = rk_cipher_tfm_init,549.exit = rk_cipher_tfm_exit,550.min_keysize = DES_KEY_SIZE,551.max_keysize = DES_KEY_SIZE,552.ivsize = DES_BLOCK_SIZE,553.setkey = rk_des_setkey,554.encrypt = rk_des_cbc_encrypt,555.decrypt = rk_des_cbc_decrypt,556},557.alg.skcipher.op = {558.do_one_request = rk_cipher_run,559},560};561562struct rk_crypto_tmp rk_ecb_des3_ede_alg = {563.type = CRYPTO_ALG_TYPE_SKCIPHER,564.alg.skcipher.base = {565.base.cra_name = "ecb(des3_ede)",566.base.cra_driver_name = "ecb-des3-ede-rk",567.base.cra_priority = 300,568.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,569.base.cra_blocksize = DES_BLOCK_SIZE,570.base.cra_ctxsize = sizeof(struct rk_cipher_ctx),571.base.cra_alignmask = 0x07,572.base.cra_module = THIS_MODULE,573574.init = rk_cipher_tfm_init,575.exit = rk_cipher_tfm_exit,576.min_keysize = DES3_EDE_KEY_SIZE,577.max_keysize = DES3_EDE_KEY_SIZE,578.setkey = rk_tdes_setkey,579.encrypt = rk_des3_ede_ecb_encrypt,580.decrypt = rk_des3_ede_ecb_decrypt,581},582.alg.skcipher.op = {583.do_one_request = rk_cipher_run,584},585};586587struct rk_crypto_tmp rk_cbc_des3_ede_alg = {588.type = CRYPTO_ALG_TYPE_SKCIPHER,589.alg.skcipher.base = {590.base.cra_name = "cbc(des3_ede)",591.base.cra_driver_name = "cbc-des3-ede-rk",592.base.cra_priority = 300,593.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,594.base.cra_blocksize = DES_BLOCK_SIZE,595.base.cra_ctxsize = sizeof(struct rk_cipher_ctx),596.base.cra_alignmask = 0x07,597.base.cra_module = THIS_MODULE,598599.init = rk_cipher_tfm_init,600.exit = rk_cipher_tfm_exit,601.min_keysize = DES3_EDE_KEY_SIZE,602.max_keysize = DES3_EDE_KEY_SIZE,603.ivsize = DES_BLOCK_SIZE,604.setkey = rk_tdes_setkey,605.encrypt = rk_des3_ede_cbc_encrypt,606.decrypt = rk_des3_ede_cbc_decrypt,607},608.alg.skcipher.op = {609.do_one_request = rk_cipher_run,610},611};612613614