cryptosoft: Use multi-block encrypt/decrypt for AES-GCM.

Reviewed by:	markj
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D33756
This commit is contained in:
John Baldwin 2022-01-11 14:20:32 -08:00
parent 69e533c3ff
commit a221a8f4a0

View file

@ -411,12 +411,12 @@ swcr_gcm(const struct swcr_session *ses, struct cryptop *crp)
struct crypto_buffer_cursor cc_in, cc_out;
const u_char *inblk;
u_char *outblk;
size_t inlen, outlen, todo;
const struct swcr_auth *swa;
const struct swcr_encdec *swe;
const struct enc_xform *exf;
void *ctx;
uint32_t *blkp;
size_t len;
int blksz, error, ivlen, r, resid;
swa = &ses->swcr_auth;
@ -441,30 +441,30 @@ swcr_gcm(const struct swcr_session *ses, struct cryptop *crp)
/* Supply MAC with AAD */
if (crp->crp_aad != NULL) {
len = rounddown(crp->crp_aad_length, blksz);
if (len != 0)
exf->update(ctx, crp->crp_aad, len);
if (crp->crp_aad_length != len) {
inlen = rounddown2(crp->crp_aad_length, blksz);
if (inlen != 0)
exf->update(ctx, crp->crp_aad, inlen);
if (crp->crp_aad_length != inlen) {
memset(blk, 0, blksz);
memcpy(blk, (char *)crp->crp_aad + len,
crp->crp_aad_length - len);
memcpy(blk, (char *)crp->crp_aad + inlen,
crp->crp_aad_length - inlen);
exf->update(ctx, blk, blksz);
}
} else {
crypto_cursor_init(&cc_in, &crp->crp_buf);
crypto_cursor_advance(&cc_in, crp->crp_aad_start);
for (resid = crp->crp_aad_length; resid >= blksz;
resid -= len) {
inblk = crypto_cursor_segment(&cc_in, &len);
if (len >= blksz) {
len = rounddown(MIN(len, resid), blksz);
crypto_cursor_advance(&cc_in, len);
resid -= inlen) {
inblk = crypto_cursor_segment(&cc_in, &inlen);
if (inlen >= blksz) {
inlen = rounddown2(MIN(inlen, resid), blksz);
crypto_cursor_advance(&cc_in, inlen);
} else {
len = blksz;
crypto_cursor_copydata(&cc_in, len, blk);
inlen = blksz;
crypto_cursor_copydata(&cc_in, inlen, blk);
inblk = blk;
}
exf->update(ctx, inblk, len);
exf->update(ctx, inblk, inlen);
}
if (resid > 0) {
memset(blk, 0, blksz);
@ -476,31 +476,57 @@ swcr_gcm(const struct swcr_session *ses, struct cryptop *crp)
/* Do encryption with MAC */
crypto_cursor_init(&cc_in, &crp->crp_buf);
crypto_cursor_advance(&cc_in, crp->crp_payload_start);
inblk = crypto_cursor_segment(&cc_in, &inlen);
if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
crypto_cursor_init(&cc_out, &crp->crp_obuf);
crypto_cursor_advance(&cc_out, crp->crp_payload_output_start);
} else
cc_out = cc_in;
for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) {
inblk = crypto_cursor_segment(&cc_in, &len);
if (len < blksz) {
outblk = crypto_cursor_segment(&cc_out, &outlen);
for (resid = crp->crp_payload_length; resid >= blksz; resid -= todo) {
if (inlen < blksz) {
crypto_cursor_copydata(&cc_in, blksz, blk);
inblk = blk;
} else {
crypto_cursor_advance(&cc_in, blksz);
inlen = blksz;
}
if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
outblk = crypto_cursor_segment(&cc_out, &len);
if (len < blksz)
if (outlen < blksz) {
outblk = blk;
exf->encrypt(ctx, inblk, outblk);
exf->update(ctx, outblk, blksz);
if (outblk == blk)
outlen = blksz;
}
todo = rounddown2(MIN(resid, MIN(inlen, outlen)),
blksz);
exf->encrypt_multi(ctx, inblk, outblk, todo);
exf->update(ctx, outblk, todo);
if (outblk == blk) {
crypto_cursor_copyback(&cc_out, blksz, blk);
else
crypto_cursor_advance(&cc_out, blksz);
outblk = crypto_cursor_segment(&cc_out, &outlen);
} else {
crypto_cursor_advance(&cc_out, todo);
outlen -= todo;
outblk += todo;
if (outlen == 0)
outblk = crypto_cursor_segment(&cc_out,
&outlen);
}
} else {
exf->update(ctx, inblk, blksz);
todo = rounddown2(MIN(resid, inlen), blksz);
exf->update(ctx, inblk, todo);
}
if (inblk == blk) {
inblk = crypto_cursor_segment(&cc_in, &inlen);
} else {
crypto_cursor_advance(&cc_in, todo);
inlen -= todo;
inblk += todo;
if (inlen == 0)
inblk = crypto_cursor_segment(&cc_in, &inlen);
}
}
if (resid > 0) {
@ -537,22 +563,48 @@ swcr_gcm(const struct swcr_session *ses, struct cryptop *crp)
/* tag matches, decrypt data */
crypto_cursor_init(&cc_in, &crp->crp_buf);
crypto_cursor_advance(&cc_in, crp->crp_payload_start);
inblk = crypto_cursor_segment(&cc_in, &inlen);
for (resid = crp->crp_payload_length; resid > blksz;
resid -= blksz) {
inblk = crypto_cursor_segment(&cc_in, &len);
if (len < blksz) {
resid -= todo) {
if (inlen < blksz) {
crypto_cursor_copydata(&cc_in, blksz, blk);
inblk = blk;
} else
crypto_cursor_advance(&cc_in, blksz);
outblk = crypto_cursor_segment(&cc_out, &len);
if (len < blksz)
inlen = blksz;
}
if (outlen < blksz) {
outblk = blk;
exf->decrypt(ctx, inblk, outblk);
if (outblk == blk)
outlen = blksz;
}
todo = rounddown2(MIN(resid, MIN(inlen, outlen)),
blksz);
exf->decrypt_multi(ctx, inblk, outblk, todo);
if (inblk == blk) {
inblk = crypto_cursor_segment(&cc_in, &inlen);
} else {
crypto_cursor_advance(&cc_in, todo);
inlen -= todo;
inblk += todo;
if (inlen == 0)
inblk = crypto_cursor_segment(&cc_in,
&inlen);
}
if (outblk == blk) {
crypto_cursor_copyback(&cc_out, blksz, blk);
else
crypto_cursor_advance(&cc_out, blksz);
outblk = crypto_cursor_segment(&cc_out,
&outlen);
} else {
crypto_cursor_advance(&cc_out, todo);
outlen -= todo;
outblk += todo;
if (outlen == 0)
outblk = crypto_cursor_segment(&cc_out,
&outlen);
}
}
if (resid > 0) {
crypto_cursor_copydata(&cc_in, resid, blk);