aboutsummaryrefslogtreecommitdiff
path: root/target
diff options
context:
space:
mode:
authorRichard Henderson <richard.henderson@linaro.org>2023-07-09 15:01:43 +0100
committerRichard Henderson <richard.henderson@linaro.org>2023-07-09 15:01:43 +0100
commit2ff49e96accc8fd9a38e9abd16f0cfa0adab1605 (patch)
tree2de7e51955f62162978b94d62b8b1667f65dc507 /target
parentfc61742e445c4ebfe3932063f683d99c6d737cee (diff)
parentff494c8e2a4c857dd37fb908d8ac8158f5e4f89b (diff)
downloadqemu-2ff49e96accc8fd9a38e9abd16f0cfa0adab1605.zip
qemu-2ff49e96accc8fd9a38e9abd16f0cfa0adab1605.tar.gz
qemu-2ff49e96accc8fd9a38e9abd16f0cfa0adab1605.tar.bz2
Merge tag 'pull-tcg-20230709' of https://gitlab.com/rth7680/qemu into staging
crypto: Provide aes-round.h and host accel # -----BEGIN PGP SIGNATURE----- # # iQFRBAABCgA7FiEEekgeeIaLTbaoWgXAZN846K9+IV8FAmSqvGodHHJpY2hhcmQu # aGVuZGVyc29uQGxpbmFyby5vcmcACgkQZN846K9+IV+1bgf9EG57jfnCQLCfMQ6C # 0bQ0MaeAkGg+7+mUwyi3OPB1VO0yjEKv5pWEnolzrGud35P0KsyoO+msqGqxnbMv # IbhPkQZbmfMsGFPG1DbswjiwmQU5cV+ciONDM+C+qepnuUN+JrzIDHoEFzQRFoQo # eQL/LnuyUkYBvR7YCKNJxFHtwILKcYQPH4jiC6a92C11AzYjDfilSxnuQ2RwL3Tn # Zwf8TKJP5QGExvUdtm8f6xJ1LT7WAvsk9ZTwudE/+XRTnw8RWk6RmZSEQPx+cBdI # p3opaoxkkMrdmcaXbr+9eSfBGq2gsVkKYPiyTDuwVW26575Nob9ZmodT3oSBNlkC # +njd4w== # =Nf5i # -----END PGP SIGNATURE----- # gpg: Signature made Sun 09 Jul 2023 02:55:54 PM BST # gpg: using RSA key 7A481E78868B4DB6A85A05C064DF38E8AF7E215F # gpg: issuer "richard.henderson@linaro.org" # gpg: Good signature from "Richard Henderson <richard.henderson@linaro.org>" [ultimate] * tag 'pull-tcg-20230709' of https://gitlab.com/rth7680/qemu: (37 commits) crypto: Unexport AES_*_rot, AES_TeN, AES_TdN crypto: Remove AES_imc crypto: Implement aesdec_IMC with AES_imc_rot crypto: Remove AES_shifts, AES_ishifts target/riscv: Use aesdec_ISB_ISR_IMC_AK target/riscv: Use aesenc_SB_SR_MC_AK target/riscv: Use aesdec_IMC target/riscv: Use aesdec_ISB_ISR_AK target/riscv: Use aesenc_SB_SR_AK target/arm: Use aesdec_IMC target/arm: Use aesenc_MC target/arm: Use aesdec_ISB_ISR_AK target/arm: Use aesenc_SB_SR_AK target/arm: Demultiplex AESE and AESMC target/i386: Use aesdec_ISB_ISR_IMC_AK target/i386: Use aesenc_SB_SR_MC_AK target/i386: Use aesdec_IMC target/i386: Use aesdec_ISB_ISR_AK target/i386: Use aesenc_SB_SR_AK target/ppc: Use aesdec_ISB_ISR_AK_IMC ... Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
Diffstat (limited to 'target')
-rw-r--r--target/arm/helper.h2
-rw-r--r--target/arm/tcg/crypto_helper.c249
-rw-r--r--target/arm/tcg/sve.decode4
-rw-r--r--target/arm/tcg/translate-a64.c13
-rw-r--r--target/arm/tcg/translate-neon.c4
-rw-r--r--target/arm/tcg/translate-sve.c8
-rw-r--r--target/i386/ops_sse.h60
-rw-r--r--target/ppc/int_helper.c50
-rw-r--r--target/riscv/crypto_helper.c138
9 files changed, 163 insertions, 365 deletions
diff --git a/target/arm/helper.h b/target/arm/helper.h
index 3335c2b..95e32a6 100644
--- a/target/arm/helper.h
+++ b/target/arm/helper.h
@@ -552,7 +552,9 @@ DEF_HELPER_FLAGS_2(neon_qzip16, TCG_CALL_NO_RWG, void, ptr, ptr)
DEF_HELPER_FLAGS_2(neon_qzip32, TCG_CALL_NO_RWG, void, ptr, ptr)
DEF_HELPER_FLAGS_4(crypto_aese, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(crypto_aesd, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_3(crypto_aesmc, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(crypto_aesimc, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(crypto_sha1su0, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(crypto_sha1c, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
diff --git a/target/arm/tcg/crypto_helper.c b/target/arm/tcg/crypto_helper.c
index d286903..fdd70ab 100644
--- a/target/arm/tcg/crypto_helper.c
+++ b/target/arm/tcg/crypto_helper.c
@@ -14,7 +14,7 @@
#include "cpu.h"
#include "exec/helper-proto.h"
#include "tcg/tcg-gvec-desc.h"
-#include "crypto/aes.h"
+#include "crypto/aes-round.h"
#include "crypto/sm4.h"
#include "vec_internal.h"
@@ -45,197 +45,104 @@ static void clear_tail_16(void *vd, uint32_t desc)
clear_tail(vd, opr_sz, max_sz);
}
-static void do_crypto_aese(uint64_t *rd, uint64_t *rn,
- uint64_t *rm, bool decrypt)
-{
- static uint8_t const * const sbox[2] = { AES_sbox, AES_isbox };
- static uint8_t const * const shift[2] = { AES_shifts, AES_ishifts };
- union CRYPTO_STATE rk = { .l = { rm[0], rm[1] } };
- union CRYPTO_STATE st = { .l = { rn[0], rn[1] } };
- int i;
+static const AESState aes_zero = { };
- /* xor state vector with round key */
- rk.l[0] ^= st.l[0];
- rk.l[1] ^= st.l[1];
+void HELPER(crypto_aese)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
- /* combine ShiftRows operation and sbox substitution */
- for (i = 0; i < 16; i++) {
- CR_ST_BYTE(st, i) = sbox[decrypt][CR_ST_BYTE(rk, shift[decrypt][i])];
+ for (i = 0; i < opr_sz; i += 16) {
+ AESState *ad = (AESState *)(vd + i);
+ AESState *st = (AESState *)(vn + i);
+ AESState *rk = (AESState *)(vm + i);
+ AESState t;
+
+ /*
+ * Our uint64_t are in the wrong order for big-endian.
+ * The Arm AddRoundKey comes first, while the API AddRoundKey
+ * comes last: perform the xor here, and provide zero to API.
+ */
+ if (HOST_BIG_ENDIAN) {
+ t.d[0] = st->d[1] ^ rk->d[1];
+ t.d[1] = st->d[0] ^ rk->d[0];
+ aesenc_SB_SR_AK(&t, &t, &aes_zero, false);
+ ad->d[0] = t.d[1];
+ ad->d[1] = t.d[0];
+ } else {
+ t.v = st->v ^ rk->v;
+ aesenc_SB_SR_AK(ad, &t, &aes_zero, false);
+ }
}
-
- rd[0] = st.l[0];
- rd[1] = st.l[1];
+ clear_tail(vd, opr_sz, simd_maxsz(desc));
}
-void HELPER(crypto_aese)(void *vd, void *vn, void *vm, uint32_t desc)
+void HELPER(crypto_aesd)(void *vd, void *vn, void *vm, uint32_t desc)
{
intptr_t i, opr_sz = simd_oprsz(desc);
- bool decrypt = simd_data(desc);
for (i = 0; i < opr_sz; i += 16) {
- do_crypto_aese(vd + i, vn + i, vm + i, decrypt);
+ AESState *ad = (AESState *)(vd + i);
+ AESState *st = (AESState *)(vn + i);
+ AESState *rk = (AESState *)(vm + i);
+ AESState t;
+
+ /* Our uint64_t are in the wrong order for big-endian. */
+ if (HOST_BIG_ENDIAN) {
+ t.d[0] = st->d[1] ^ rk->d[1];
+ t.d[1] = st->d[0] ^ rk->d[0];
+ aesdec_ISB_ISR_AK(&t, &t, &aes_zero, false);
+ ad->d[0] = t.d[1];
+ ad->d[1] = t.d[0];
+ } else {
+ t.v = st->v ^ rk->v;
+ aesdec_ISB_ISR_AK(ad, &t, &aes_zero, false);
+ }
}
clear_tail(vd, opr_sz, simd_maxsz(desc));
}
-static void do_crypto_aesmc(uint64_t *rd, uint64_t *rm, bool decrypt)
-{
- static uint32_t const mc[][256] = { {
- /* MixColumns lookup table */
- 0x00000000, 0x03010102, 0x06020204, 0x05030306,
- 0x0c040408, 0x0f05050a, 0x0a06060c, 0x0907070e,
- 0x18080810, 0x1b090912, 0x1e0a0a14, 0x1d0b0b16,
- 0x140c0c18, 0x170d0d1a, 0x120e0e1c, 0x110f0f1e,
- 0x30101020, 0x33111122, 0x36121224, 0x35131326,
- 0x3c141428, 0x3f15152a, 0x3a16162c, 0x3917172e,
- 0x28181830, 0x2b191932, 0x2e1a1a34, 0x2d1b1b36,
- 0x241c1c38, 0x271d1d3a, 0x221e1e3c, 0x211f1f3e,
- 0x60202040, 0x63212142, 0x66222244, 0x65232346,
- 0x6c242448, 0x6f25254a, 0x6a26264c, 0x6927274e,
- 0x78282850, 0x7b292952, 0x7e2a2a54, 0x7d2b2b56,
- 0x742c2c58, 0x772d2d5a, 0x722e2e5c, 0x712f2f5e,
- 0x50303060, 0x53313162, 0x56323264, 0x55333366,
- 0x5c343468, 0x5f35356a, 0x5a36366c, 0x5937376e,
- 0x48383870, 0x4b393972, 0x4e3a3a74, 0x4d3b3b76,
- 0x443c3c78, 0x473d3d7a, 0x423e3e7c, 0x413f3f7e,
- 0xc0404080, 0xc3414182, 0xc6424284, 0xc5434386,
- 0xcc444488, 0xcf45458a, 0xca46468c, 0xc947478e,
- 0xd8484890, 0xdb494992, 0xde4a4a94, 0xdd4b4b96,
- 0xd44c4c98, 0xd74d4d9a, 0xd24e4e9c, 0xd14f4f9e,
- 0xf05050a0, 0xf35151a2, 0xf65252a4, 0xf55353a6,
- 0xfc5454a8, 0xff5555aa, 0xfa5656ac, 0xf95757ae,
- 0xe85858b0, 0xeb5959b2, 0xee5a5ab4, 0xed5b5bb6,
- 0xe45c5cb8, 0xe75d5dba, 0xe25e5ebc, 0xe15f5fbe,
- 0xa06060c0, 0xa36161c2, 0xa66262c4, 0xa56363c6,
- 0xac6464c8, 0xaf6565ca, 0xaa6666cc, 0xa96767ce,
- 0xb86868d0, 0xbb6969d2, 0xbe6a6ad4, 0xbd6b6bd6,
- 0xb46c6cd8, 0xb76d6dda, 0xb26e6edc, 0xb16f6fde,
- 0x907070e0, 0x937171e2, 0x967272e4, 0x957373e6,
- 0x9c7474e8, 0x9f7575ea, 0x9a7676ec, 0x997777ee,
- 0x887878f0, 0x8b7979f2, 0x8e7a7af4, 0x8d7b7bf6,
- 0x847c7cf8, 0x877d7dfa, 0x827e7efc, 0x817f7ffe,
- 0x9b80801b, 0x98818119, 0x9d82821f, 0x9e83831d,
- 0x97848413, 0x94858511, 0x91868617, 0x92878715,
- 0x8388880b, 0x80898909, 0x858a8a0f, 0x868b8b0d,
- 0x8f8c8c03, 0x8c8d8d01, 0x898e8e07, 0x8a8f8f05,
- 0xab90903b, 0xa8919139, 0xad92923f, 0xae93933d,
- 0xa7949433, 0xa4959531, 0xa1969637, 0xa2979735,
- 0xb398982b, 0xb0999929, 0xb59a9a2f, 0xb69b9b2d,
- 0xbf9c9c23, 0xbc9d9d21, 0xb99e9e27, 0xba9f9f25,
- 0xfba0a05b, 0xf8a1a159, 0xfda2a25f, 0xfea3a35d,
- 0xf7a4a453, 0xf4a5a551, 0xf1a6a657, 0xf2a7a755,
- 0xe3a8a84b, 0xe0a9a949, 0xe5aaaa4f, 0xe6abab4d,
- 0xefacac43, 0xecadad41, 0xe9aeae47, 0xeaafaf45,
- 0xcbb0b07b, 0xc8b1b179, 0xcdb2b27f, 0xceb3b37d,
- 0xc7b4b473, 0xc4b5b571, 0xc1b6b677, 0xc2b7b775,
- 0xd3b8b86b, 0xd0b9b969, 0xd5baba6f, 0xd6bbbb6d,
- 0xdfbcbc63, 0xdcbdbd61, 0xd9bebe67, 0xdabfbf65,
- 0x5bc0c09b, 0x58c1c199, 0x5dc2c29f, 0x5ec3c39d,
- 0x57c4c493, 0x54c5c591, 0x51c6c697, 0x52c7c795,
- 0x43c8c88b, 0x40c9c989, 0x45caca8f, 0x46cbcb8d,
- 0x4fcccc83, 0x4ccdcd81, 0x49cece87, 0x4acfcf85,
- 0x6bd0d0bb, 0x68d1d1b9, 0x6dd2d2bf, 0x6ed3d3bd,
- 0x67d4d4b3, 0x64d5d5b1, 0x61d6d6b7, 0x62d7d7b5,
- 0x73d8d8ab, 0x70d9d9a9, 0x75dadaaf, 0x76dbdbad,
- 0x7fdcdca3, 0x7cdddda1, 0x79dedea7, 0x7adfdfa5,
- 0x3be0e0db, 0x38e1e1d9, 0x3de2e2df, 0x3ee3e3dd,
- 0x37e4e4d3, 0x34e5e5d1, 0x31e6e6d7, 0x32e7e7d5,
- 0x23e8e8cb, 0x20e9e9c9, 0x25eaeacf, 0x26ebebcd,
- 0x2fececc3, 0x2cededc1, 0x29eeeec7, 0x2aefefc5,
- 0x0bf0f0fb, 0x08f1f1f9, 0x0df2f2ff, 0x0ef3f3fd,
- 0x07f4f4f3, 0x04f5f5f1, 0x01f6f6f7, 0x02f7f7f5,
- 0x13f8f8eb, 0x10f9f9e9, 0x15fafaef, 0x16fbfbed,
- 0x1ffcfce3, 0x1cfdfde1, 0x19fefee7, 0x1affffe5,
- }, {
- /* Inverse MixColumns lookup table */
- 0x00000000, 0x0b0d090e, 0x161a121c, 0x1d171b12,
- 0x2c342438, 0x27392d36, 0x3a2e3624, 0x31233f2a,
- 0x58684870, 0x5365417e, 0x4e725a6c, 0x457f5362,
- 0x745c6c48, 0x7f516546, 0x62467e54, 0x694b775a,
- 0xb0d090e0, 0xbbdd99ee, 0xa6ca82fc, 0xadc78bf2,
- 0x9ce4b4d8, 0x97e9bdd6, 0x8afea6c4, 0x81f3afca,
- 0xe8b8d890, 0xe3b5d19e, 0xfea2ca8c, 0xf5afc382,
- 0xc48cfca8, 0xcf81f5a6, 0xd296eeb4, 0xd99be7ba,
- 0x7bbb3bdb, 0x70b632d5, 0x6da129c7, 0x66ac20c9,
- 0x578f1fe3, 0x5c8216ed, 0x41950dff, 0x4a9804f1,
- 0x23d373ab, 0x28de7aa5, 0x35c961b7, 0x3ec468b9,
- 0x0fe75793, 0x04ea5e9d, 0x19fd458f, 0x12f04c81,
- 0xcb6bab3b, 0xc066a235, 0xdd71b927, 0xd67cb029,
- 0xe75f8f03, 0xec52860d, 0xf1459d1f, 0xfa489411,
- 0x9303e34b, 0x980eea45, 0x8519f157, 0x8e14f859,
- 0xbf37c773, 0xb43ace7d, 0xa92dd56f, 0xa220dc61,
- 0xf66d76ad, 0xfd607fa3, 0xe07764b1, 0xeb7a6dbf,
- 0xda595295, 0xd1545b9b, 0xcc434089, 0xc74e4987,
- 0xae053edd, 0xa50837d3, 0xb81f2cc1, 0xb31225cf,
- 0x82311ae5, 0x893c13eb, 0x942b08f9, 0x9f2601f7,
- 0x46bde64d, 0x4db0ef43, 0x50a7f451, 0x5baafd5f,
- 0x6a89c275, 0x6184cb7b, 0x7c93d069, 0x779ed967,
- 0x1ed5ae3d, 0x15d8a733, 0x08cfbc21, 0x03c2b52f,
- 0x32e18a05, 0x39ec830b, 0x24fb9819, 0x2ff69117,
- 0x8dd64d76, 0x86db4478, 0x9bcc5f6a, 0x90c15664,
- 0xa1e2694e, 0xaaef6040, 0xb7f87b52, 0xbcf5725c,
- 0xd5be0506, 0xdeb30c08, 0xc3a4171a, 0xc8a91e14,
- 0xf98a213e, 0xf2872830, 0xef903322, 0xe49d3a2c,
- 0x3d06dd96, 0x360bd498, 0x2b1ccf8a, 0x2011c684,
- 0x1132f9ae, 0x1a3ff0a0, 0x0728ebb2, 0x0c25e2bc,
- 0x656e95e6, 0x6e639ce8, 0x737487fa, 0x78798ef4,
- 0x495ab1de, 0x4257b8d0, 0x5f40a3c2, 0x544daacc,
- 0xf7daec41, 0xfcd7e54f, 0xe1c0fe5d, 0xeacdf753,
- 0xdbeec879, 0xd0e3c177, 0xcdf4da65, 0xc6f9d36b,
- 0xafb2a431, 0xa4bfad3f, 0xb9a8b62d, 0xb2a5bf23,
- 0x83868009, 0x888b8907, 0x959c9215, 0x9e919b1b,
- 0x470a7ca1, 0x4c0775af, 0x51106ebd, 0x5a1d67b3,
- 0x6b3e5899, 0x60335197, 0x7d244a85, 0x7629438b,
- 0x1f6234d1, 0x146f3ddf, 0x097826cd, 0x02752fc3,
- 0x335610e9, 0x385b19e7, 0x254c02f5, 0x2e410bfb,
- 0x8c61d79a, 0x876cde94, 0x9a7bc586, 0x9176cc88,
- 0xa055f3a2, 0xab58faac, 0xb64fe1be, 0xbd42e8b0,
- 0xd4099fea, 0xdf0496e4, 0xc2138df6, 0xc91e84f8,
- 0xf83dbbd2, 0xf330b2dc, 0xee27a9ce, 0xe52aa0c0,
- 0x3cb1477a, 0x37bc4e74, 0x2aab5566, 0x21a65c68,
- 0x10856342, 0x1b886a4c, 0x069f715e, 0x0d927850,
- 0x64d90f0a, 0x6fd40604, 0x72c31d16, 0x79ce1418,
- 0x48ed2b32, 0x43e0223c, 0x5ef7392e, 0x55fa3020,
- 0x01b79aec, 0x0aba93e2, 0x17ad88f0, 0x1ca081fe,
- 0x2d83bed4, 0x268eb7da, 0x3b99acc8, 0x3094a5c6,
- 0x59dfd29c, 0x52d2db92, 0x4fc5c080, 0x44c8c98e,
- 0x75ebf6a4, 0x7ee6ffaa, 0x63f1e4b8, 0x68fcedb6,
- 0xb1670a0c, 0xba6a0302, 0xa77d1810, 0xac70111e,
- 0x9d532e34, 0x965e273a, 0x8b493c28, 0x80443526,
- 0xe90f427c, 0xe2024b72, 0xff155060, 0xf418596e,
- 0xc53b6644, 0xce366f4a, 0xd3217458, 0xd82c7d56,
- 0x7a0ca137, 0x7101a839, 0x6c16b32b, 0x671bba25,
- 0x5638850f, 0x5d358c01, 0x40229713, 0x4b2f9e1d,
- 0x2264e947, 0x2969e049, 0x347efb5b, 0x3f73f255,
- 0x0e50cd7f, 0x055dc471, 0x184adf63, 0x1347d66d,
- 0xcadc31d7, 0xc1d138d9, 0xdcc623cb, 0xd7cb2ac5,
- 0xe6e815ef, 0xede51ce1, 0xf0f207f3, 0xfbff0efd,
- 0x92b479a7, 0x99b970a9, 0x84ae6bbb, 0x8fa362b5,
- 0xbe805d9f, 0xb58d5491, 0xa89a4f83, 0xa397468d,
- } };
-
- union CRYPTO_STATE st = { .l = { rm[0], rm[1] } };
- int i;
+void HELPER(crypto_aesmc)(void *vd, void *vm, uint32_t desc)
+{
+ intptr_t i, opr_sz = simd_oprsz(desc);
- for (i = 0; i < 16; i += 4) {
- CR_ST_WORD(st, i >> 2) =
- mc[decrypt][CR_ST_BYTE(st, i)] ^
- rol32(mc[decrypt][CR_ST_BYTE(st, i + 1)], 8) ^
- rol32(mc[decrypt][CR_ST_BYTE(st, i + 2)], 16) ^
- rol32(mc[decrypt][CR_ST_BYTE(st, i + 3)], 24);
+ for (i = 0; i < opr_sz; i += 16) {
+ AESState *ad = (AESState *)(vd + i);
+ AESState *st = (AESState *)(vm + i);
+ AESState t;
+
+ /* Our uint64_t are in the wrong order for big-endian. */
+ if (HOST_BIG_ENDIAN) {
+ t.d[0] = st->d[1];
+ t.d[1] = st->d[0];
+ aesenc_MC(&t, &t, false);
+ ad->d[0] = t.d[1];
+ ad->d[1] = t.d[0];
+ } else {
+ aesenc_MC(ad, st, false);
+ }
}
-
- rd[0] = st.l[0];
- rd[1] = st.l[1];
+ clear_tail(vd, opr_sz, simd_maxsz(desc));
}
-void HELPER(crypto_aesmc)(void *vd, void *vm, uint32_t desc)
+void HELPER(crypto_aesimc)(void *vd, void *vm, uint32_t desc)
{
intptr_t i, opr_sz = simd_oprsz(desc);
- bool decrypt = simd_data(desc);
for (i = 0; i < opr_sz; i += 16) {
- do_crypto_aesmc(vd + i, vm + i, decrypt);
+ AESState *ad = (AESState *)(vd + i);
+ AESState *st = (AESState *)(vm + i);
+ AESState t;
+
+ /* Our uint64_t are in the wrong order for big-endian. */
+ if (HOST_BIG_ENDIAN) {
+ t.d[0] = st->d[1];
+ t.d[1] = st->d[0];
+ aesdec_IMC(&t, &t, false);
+ ad->d[0] = t.d[1];
+ ad->d[1] = t.d[0];
+ } else {
+ aesdec_IMC(ad, st, false);
+ }
}
clear_tail(vd, opr_sz, simd_maxsz(desc));
}
diff --git a/target/arm/tcg/sve.decode b/target/arm/tcg/sve.decode
index 14b3a69..04b6fcc 100644
--- a/target/arm/tcg/sve.decode
+++ b/target/arm/tcg/sve.decode
@@ -1629,8 +1629,8 @@ STNT1_zprz 1110010 .. 10 ..... 001 ... ..... ..... \
### SVE2 Crypto Extensions
# SVE2 crypto unary operations
-# AESMC and AESIMC
-AESMC 01000101 00 10000011100 decrypt:1 00000 rd:5
+AESMC 01000101 00 10000011100 0 00000 rd:5
+AESIMC 01000101 00 10000011100 1 00000 rd:5
# SVE2 crypto destructive binary operations
AESE 01000101 00 10001 0 11100 0 ..... ..... @rdn_rm_e0
diff --git a/target/arm/tcg/translate-a64.c b/target/arm/tcg/translate-a64.c
index 3baab6a..7d0c8f7 100644
--- a/target/arm/tcg/translate-a64.c
+++ b/target/arm/tcg/translate-a64.c
@@ -13210,7 +13210,6 @@ static void disas_crypto_aes(DisasContext *s, uint32_t insn)
int opcode = extract32(insn, 12, 5);
int rn = extract32(insn, 5, 5);
int rd = extract32(insn, 0, 5);
- int decrypt;
gen_helper_gvec_2 *genfn2 = NULL;
gen_helper_gvec_3 *genfn3 = NULL;
@@ -13221,20 +13220,16 @@ static void disas_crypto_aes(DisasContext *s, uint32_t insn)
switch (opcode) {
case 0x4: /* AESE */
- decrypt = 0;
genfn3 = gen_helper_crypto_aese;
break;
case 0x6: /* AESMC */
- decrypt = 0;
genfn2 = gen_helper_crypto_aesmc;
break;
case 0x5: /* AESD */
- decrypt = 1;
- genfn3 = gen_helper_crypto_aese;
+ genfn3 = gen_helper_crypto_aesd;
break;
case 0x7: /* AESIMC */
- decrypt = 1;
- genfn2 = gen_helper_crypto_aesmc;
+ genfn2 = gen_helper_crypto_aesimc;
break;
default:
unallocated_encoding(s);
@@ -13245,9 +13240,9 @@ static void disas_crypto_aes(DisasContext *s, uint32_t insn)
return;
}
if (genfn2) {
- gen_gvec_op2_ool(s, true, rd, rn, decrypt, genfn2);
+ gen_gvec_op2_ool(s, true, rd, rn, 0, genfn2);
} else {
- gen_gvec_op3_ool(s, true, rd, rd, rn, decrypt, genfn3);
+ gen_gvec_op3_ool(s, true, rd, rd, rn, 0, genfn3);
}
}
diff --git a/target/arm/tcg/translate-neon.c b/target/arm/tcg/translate-neon.c
index 03913de..8de4ceb 100644
--- a/target/arm/tcg/translate-neon.c
+++ b/target/arm/tcg/translate-neon.c
@@ -3451,9 +3451,9 @@ static bool trans_VMVN(DisasContext *s, arg_2misc *a)
}
WRAP_2M_3_OOL_FN(gen_AESE, gen_helper_crypto_aese, 0)
-WRAP_2M_3_OOL_FN(gen_AESD, gen_helper_crypto_aese, 1)
+WRAP_2M_3_OOL_FN(gen_AESD, gen_helper_crypto_aesd, 0)
WRAP_2M_2_OOL_FN(gen_AESMC, gen_helper_crypto_aesmc, 0)
-WRAP_2M_2_OOL_FN(gen_AESIMC, gen_helper_crypto_aesmc, 1)
+WRAP_2M_2_OOL_FN(gen_AESIMC, gen_helper_crypto_aesimc, 0)
WRAP_2M_2_OOL_FN(gen_SHA1H, gen_helper_crypto_sha1h, 0)
WRAP_2M_2_OOL_FN(gen_SHA1SU1, gen_helper_crypto_sha1su1, 0)
WRAP_2M_2_OOL_FN(gen_SHA256SU0, gen_helper_crypto_sha256su0, 0)
diff --git a/target/arm/tcg/translate-sve.c b/target/arm/tcg/translate-sve.c
index 225d358..8350a65 100644
--- a/target/arm/tcg/translate-sve.c
+++ b/target/arm/tcg/translate-sve.c
@@ -7151,12 +7151,14 @@ TRANS_FEAT(USDOT_zzzz, aa64_sve_i8mm, gen_gvec_ool_arg_zzzz,
a->esz == 2 ? gen_helper_gvec_usdot_b : NULL, a, 0)
TRANS_FEAT_NONSTREAMING(AESMC, aa64_sve2_aes, gen_gvec_ool_zz,
- gen_helper_crypto_aesmc, a->rd, a->rd, a->decrypt)
+ gen_helper_crypto_aesmc, a->rd, a->rd, 0)
+TRANS_FEAT_NONSTREAMING(AESIMC, aa64_sve2_aes, gen_gvec_ool_zz,
+ gen_helper_crypto_aesimc, a->rd, a->rd, 0)
TRANS_FEAT_NONSTREAMING(AESE, aa64_sve2_aes, gen_gvec_ool_arg_zzz,
- gen_helper_crypto_aese, a, false)
+ gen_helper_crypto_aese, a, 0)
TRANS_FEAT_NONSTREAMING(AESD, aa64_sve2_aes, gen_gvec_ool_arg_zzz,
- gen_helper_crypto_aese, a, true)
+ gen_helper_crypto_aesd, a, 0)
TRANS_FEAT_NONSTREAMING(SM4E, aa64_sve2_sm4, gen_gvec_ool_arg_zzz,
gen_helper_crypto_sm4e, a, 0)
diff --git a/target/i386/ops_sse.h b/target/i386/ops_sse.h
index fb63af7..a0e4257 100644
--- a/target/i386/ops_sse.h
+++ b/target/i386/ops_sse.h
@@ -19,6 +19,7 @@
*/
#include "crypto/aes.h"
+#include "crypto/aes-round.h"
#if SHIFT == 0
#define Reg MMXReg
@@ -2161,68 +2162,55 @@ void glue(helper_pclmulqdq, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s,
void glue(helper_aesdec, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s)
{
- int i;
- Reg st = *v;
- Reg rk = *s;
+ for (int i = 0; i < SHIFT; i++) {
+ AESState *ad = (AESState *)&d->ZMM_X(i);
+ AESState *st = (AESState *)&v->ZMM_X(i);
+ AESState *rk = (AESState *)&s->ZMM_X(i);
- for (i = 0 ; i < 2 << SHIFT ; i++) {
- int j = i & 3;
- d->L(i) = rk.L(i) ^ bswap32(AES_Td0[st.B(AES_ishifts[4 * j + 0])] ^
- AES_Td1[st.B(AES_ishifts[4 * j + 1])] ^
- AES_Td2[st.B(AES_ishifts[4 * j + 2])] ^
- AES_Td3[st.B(AES_ishifts[4 * j + 3])]);
+ aesdec_ISB_ISR_IMC_AK(ad, st, rk, false);
}
}
void glue(helper_aesdeclast, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s)
{
- int i;
- Reg st = *v;
- Reg rk = *s;
+ for (int i = 0; i < SHIFT; i++) {
+ AESState *ad = (AESState *)&d->ZMM_X(i);
+ AESState *st = (AESState *)&v->ZMM_X(i);
+ AESState *rk = (AESState *)&s->ZMM_X(i);
- for (i = 0; i < 8 << SHIFT; i++) {
- d->B(i) = rk.B(i) ^ (AES_isbox[st.B(AES_ishifts[i & 15] + (i & ~15))]);
+ aesdec_ISB_ISR_AK(ad, st, rk, false);
}
}
void glue(helper_aesenc, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s)
{
- int i;
- Reg st = *v;
- Reg rk = *s;
+ for (int i = 0; i < SHIFT; i++) {
+ AESState *ad = (AESState *)&d->ZMM_X(i);
+ AESState *st = (AESState *)&v->ZMM_X(i);
+ AESState *rk = (AESState *)&s->ZMM_X(i);
- for (i = 0 ; i < 2 << SHIFT ; i++) {
- int j = i & 3;
- d->L(i) = rk.L(i) ^ bswap32(AES_Te0[st.B(AES_shifts[4 * j + 0])] ^
- AES_Te1[st.B(AES_shifts[4 * j + 1])] ^
- AES_Te2[st.B(AES_shifts[4 * j + 2])] ^
- AES_Te3[st.B(AES_shifts[4 * j + 3])]);
+ aesenc_SB_SR_MC_AK(ad, st, rk, false);
}
}
void glue(helper_aesenclast, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s)
{
- int i;
- Reg st = *v;
- Reg rk = *s;
+ for (int i = 0; i < SHIFT; i++) {
+ AESState *ad = (AESState *)&d->ZMM_X(i);
+ AESState *st = (AESState *)&v->ZMM_X(i);
+ AESState *rk = (AESState *)&s->ZMM_X(i);
- for (i = 0; i < 8 << SHIFT; i++) {
- d->B(i) = rk.B(i) ^ (AES_sbox[st.B(AES_shifts[i & 15] + (i & ~15))]);
+ aesenc_SB_SR_AK(ad, st, rk, false);
}
}
#if SHIFT == 1
void glue(helper_aesimc, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
{
- int i;
- Reg tmp = *s;
+ AESState *ad = (AESState *)&d->ZMM_X(0);
+ AESState *st = (AESState *)&s->ZMM_X(0);
- for (i = 0 ; i < 4 ; i++) {
- d->L(i) = bswap32(AES_imc[tmp.B(4 * i + 0)][0] ^
- AES_imc[tmp.B(4 * i + 1)][1] ^
- AES_imc[tmp.B(4 * i + 2)][2] ^
- AES_imc[tmp.B(4 * i + 3)][3]);
- }
+ aesdec_IMC(ad, st, false);
}
void glue(helper_aeskeygenassist, SUFFIX)(CPUX86State *env, Reg *d, Reg *s,
diff --git a/target/ppc/int_helper.c b/target/ppc/int_helper.c
index d97a7f1..834da80 100644
--- a/target/ppc/int_helper.c
+++ b/target/ppc/int_helper.c
@@ -25,6 +25,7 @@
#include "qemu/log.h"
#include "exec/helper-proto.h"
#include "crypto/aes.h"
+#include "crypto/aes-round.h"
#include "fpu/softfloat.h"
#include "qapi/error.h"
#include "qemu/guest-random.h"
@@ -2932,59 +2933,30 @@ void helper_vsbox(ppc_avr_t *r, ppc_avr_t *a)
void helper_vcipher(ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b)
{
- ppc_avr_t result;
- int i;
+ AESState *ad = (AESState *)r;
+ AESState *st = (AESState *)a;
+ AESState *rk = (AESState *)b;
- VECTOR_FOR_INORDER_I(i, u32) {
- result.VsrW(i) = b->VsrW(i) ^
- (AES_Te0[a->VsrB(AES_shifts[4 * i + 0])] ^
- AES_Te1[a->VsrB(AES_shifts[4 * i + 1])] ^
- AES_Te2[a->VsrB(AES_shifts[4 * i + 2])] ^
- AES_Te3[a->VsrB(AES_shifts[4 * i + 3])]);
- }
- *r = result;
+ aesenc_SB_SR_MC_AK(ad, st, rk, true);
}
void helper_vcipherlast(ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b)
{
- ppc_avr_t result;
- int i;
-
- VECTOR_FOR_INORDER_I(i, u8) {
- result.VsrB(i) = b->VsrB(i) ^ (AES_sbox[a->VsrB(AES_shifts[i])]);
- }
- *r = result;
+ aesenc_SB_SR_AK((AESState *)r, (AESState *)a, (AESState *)b, true);
}
void helper_vncipher(ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b)
{
- /* This differs from what is written in ISA V2.07. The RTL is */
- /* incorrect and will be fixed in V2.07B. */
- int i;
- ppc_avr_t tmp;
+ AESState *ad = (AESState *)r;
+ AESState *st = (AESState *)a;
+ AESState *rk = (AESState *)b;
- VECTOR_FOR_INORDER_I(i, u8) {
- tmp.VsrB(i) = b->VsrB(i) ^ AES_isbox[a->VsrB(AES_ishifts[i])];
- }
-
- VECTOR_FOR_INORDER_I(i, u32) {
- r->VsrW(i) =
- AES_imc[tmp.VsrB(4 * i + 0)][0] ^
- AES_imc[tmp.VsrB(4 * i + 1)][1] ^
- AES_imc[tmp.VsrB(4 * i + 2)][2] ^
- AES_imc[tmp.VsrB(4 * i + 3)][3];
- }
+ aesdec_ISB_ISR_AK_IMC(ad, st, rk, true);
}
void helper_vncipherlast(ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b)
{
- ppc_avr_t result;
- int i;
-
- VECTOR_FOR_INORDER_I(i, u8) {
- result.VsrB(i) = b->VsrB(i) ^ (AES_isbox[a->VsrB(AES_ishifts[i])]);
- }
- *r = result;
+ aesdec_ISB_ISR_AK((AESState *)r, (AESState *)a, (AESState *)b, true);
}
void helper_vshasigmaw(ppc_avr_t *r, ppc_avr_t *a, uint32_t st_six)
diff --git a/target/riscv/crypto_helper.c b/target/riscv/crypto_helper.c
index 2ef3028..99d85a6 100644
--- a/target/riscv/crypto_helper.c
+++ b/target/riscv/crypto_helper.c
@@ -22,6 +22,7 @@
#include "exec/exec-all.h"
#include "exec/helper-proto.h"
#include "crypto/aes.h"
+#include "crypto/aes-round.h"
#include "crypto/sm4.h"
#define AES_XTIME(a) \
@@ -103,114 +104,50 @@ target_ulong HELPER(aes32dsi)(target_ulong rs1, target_ulong rs2,
return aes32_operation(shamt, rs1, rs2, false, false);
}
-#define BY(X, I) ((X >> (8 * I)) & 0xFF)
-
-#define AES_SHIFROWS_LO(RS1, RS2) ( \
- (((RS1 >> 24) & 0xFF) << 56) | (((RS2 >> 48) & 0xFF) << 48) | \
- (((RS2 >> 8) & 0xFF) << 40) | (((RS1 >> 32) & 0xFF) << 32) | \
- (((RS2 >> 56) & 0xFF) << 24) | (((RS2 >> 16) & 0xFF) << 16) | \
- (((RS1 >> 40) & 0xFF) << 8) | (((RS1 >> 0) & 0xFF) << 0))
-
-#define AES_INVSHIFROWS_LO(RS1, RS2) ( \
- (((RS2 >> 24) & 0xFF) << 56) | (((RS2 >> 48) & 0xFF) << 48) | \
- (((RS1 >> 8) & 0xFF) << 40) | (((RS1 >> 32) & 0xFF) << 32) | \
- (((RS1 >> 56) & 0xFF) << 24) | (((RS2 >> 16) & 0xFF) << 16) | \
- (((RS2 >> 40) & 0xFF) << 8) | (((RS1 >> 0) & 0xFF) << 0))
-
-#define AES_MIXBYTE(COL, B0, B1, B2, B3) ( \
- BY(COL, B3) ^ BY(COL, B2) ^ AES_GFMUL(BY(COL, B1), 3) ^ \
- AES_GFMUL(BY(COL, B0), 2))
-
-#define AES_MIXCOLUMN(COL) ( \
- AES_MIXBYTE(COL, 3, 0, 1, 2) << 24 | \
- AES_MIXBYTE(COL, 2, 3, 0, 1) << 16 | \
- AES_MIXBYTE(COL, 1, 2, 3, 0) << 8 | AES_MIXBYTE(COL, 0, 1, 2, 3) << 0)
-
-#define AES_INVMIXBYTE(COL, B0, B1, B2, B3) ( \
- AES_GFMUL(BY(COL, B3), 0x9) ^ AES_GFMUL(BY(COL, B2), 0xd) ^ \
- AES_GFMUL(BY(COL, B1), 0xb) ^ AES_GFMUL(BY(COL, B0), 0xe))
-
-#define AES_INVMIXCOLUMN(COL) ( \
- AES_INVMIXBYTE(COL, 3, 0, 1, 2) << 24 | \
- AES_INVMIXBYTE(COL, 2, 3, 0, 1) << 16 | \
- AES_INVMIXBYTE(COL, 1, 2, 3, 0) << 8 | \
- AES_INVMIXBYTE(COL, 0, 1, 2, 3) << 0)
-
-static inline target_ulong aes64_operation(target_ulong rs1, target_ulong rs2,
- bool enc, bool mix)
-{
- uint64_t RS1 = rs1;
- uint64_t RS2 = rs2;
- uint64_t result;
- uint64_t temp;
- uint32_t col_0;
- uint32_t col_1;
-
- if (enc) {
- temp = AES_SHIFROWS_LO(RS1, RS2);
- temp = (((uint64_t)AES_sbox[(temp >> 0) & 0xFF] << 0) |
- ((uint64_t)AES_sbox[(temp >> 8) & 0xFF] << 8) |
- ((uint64_t)AES_sbox[(temp >> 16) & 0xFF] << 16) |
- ((uint64_t)AES_sbox[(temp >> 24) & 0xFF] << 24) |
- ((uint64_t)AES_sbox[(temp >> 32) & 0xFF] << 32) |
- ((uint64_t)AES_sbox[(temp >> 40) & 0xFF] << 40) |
- ((uint64_t)AES_sbox[(temp >> 48) & 0xFF] << 48) |
- ((uint64_t)AES_sbox[(temp >> 56) & 0xFF] << 56));
- if (mix) {
- col_0 = temp & 0xFFFFFFFF;
- col_1 = temp >> 32;
-
- col_0 = AES_MIXCOLUMN(col_0);
- col_1 = AES_MIXCOLUMN(col_1);
-
- result = ((uint64_t)col_1 << 32) | col_0;
- } else {
- result = temp;
- }
- } else {
- temp = AES_INVSHIFROWS_LO(RS1, RS2);
- temp = (((uint64_t)AES_isbox[(temp >> 0) & 0xFF] << 0) |
- ((uint64_t)AES_isbox[(temp >> 8) & 0xFF] << 8) |
- ((uint64_t)AES_isbox[(temp >> 16) & 0xFF] << 16) |
- ((uint64_t)AES_isbox[(temp >> 24) & 0xFF] << 24) |
- ((uint64_t)AES_isbox[(temp >> 32) & 0xFF] << 32) |
- ((uint64_t)AES_isbox[(temp >> 40) & 0xFF] << 40) |
- ((uint64_t)AES_isbox[(temp >> 48) & 0xFF] << 48) |
- ((uint64_t)AES_isbox[(temp >> 56) & 0xFF] << 56));
- if (mix) {
- col_0 = temp & 0xFFFFFFFF;
- col_1 = temp >> 32;
-
- col_0 = AES_INVMIXCOLUMN(col_0);
- col_1 = AES_INVMIXCOLUMN(col_1);
-
- result = ((uint64_t)col_1 << 32) | col_0;
- } else {
- result = temp;
- }
- }
-
- return result;
-}
+static const AESState aes_zero = { };
target_ulong HELPER(aes64esm)(target_ulong rs1, target_ulong rs2)
{
- return aes64_operation(rs1, rs2, true, true);
+ AESState t;
+
+ t.d[HOST_BIG_ENDIAN] = rs1;
+ t.d[!HOST_BIG_ENDIAN] = rs2;
+ aesenc_SB_SR_MC_AK(&t, &t, &aes_zero, false);
+ return t.d[HOST_BIG_ENDIAN];
}
target_ulong HELPER(aes64es)(target_ulong rs1, target_ulong rs2)
{
- return aes64_operation(rs1, rs2, true, false);
+ AESState t;
+
+ t.d[HOST_BIG_ENDIAN] = rs1;
+ t.d[!HOST_BIG_ENDIAN] = rs2;
+ aesenc_SB_SR_AK(&t, &t, &aes_zero, false);
+ return t.d[HOST_BIG_ENDIAN];
}
target_ulong HELPER(aes64ds)(target_ulong rs1, target_ulong rs2)
{
- return aes64_operation(rs1, rs2, false, false);
+ AESState t;
+
+ t.d[HOST_BIG_ENDIAN] = rs1;
+ t.d[!HOST_BIG_ENDIAN] = rs2;
+ aesdec_ISB_ISR_AK(&t, &t, &aes_zero, false);
+ return t.d[HOST_BIG_ENDIAN];
}
target_ulong HELPER(aes64dsm)(target_ulong rs1, target_ulong rs2)
{
- return aes64_operation(rs1, rs2, false, true);
+ AESState t, z = { };
+
+ /*
+ * This instruction does not include a round key,
+ * so supply a zero to our primitive.
+ */
+ t.d[HOST_BIG_ENDIAN] = rs1;
+ t.d[!HOST_BIG_ENDIAN] = rs2;
+ aesdec_ISB_ISR_IMC_AK(&t, &t, &z, false);
+ return t.d[HOST_BIG_ENDIAN];
}
target_ulong HELPER(aes64ks2)(target_ulong rs1, target_ulong rs2)
@@ -259,17 +196,12 @@ target_ulong HELPER(aes64ks1i)(target_ulong rs1, target_ulong rnum)
target_ulong HELPER(aes64im)(target_ulong rs1)
{
- uint64_t RS1 = rs1;
- uint32_t col_0 = RS1 & 0xFFFFFFFF;
- uint32_t col_1 = RS1 >> 32;
- target_ulong result;
-
- col_0 = AES_INVMIXCOLUMN(col_0);
- col_1 = AES_INVMIXCOLUMN(col_1);
+ AESState t;
- result = ((uint64_t)col_1 << 32) | col_0;
-
- return result;
+ t.d[HOST_BIG_ENDIAN] = rs1;
+ t.d[!HOST_BIG_ENDIAN] = 0;
+ aesdec_IMC(&t, &t, false);
+ return t.d[HOST_BIG_ENDIAN];
}
target_ulong HELPER(sm4ed)(target_ulong rs1, target_ulong rs2,