Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
openSUSE:Leap:15.5:Update
openssl-1_1.31477
openssl-1_1-Optimize-AES-GCM-uarchs.patch
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File openssl-1_1-Optimize-AES-GCM-uarchs.patch of Package openssl-1_1.31477
From 954f45ba4c504570206ff5bed811e512cf92dc8e Mon Sep 17 00:00:00 2001 From: XiaokangQian <xiaokang.qian@arm.com> Date: Wed, 9 Jun 2021 06:35:46 +0000 Subject: [PATCH] Optimize AES-GCM for uarchs with unroll and new instructions Increase the block numbers to 8 for every iteration. Increase the hash table capacity. Make use of EOR3 instruction to improve the performance. This can improve performance 25-40% on out-of-order microarchitectures with a large number of fast execution units, such as Neoverse V1. We also see 20-30% performance improvements on other architectures such as the M1. Assembly code reviewd by Tom Cosgrove (ARM). Reviewed-by: Bernd Edlinger <bernd.edlinger@hotmail.de> Reviewed-by: Paul Dale <pauli@openssl.org> (Merged from https://github.com/openssl/openssl/pull/15916) --- crypto/arm64cpuid.pl | 8 + crypto/arm_arch.h | 6 + crypto/armcap.c | 24 +- crypto/modes/asm/aes-gcm-armv8-unroll8_64.pl | 7369 +++++++++++++++++ crypto/modes/asm/ghashv8-armx.pl | 105 +- crypto/modes/build.info | 4 +- include/crypto/aes_platform.h | 12 + .../ciphers/cipher_aes_gcm_hw_armv8.inc | 36 +- 8 files changed, 7546 insertions(+), 18 deletions(-) create mode 100644 crypto/modes/asm/aes-gcm-armv8-unroll8_64.pl Index: openssl-1.1.1l/crypto/arm64cpuid.pl =================================================================== --- openssl-1.1.1l.orig/crypto/arm64cpuid.pl +++ openssl-1.1.1l/crypto/arm64cpuid.pl @@ -78,6 +78,14 @@ _armv8_sha512_probe: ret .size _armv8_sha512_probe,.-_armv8_sha512_probe +.globl _armv8_eor3_probe +.type _armv8_eor3_probe,%function +_armv8_eor3_probe: + AARCH64_VALID_CALL_TARGET + .long 0xce010800 // eor3 v0.16b, v0.16b, v1.16b, v2.16b + ret +.size _armv8_eor3_probe,.-_armv8_eor3_probe + .globl _armv8_cpuid_probe .type _armv8_cpuid_probe,%function _armv8_cpuid_probe: Index: openssl-1.1.1l/crypto/arm_arch.h =================================================================== --- openssl-1.1.1l.orig/crypto/arm_arch.h +++ openssl-1.1.1l/crypto/arm_arch.h @@ -83,6 +83,9 @@ extern unsigned int OPENSSL_arm_midr; # define ARMV8_SHA512 (1<<6) # define ARMV8_CPUID (1<<7) +# define ARMV8_SHA3 (1<<11) +# define ARMV8_UNROLL8_EOR3 (1<<12) + /* * MIDR_EL1 system register * @@ -97,6 +100,7 @@ extern unsigned int OPENSSL_arm_midr; # define ARM_CPU_PART_CORTEX_A72 0xD08 # define ARM_CPU_PART_N1 0xD0C +# define ARM_CPU_PART_V1 0xD40 # define MIDR_PARTNUM_SHIFT 4 # define MIDR_PARTNUM_MASK (0xfff << MIDR_PARTNUM_SHIFT) @@ -125,4 +129,29 @@ extern unsigned int OPENSSL_arm_midr; # define MIDR_IS_CPU_MODEL(midr, imp, partnum) \ (((midr) & MIDR_CPU_MODEL_MASK) == MIDR_CPU_MODEL(imp, partnum)) + +# define IS_CPU_SUPPORT_UNROLL8_EOR3() \ + (OPENSSL_armcap_P & ARMV8_UNROLL8_EOR3) + +#if defined(__ASSEMBLER__) + + /* + * Support macros for + * - Armv8.3-A Pointer Authentication and + * - Armv8.5-A Branch Target Identification + * features which require emitting a .note.gnu.property section with the + * appropriate architecture-dependent feature bits set. + * Read more: "ELF for the Arm® 64-bit Architecture" + */ + +# if defined(__ARM_FEATURE_BTI_DEFAULT) && __ARM_FEATURE_BTI_DEFAULT == 1 +# define GNU_PROPERTY_AARCH64_BTI (1 << 0) /* Has Branch Target Identification */ +# define AARCH64_VALID_CALL_TARGET hint #34 /* BTI 'c' */ +# else +# define GNU_PROPERTY_AARCH64_BTI 0 /* No Branch Target Identification */ +# define AARCH64_VALID_CALL_TARGET +# endif + +# endif /* defined __ASSEMBLER__ */ + #endif Index: openssl-1.1.1l/crypto/armcap.c =================================================================== --- openssl-1.1.1l.orig/crypto/armcap.c +++ openssl-1.1.1l/crypto/armcap.c @@ -13,6 +13,9 @@ #include <setjmp.h> #include <signal.h> #include <openssl/crypto.h> +#ifdef __APPLE__ +#include <sys/sysctl.h> +#endif #include "internal/cryptlib.h" #include "arm_arch.h" @@ -109,20 +112,23 @@ static unsigned long getauxval(unsigned * ARM puts the feature bits for Crypto Extensions in AT_HWCAP2, whereas * AArch64 used AT_HWCAP. */ +# ifndef AT_HWCAP +# define AT_HWCAP 16 +# endif +# ifndef AT_HWCAP2 +# define AT_HWCAP2 26 +# endif # if defined(__arm__) || defined (__arm) -# define HWCAP 16 - /* AT_HWCAP */ +# define HWCAP AT_HWCAP # define HWCAP_NEON (1 << 12) -# define HWCAP_CE 26 - /* AT_HWCAP2 */ +# define HWCAP_CE AT_HWCAP2 # define HWCAP_CE_AES (1 << 0) # define HWCAP_CE_PMULL (1 << 1) # define HWCAP_CE_SHA1 (1 << 2) # define HWCAP_CE_SHA256 (1 << 3) # elif defined(__aarch64__) -# define HWCAP 16 - /* AT_HWCAP */ +# define HWCAP AT_HWCAP # define HWCAP_NEON (1 << 1) # define HWCAP_CE HWCAP @@ -131,6 +137,7 @@ static unsigned long getauxval(unsigned # define HWCAP_CE_SHA1 (1 << 5) # define HWCAP_CE_SHA256 (1 << 6) # define HWCAP_CPUID (1 << 11) +# define HWCAP_SHA3 (1 << 17) # define HWCAP_CE_SHA512 (1 << 21) # endif @@ -145,12 +152,15 @@ void OPENSSL_cpuid_setup(void) return; trigger = 1; + OPENSSL_armcap_P = 0; + if ((e = getenv("OPENSSL_armcap"))) { OPENSSL_armcap_P = (unsigned int)strtoul(e, NULL, 0); return; } -# if defined(__APPLE__) && !defined(__aarch64__) +# if defined(__APPLE__) +# if !defined(__aarch64__) /* * Capability probing by catching SIGILL appears to be problematic * on iOS. But since Apple universe is "monocultural", it's actually @@ -166,9 +176,25 @@ void OPENSSL_cpuid_setup(void) * Unified code works because it never triggers SIGILL on Apple * devices... */ -# endif +# else + { + unsigned int feature; + size_t len = sizeof(feature); + char uarch[64]; - OPENSSL_armcap_P = 0; + if (sysctlbyname("hw.optional.armv8_2_sha512", &feature, &len, NULL, 0) == 0 && feature == 1) + OPENSSL_armcap_P |= ARMV8_SHA512; + feature = 0; + if (sysctlbyname("hw.optional.armv8_2_sha3", &feature, &len, NULL, 0) == 0 && feature == 1) { + OPENSSL_armcap_P |= ARMV8_SHA3; + len = sizeof(uarch); + if ((sysctlbyname("machdep.cpu.brand_string", uarch, &len, NULL, 0) == 0) && + (strncmp(uarch, "Apple M1", 8) == 0)) + OPENSSL_armcap_P |= ARMV8_UNROLL8_EOR3; + } + } +# endif +# endif # ifdef OSSL_IMPLEMENT_GETAUXVAL if (getauxval(HWCAP) & HWCAP_NEON) { @@ -194,6 +220,9 @@ void OPENSSL_cpuid_setup(void) if (hwcap & HWCAP_CPUID) OPENSSL_armcap_P |= ARMV8_CPUID; + + if (hwcap & HWCAP_SHA3) + OPENSSL_armcap_P |= ARMV8_SHA3; # endif } # endif @@ -237,6 +266,10 @@ void OPENSSL_cpuid_setup(void) _armv8_sha512_probe(); OPENSSL_armcap_P |= ARMV8_SHA512; } + if (sigsetjmp(ill_jmp, 1) == 0) { + _armv8_eor3_probe(); + OPENSSL_armcap_P |= ARMV8_SHA3; + } # endif } # endif @@ -259,6 +292,9 @@ void OPENSSL_cpuid_setup(void) (OPENSSL_armcap_P & ARMV7_NEON)) { OPENSSL_armv8_rsa_neonized = 1; } + if ((MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_ARM, ARM_CPU_PART_V1)) && + (OPENSSL_armcap_P & ARMV8_SHA3)) + OPENSSL_armcap_P |= ARMV8_UNROLL8_EOR3; # endif } #endif Index: openssl-1.1.1l/crypto/modes/asm/aes-gcm-armv8-unroll8_64.pl =================================================================== --- /dev/null +++ openssl-1.1.1l/crypto/modes/asm/aes-gcm-armv8-unroll8_64.pl @@ -0,0 +1,7369 @@ +#! /usr/bin/env perl +# Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +# +#======================================================================== +# Written by Xiaokang Qian <xiaokang.qian@arm.com> for the OpenSSL project, +# derived from https://github.com/ARM-software/AArch64cryptolib, original +# author Samuel Lee <Samuel.Lee@arm.com>. The module is, however, dual +# licensed under OpenSSL and SPDX BSD-3-Clause licenses depending on where you +# obtain it. +#======================================================================== +# +# Approach - We want to reload constants as we have plenty of spare ASIMD slots around crypto units for loading +# Unroll x8 in main loop, main loop to act on 8 16B blocks per iteration, and then do modulo of the accumulated +# intermediate hashesfrom the 8 blocks. +# +# ____________________________________________________ +# | | +# | PRE | +# |____________________________________________________| +# | | | | +# | CTR block 8k+13| AES block 8k+8 | GHASH block 8k+0 | +# |________________|________________|__________________| +# | | | | +# | CTR block 8k+14| AES block 8k+9 | GHASH block 8k+1 | +# |________________|________________|__________________| +# | | | | +# | CTR block 8k+15| AES block 8k+10| GHASH block 8k+2 | +# |________________|________________|__________________| +# | | | | +# | CTR block 8k+16| AES block 8k+11| GHASH block 8k+3 | +# |________________|________________|__________________| +# | | | | +# | CTR block 8k+17| AES block 8k+12| GHASH block 8k+4 | +# |________________|________________|__________________| +# | | | | +# | CTR block 8k+18| AES block 8k+13| GHASH block 8k+5 | +# |________________|________________|__________________| +# | | | | +# | CTR block 8k+19| AES block 8k+14| GHASH block 8k+6 | +# |________________|________________|__________________| +# | | | | +# | CTR block 8k+20| AES block 8k+15| GHASH block 8k+7 | +# |________________|____(mostly)____|__________________| +# | | +# | MODULO | +# |____________________________________________________| +# +# PRE: +# Ensure previous generated intermediate hash is aligned and merged with result for GHASH 4k+0 +# EXT low_acc, low_acc, low_acc, #8 +# EOR res_curr (8k+0), res_curr (4k+0), low_acc +# +# CTR block: +# Increment and byte reverse counter in scalar registers and transfer to SIMD registers +# REV ctr32, rev_ctr32 +# ORR ctr64, constctr96_top32, ctr32, LSL #32 +# INS ctr_next.d[0], constctr96_bottom64 // Keeping this in scalar registers to free up space in SIMD RF +# INS ctr_next.d[1], ctr64X +# ADD rev_ctr32, #1 +# +# AES block: +# Do AES encryption/decryption on CTR block X and EOR it with input block X. Take 256 bytes key below for example. +# Doing small trick here of loading input in scalar registers, EORing with last key and then transferring +# Given we are very constrained in our ASIMD registers this is quite important +# +# Encrypt: +# LDR input_low, [ input_ptr ], #8 +# LDR input_high, [ input_ptr ], #8 +# EOR input_low, k14_low +# EOR input_high, k14_high +# INS res_curr.d[0], input_low +# INS res_curr.d[1], input_high +# AESE ctr_curr, k0; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k1; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k2; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k3; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k4; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k5; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k6; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k7; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k8; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k9; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k10; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k11; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k12; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k13 +# EOR res_curr, res_curr, ctr_curr +# ST1 { res_curr.16b }, [ output_ptr ], #16 +# +# Decrypt: +# AESE ctr_curr, k0; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k1; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k2; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k3; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k4; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k5; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k6; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k7; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k8; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k9; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k10; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k11; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k12; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k13 +# LDR res_curr, [ input_ptr ], #16 +# EOR res_curr, res_curr, ctr_curr +# MOV output_low, res_curr.d[0] +# MOV output_high, res_curr.d[1] +# EOR output_low, k14_low +# EOR output_high, k14_high +# STP output_low, output_high, [ output_ptr ], #16 + +# GHASH block X: +# Do 128b karatsuba polynomial multiplication on block +# We only have 64b->128b polynomial multipliers, naively that means we need to do 4 64b multiplies to generate a 128b +# +# multiplication: +# Pmull(A,B) == (Pmull(Ah,Bh)<<128 | Pmull(Al,Bl)) ^ (Pmull(Ah,Bl) ^ Pmull(Al,Bh))<<64 +# +# The idea behind Karatsuba multiplication is that we can do just 3 64b multiplies: +# Pmull(A,B) == (Pmull(Ah,Bh)<<128 | Pmull(Al,Bl)) ^ (Pmull(Ah^Al,Bh^Bl) ^ Pmull(Ah,Bh) ^ Pmull(Al,Bl))<<64 +# +# There is some complication here because the bit order of GHASH's PMULL is reversed compared to elsewhere, so we are +# multiplying with "twisted" powers of H +# +# Note: We can PMULL directly into the acc_x in first GHASH of the loop +# Note: For scheduling big cores we want to split the processing to happen over two loop iterations - otherwise the critical +# path latency dominates the performance +# +# This has a knock on effect on register pressure, so we have to be a bit more clever with our temporary registers +# than indicated here +# REV64 res_curr, res_curr +# INS t_m.d[0], res_curr.d[1] +# EOR t_m.8B, t_m.8B, res_curr.8B +# PMULL2 t_h, res_curr, HX +# PMULL t_l, res_curr, HX +# PMULL t_m, t_m, HX_k +# EOR acc_h, acc_h, t_h +# EOR acc_l, acc_l, t_l +# EOR acc_m, acc_m, t_m +# +# MODULO: take the partial accumulators (~representing sum of 256b multiplication results), from GHASH and do modulo reduction on them +# There is some complication here because the bit order of GHASH's PMULL is reversed compared to elsewhere, so we are doing modulo +# with a reversed constant +# EOR3 acc_m, acc_m, acc_l, acc_h // Finish off karatsuba processing +# PMULL t_mod, acc_h, mod_constant +# EXT acc_h, acc_h, acc_h, #8 +# EOR3 acc_m, acc_m, t_mod, acc_h +# PMULL acc_h, acc_m, mod_constant +# EXT acc_m, acc_m, acc_m, #8 +# EOR3 acc_l, acc_l, acc_m, acc_h + +$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate ) or +die "can't locate arm-xlate.pl"; + +die "only for 64 bit" if $flavour !~ /64/; + +open OUT,"| \"$^X\" $xlate $flavour $output"; +*STDOUT=*OUT; + +$code=<<___; +#include "arm_arch.h" + +#if __ARM_MAX_ARCH__>=8 +___ +$code.=".arch armv8.2-a+crypto\n.arch_extension sha3\n.text\n"; + +$input_ptr="x0"; #argument block +$bit_length="x1"; +$output_ptr="x2"; +$current_tag="x3"; +$counter="x16"; +$constant_temp="x15"; +$modulo_constant="x10"; +$cc="x8"; +{ +my ($end_input_ptr,$main_end_input_ptr,$temp0_x,$temp1_x)=map("x$_",(4..7)); +my ($temp2_x,$temp3_x)=map("x$_",(13..14)); +my ($ctr0b,$ctr1b,$ctr2b,$ctr3b,$ctr4b,$ctr5b,$ctr6b,$ctr7b,$res0b,$res1b,$res2b,$res3b,$res4b,$res5b,$res6b,$res7b)=map("v$_.16b",(0..15)); +my ($ctr0,$ctr1,$ctr2,$ctr3,$ctr4,$ctr5,$ctr6,$ctr7,$res0,$res1,$res2,$res3,$res4,$res5,$res6,$res7)=map("v$_",(0..15)); +my ($ctr0d,$ctr1d,$ctr2d,$ctr3d,$ctr4d,$ctr5d,$ctr6d,$ctr7d)=map("d$_",(0..7)); +my ($ctr0q,$ctr1q,$ctr2q,$ctr3q,$ctr4q,$ctr5q,$ctr6q,$ctr7q)=map("q$_",(0..7)); +my ($res0q,$res1q,$res2q,$res3q,$res4q,$res5q,$res6q,$res7q)=map("q$_",(8..15)); + +my ($ctr_t0,$ctr_t1,$ctr_t2,$ctr_t3,$ctr_t4,$ctr_t5,$ctr_t6,$ctr_t7)=map("v$_",(8..15)); +my ($ctr_t0b,$ctr_t1b,$ctr_t2b,$ctr_t3b,$ctr_t4b,$ctr_t5b,$ctr_t6b,$ctr_t7b)=map("v$_.16b",(8..15)); +my ($ctr_t0q,$ctr_t1q,$ctr_t2q,$ctr_t3q,$ctr_t4q,$ctr_t5q,$ctr_t6q,$ctr_t7q)=map("q$_",(8..15)); + +my ($acc_hb,$acc_mb,$acc_lb)=map("v$_.16b",(17..19)); +my ($acc_h,$acc_m,$acc_l)=map("v$_",(17..19)); + +my ($h1,$h12k,$h2,$h3,$h34k,$h4)=map("v$_",(20..25)); +my ($h5,$h56k,$h6,$h7,$h78k,$h8)=map("v$_",(20..25)); +my ($h1q,$h12kq,$h2q,$h3q,$h34kq,$h4q)=map("q$_",(20..25)); +my ($h5q,$h56kq,$h6q,$h7q,$h78kq,$h8q)=map("q$_",(20..25)); + +my $t0="v16"; +my $t0d="d16"; + +my $t1="v29"; +my $t2=$res1; +my $t3=$t1; + +my $t4=$res0; +my $t5=$res2; +my $t6=$t0; + +my $t7=$res3; +my $t8=$res4; +my $t9=$res5; + +my $t10=$res6; +my $t11="v21"; +my $t12=$t1; + +my $rtmp_ctr="v30"; +my $rtmp_ctrq="q30"; +my $rctr_inc="v31"; +my $rctr_incd="d31"; + +my $mod_constantd=$t0d; +my $mod_constant=$t0; + +my ($rk0,$rk1,$rk2)=map("v$_.16b",(26..28)); +my ($rk3,$rk4,$rk5)=map("v$_.16b",(26..28)); +my ($rk6,$rk7,$rk8)=map("v$_.16b",(26..28)); +my ($rk9,$rk10,$rk11)=map("v$_.16b",(26..28)); +my ($rk12,$rk13,$rk14)=map("v$_.16b",(26..28)); +my ($rk0q,$rk1q,$rk2q)=map("q$_",(26..28)); +my ($rk3q,$rk4q,$rk5q)=map("q$_",(26..28)); +my ($rk6q,$rk7q,$rk8q)=map("q$_",(26..28)); +my ($rk9q,$rk10q,$rk11q)=map("q$_",(26..28)); +my ($rk12q,$rk13q,$rk14q)=map("q$_",(26..28)); +my $rk2q1="v28.1q"; +my $rk3q1="v26.1q"; +my $rk4v="v27"; + + +######################################################################################### +# size_t unroll8_eor3_aes_gcm_enc_128_kernel(const unsigned char *in, +# size_t len, +# unsigned char *out, +# const void *key, +# unsigned char ivec[16], +# u64 *Xi); +# +$code.=<<___; +.global unroll8_eor3_aes_gcm_enc_128_kernel +.type unroll8_eor3_aes_gcm_enc_128_kernel,%function +.align 4 +unroll8_eor3_aes_gcm_enc_128_kernel: + AARCH64_VALID_CALL_TARGET + cbz x1, .L128_enc_ret + stp d8, d9, [sp, #-80]! + mov $counter, x4 + mov $cc, x5 + stp d10, d11, [sp, #16] + stp d12, d13, [sp, #32] + stp d14, d15, [sp, #48] + mov x5, #0xc200000000000000 + stp x5, xzr, [sp, #64] + add $modulo_constant, sp, #64 + + mov $constant_temp, #0x100000000 @ set up counter increment + movi $rctr_inc.16b, #0x0 + mov $rctr_inc.d[1], $constant_temp + lsr $main_end_input_ptr, $bit_length, #3 @ byte_len + ld1 { $ctr0b}, [$counter] @ CTR block 0 + + sub $main_end_input_ptr, $main_end_input_ptr, #1 @ byte_len - 1 + + and $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffff80 @ number of bytes to be processed in main loop (at least 1 byte must be handled by tail) + + rev32 $rtmp_ctr.16b, $ctr0.16b @ set up reversed counter + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 0 + + rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 1 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 1 + + rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 2 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 2 + + rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 3 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 3 + + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 4 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 4 + + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 5 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 5 + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 6 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 6 + + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 7 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 7 + + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 0 + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 0 + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 0 + + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 0 + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 0 + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 0 + + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 0 + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 0 + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 1 + + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 1 + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 1 + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 1 + + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 1 + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 1 + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 1 + + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 2 + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 1 + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 2 + + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 2 + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 2 + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 2 + + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 2 + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 2 + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 2 + + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 3 + + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 3 + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 3 + + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 3 + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 3 + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 3 + + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 3 + + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 4 + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 3 + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 4 + + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 4 + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 4 + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 4 + + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 4 + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 4 + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 4 + + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 5 + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 5 + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 5 + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 5 + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 5 + + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 5 + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 5 + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 5 + + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 6 + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 6 + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 6 + + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 6 + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 6 + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 6 + + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 6 + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 6 + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 7 + + ld1 { $acc_lb}, [$current_tag] + ext $acc_lb, $acc_lb, $acc_lb, #8 + rev64 $acc_lb, $acc_lb + + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 7 + + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 7 + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 7 + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 7 + + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 7 + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 7 + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 7 + + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8 + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8 + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8 + + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8 + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8 + ldr $rk10q, [$cc, #160] @ load rk10 + + aese $ctr3b, $rk9 @ AES block 8k+11 - round 9 + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8 + aese $ctr2b, $rk9 @ AES block 8k+10 - round 9 + + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8 + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8 + aese $ctr6b, $rk9 @ AES block 8k+14 - round 9 + + aese $ctr4b, $rk9 @ AES block 8k+12 - round 9 + add $main_end_input_ptr, $main_end_input_ptr, $input_ptr + aese $ctr0b, $rk9 @ AES block 8k+8 - round 9 + + aese $ctr7b, $rk9 @ AES block 8k+15 - round 9 + aese $ctr5b, $rk9 @ AES block 8k+13 - round 9 + aese $ctr1b, $rk9 @ AES block 8k+9 - round 9 + + add $end_input_ptr, $input_ptr, $bit_length, lsr #3 @ end_input_ptr + cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks + b.ge .L128_enc_tail @ handle tail + + ldp $ctr_t0q, $ctr_t1q, [$input_ptr], #32 @ AES block 0, 1 - load plaintext + + ldp $ctr_t2q, $ctr_t3q, [$input_ptr], #32 @ AES block 2, 3 - load plaintext + + ldp $ctr_t4q, $ctr_t5q, [$input_ptr], #32 @ AES block 4, 5 - load plaintext + + ldp $ctr_t6q, $ctr_t7q, [$input_ptr], #32 @ AES block 6, 7 - load plaintext + cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks + + eor3 $res0b, $ctr_t0b, $ctr0b, $rk10 @ AES block 0 - result + rev32 $ctr0.16b, $rtmp_ctr.16b @ CTR block 8 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8 + + eor3 $res1b, $ctr_t1b, $ctr1b, $rk10 @ AES block 1 - result + stp $res0q, $res1q, [$output_ptr], #32 @ AES block 0, 1 - store result + + rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 9 + eor3 $res5b, $ctr_t5b, $ctr5b, $rk10 @ AES block 5 - result + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 9 + + eor3 $res2b, $ctr_t2b, $ctr2b, $rk10 @ AES block 2 - result + eor3 $res6b, $ctr_t6b, $ctr6b, $rk10 @ AES block 6 - result + eor3 $res4b, $ctr_t4b, $ctr4b, $rk10 @ AES block 4 - result + + rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 10 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 10 + + eor3 $res3b, $ctr_t3b, $ctr3b, $rk10 @ AES block 3 - result + eor3 $res7b, $ctr_t7b, $ctr7b,$rk10 @ AES block 7 - result + stp $res2q, $res3q, [$output_ptr], #32 @ AES block 2, 3 - store result + + rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 11 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 11 + stp $res4q, $res5q, [$output_ptr], #32 @ AES block 4, 5 - store result + + stp $res6q, $res7q, [$output_ptr], #32 @ AES block 6, 7 - store result + + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 12 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 12 + b.ge .L128_enc_prepretail @ do prepretail + +.L128_enc_main_loop: @ main loop start + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13 + ldr $h5q, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + ldr $h6q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13 + + rev64 $res1b, $res1b @ GHASH block 8k+1 + rev64 $res0b, $res0b @ GHASH block 8k + ldr $h7q, [$current_tag, #176] @ load h7l | h7h + ext $h7.16b, $h7.16b, $h7.16b, #8 + ldr $h8q, [$current_tag, #208] @ load h8l | h8h + ext $h8.16b, $h8.16b, $h8.16b, #8 + + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14 + ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0 + + ldr $h56kq, [$current_tag, #144] @ load h6k | h5k + ldr $h78kq, [$current_tag, #192] @ load h8k | h7k + rev64 $res5b, $res5b @ GHASH block 8k+5 (t0, t1, t2 and t3 free) + rev64 $res3b, $res3b @ GHASH block 8k+3 + + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + eor $res0b, $res0b, $acc_lb @ PRE 1 + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15 + + rev64 $res7b, $res7b @ GHASH block 8k+7 (t0, t1, t2 and t3 free) + + pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high + rev64 $res2b, $res2b @ GHASH block 8k+2 + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high + + pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low + trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low + + trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high + pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high + + eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + ldr $h4q, [$current_tag, #112] @ load h3l | h3h + ext $h4.16b, $h4.16b, $h4.16b, #8 + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0 + + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0 + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0 + eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15 + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0 + eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid + + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0 + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1 + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0 + + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1 + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0 + pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low + + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1 + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0 + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1 + + eor3 $acc_hb, $acc_hb, $t1.16b,$t2.16b @ GHASH block 8k+2, 8k+3 - high + trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1 + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1 + + pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1 + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1 + + pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid + eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid + + rev64 $res6b, $res6b @ GHASH block 8k+6 (t0, t1, and t2 free) + eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low + + pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid + eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid + pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid + + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2 + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2 + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2 + + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2 + eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2 + + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2 + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2 + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2 + + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3 + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + rev64 $res4b, $res4b @ GHASH block 8k+4 (t0, t1, and t2 free) + + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3 + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3 + + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + ldr $h2q, [$current_tag, #64] @ load h1l | h1h + ext $h2.16b, $h2.16b, $h2.16b, #8 + pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high + pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low + + trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3 + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3 + + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3 + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3 + + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3 + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4 + + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4 + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4 + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4 + + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4 + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4 + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4 + + pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high + eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low + + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4 + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + + pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid + pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid + pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high + + pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5 + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5 + + pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low + eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high + trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + + eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5 + + eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5 + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5 + + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5 + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5 + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5 + + eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low + + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6 + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6 + + pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6 + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6 + + pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid + eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low + ldp $ctr_t0q, $ctr_t1q, [$input_ptr], #32 @ AES block 8k+8, 8k+9 - load plaintext + + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6 + rev32 $h1.16b, $rtmp_ctr.16b @ CTR block 8k+16 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+16 + + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6 + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6 + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6 + + eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high + + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7 + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7 + ldp $ctr_t2q, $ctr_t3q, [$input_ptr], #32 @ AES block 8k+10, 8k+11 - load plaintext + + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7 + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7 + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7 + + pmull $t11.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7 + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7 + + rev32 $h2.16b, $rtmp_ctr.16b @ CTR block 8k+17 + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7 + + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8 + ldp $ctr_t4q, $ctr_t5q, [$input_ptr], #32 @ AES block 8k+12, 8k+13 - load plaintext + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+17 + + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8 + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8 + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8 + + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8 + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + ldr $rk10q, [$cc, #160] @ load rk10 + + ext $t12.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + rev32 $h3.16b, $rtmp_ctr.16b @ CTR block 8k+18 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+18 + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8 + + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8 + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8 + + aese $ctr2b, $rk9 @ AES block 8k+10 - round 9 + aese $ctr4b, $rk9 @ AES block 8k+12 - round 9 + aese $ctr1b, $rk9 @ AES block 8k+9 - round 9 + + ldp $ctr_t6q, $ctr_t7q, [$input_ptr], #32 @ AES block 8k+14, 8k+15 - load plaintext + rev32 $h4.16b, $rtmp_ctr.16b @ CTR block 8k+19 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+19 + + cmp $input_ptr, $main_end_input_ptr @ LOOP CONTROL + eor3 $res4b, $ctr_t4b, $ctr4b, $rk10 @ AES block 4 - result + aese $ctr7b, $rk9 @ AES block 8k+15 - round 9 + + aese $ctr6b, $rk9 @ AES block 8k+14 - round 9 + aese $ctr3b, $rk9 @ AES block 8k+11 - round 9 + + eor3 $res2b, $ctr_t2b, $ctr2b, $rk10 @ AES block 8k+10 - result + + mov $ctr2.16b, $h3.16b @ CTR block 8k+18 + aese $ctr0b, $rk9 @ AES block 8k+8 - round 9 + + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 8k+20 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+20 + + eor3 $res7b, $ctr_t7b, $ctr7b, $rk10 @ AES block 7 - result + aese $ctr5b, $rk9 @ AES block 8k+13 - round 9 + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + + eor3 $res1b, $ctr_t1b, $ctr1b, $rk10 @ AES block 8k+9 - result + eor3 $res3b, $ctr_t3b, $ctr3b, $rk10 @ AES block 8k+11 - result + mov $ctr3.16b, $h4.16b @ CTR block 8k+19 + + ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + eor3 $res5b, $ctr_t5b, $ctr5b, $rk10 @ AES block 5 - result + mov $ctr1.16b, $h2.16b @ CTR block 8k+17 + + eor3 $res0b, $ctr_t0b, $ctr0b, $rk10 @ AES block 8k+8 - result + mov $ctr0.16b, $h1.16b @ CTR block 8k+16 + stp $res0q, $res1q, [$output_ptr], #32 @ AES block 8k+8, 8k+9 - store result + + stp $res2q, $res3q, [$output_ptr], #32 @ AES block 8k+10, 8k+11 - store result + eor3 $res6b, $ctr_t6b, $ctr6b, $rk10 @ AES block 6 - result + + stp $res4q, $res5q, [$output_ptr], #32 @ AES block 8k+12, 8k+13 - store result + eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low + + stp $res6q, $res7q, [$output_ptr], #32 @ AES block 8k+14, 8k+15 - store result + b.lt .L128_enc_main_loop + +.L128_enc_prepretail: @ PREPRETAIL + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13 + ldr $h7q, [$current_tag, #176] @ load h7l | h7h + ext $h7.16b, $h7.16b, $h7.16b, #8 + ldr $h8q, [$current_tag, #208] @ load h8l | h8h + ext $h8.16b, $h8.16b, $h8.16b, #8 + ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0 + + ldr $h5q, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + ldr $h6q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + rev64 $res0b, $res0b @ GHASH block 8k + rev64 $res1b, $res1b @ GHASH block 8k+1 + + ldr $h56kq, [$current_tag, #144] @ load h6k | h5k + ldr $h78kq, [$current_tag, #192] @ load h6k | h5k + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13 + rev64 $res3b, $res3b @ GHASH block 8k+3 + + rev64 $res2b, $res2b @ GHASH block 8k+2 + eor $res0b, $res0b, $acc_lb @ PRE 1 + + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14 + + pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high + + rev64 $res5b, $res5b @ GHASH block 8k+5 (t0, t1, t2 and t3 free) + trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + + pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low + eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high + trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + + eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low + eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid + + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14 + + pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid + pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid + + rev64 $res4b, $res4b @ GHASH block 8k+4 (t0, t1, and t2 free) + rev64 $res7b, $res7b @ GHASH block 8k+7 (t0, t1, t2 and t3 free) + + eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid + + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15 + + rev64 $res6b, $res6b @ GHASH block 8k+6 (t0, t1, and t2 free) + + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0 + + pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high + pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high + + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0 + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0 + + pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0 + + eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high + trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0 + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0 + + eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0 + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0 + + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1 + pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low + + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1 + pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid + + eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low + pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid + + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1 + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1 + + eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1 + + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1 + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1 + + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2 + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2 + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2 + + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2 + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1 + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2 + + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3 + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2 + + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2 + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2 + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + ldr $h2q, [$current_tag, #64] @ load h1l | h1h + ext $h2.16b, $h2.16b, $h2.16b, #8 + trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3 + + pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3 + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3 + + pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low + trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high + + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15 + + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3 + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3 + eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + + pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3 + pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high + + trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low + trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4 + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4 + eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high + + eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low + eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid + + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5 + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4 + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4 + + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4 + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4 + + pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4 + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4 + + pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low + + eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid + pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid + + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5 + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5 + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5 + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5 + + eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5 + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5 + + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5 + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6 + + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6 + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6 + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6 + + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6 + eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low + eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6 + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6 + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6 + + pmull $t11.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7 + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7 + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7 + ext $t12.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7 + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7 + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7 + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7 + + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7 + + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8 + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8 + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8 + ext $acc_mb, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8 + eor3 $acc_lb, $acc_lb, $acc_hb, $acc_mb @ MODULO - fold into low + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8 + + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8 + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8 + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8 + + ldr $rk10q, [$cc, #160] @ load rk10 + aese $ctr6b, $rk9 @ AES block 8k+14 - round 9 + aese $ctr2b, $rk9 @ AES block 8k+10 - round 9 + + aese $ctr0b, $rk9 @ AES block 8k+8 - round 9 + aese $ctr1b, $rk9 @ AES block 8k+9 - round 9 + + aese $ctr3b, $rk9 @ AES block 8k+11 - round 9 + aese $ctr5b, $rk9 @ AES block 8k+13 - round 9 + + aese $ctr4b, $rk9 @ AES block 8k+12 - round 9 + aese $ctr7b, $rk9 @ AES block 8k+15 - round 9 +.L128_enc_tail: @ TAIL + + sub $main_end_input_ptr, $end_input_ptr, $input_ptr @ main_end_input_ptr is number of bytes left to process + ldr $ctr_t0q, [$input_ptr], #16 @ AES block 8k+8 - load plaintext + + mov $t1.16b, $rk10 + ldp $h5q, $h56kq, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + + eor3 $res1b, $ctr_t0b, $ctr0b, $t1.16b @ AES block 8k+8 - result + ext $t0.16b, $acc_lb, $acc_lb, #8 @ prepare final partial tag + ldp $h6q, $h7q, [$current_tag, #160] @ load h6k | h5k + ext $h6.16b, $h6.16b, $h6.16b, #8 + ext $h7.16b, $h7.16b, $h7.16b, #8 + + ldp $h78kq, $h8q, [$current_tag, #192] @ load h7l | h7h + ext $h8.16b, $h8.16b, $h8.16b, #8 + cmp $main_end_input_ptr, #112 + b.gt .L128_enc_blocks_more_than_7 + + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr5b + movi $acc_h.8b, #0 + + cmp $main_end_input_ptr, #96 + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + mov $ctr5b, $ctr4b + + mov $ctr4b, $ctr3b + mov $ctr3b, $ctr2b + mov $ctr2b, $ctr1b + + movi $acc_l.8b, #0 + movi $acc_m.8b, #0 + b.gt .L128_enc_blocks_more_than_6 + + mov $ctr7b, $ctr6b + cmp $main_end_input_ptr, #80 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + mov $ctr6b, $ctr5b + mov $ctr5b, $ctr4b + + mov $ctr4b, $ctr3b + mov $ctr3b, $ctr1b + b.gt .L128_enc_blocks_more_than_5 + + cmp $main_end_input_ptr, #64 + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr5b + + mov $ctr5b, $ctr4b + mov $ctr4b, $ctr1b + b.gt .L128_enc_blocks_more_than_4 + + mov $ctr7b, $ctr6b + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + mov $ctr6b, $ctr5b + + mov $ctr5b, $ctr1b + cmp $main_end_input_ptr, #48 + b.gt .L128_enc_blocks_more_than_3 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr1b + + cmp $main_end_input_ptr, #32 + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + b.gt .L128_enc_blocks_more_than_2 + + cmp $main_end_input_ptr, #16 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + mov $ctr7b, $ctr1b + b.gt .L128_enc_blocks_more_than_1 + + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + b .L128_enc_blocks_less_than_1 +.L128_enc_blocks_more_than_7: @ blocks left > 7 + st1 { $res1b}, [$output_ptr], #16 @ AES final-7 block - store result + + rev64 $res0b, $res1b @ GHASH final-7 block + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-6 block - load plaintext + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-7 block - mid + + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH final-7 block - high + + ins $acc_m.d[0], $h78k.d[1] @ GHASH final-7 block - mid + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-7 block - mid + movi $t0.8b, #0 @ surpress further partial tag feed in + + eor3 $res1b, $ctr_t1b, $ctr1b, $t1.16b @ AES final-6 block - result + + pmull $acc_m.1q, $rk4v.1d, $acc_m.1d @ GHASH final-7 block - mid + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH final-7 block - low +.L128_enc_blocks_more_than_6: @ blocks left > 6 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-6 block - store result + + rev64 $res0b, $res1b @ GHASH final-6 block + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-5 block - load plaintext + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-6 block - mid + + eor3 $res1b, $ctr_t1b, $ctr2b, $t1.16b @ AES final-5 block - result + pmull $rk3q1, $res0.1d, $h7.1d @ GHASH final-6 block - low + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-6 block - mid + movi $t0.8b, #0 @ surpress further partial tag feed in + + pmull $rk4v.1q, $rk4v.1d, $h78k.1d @ GHASH final-6 block - mid + pmull2 $rk2q1, $res0.2d, $h7.2d @ GHASH final-6 block - high + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-6 block - low + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-6 block - mid + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-6 block - high +.L128_enc_blocks_more_than_5: @ blocks left > 5 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-5 block - store result + + rev64 $res0b, $res1b @ GHASH final-5 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-5 block - mid + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-4 block - load plaintext + pmull2 $rk2q1, $res0.2d, $h6.2d @ GHASH final-5 block - high + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-5 block - high + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-5 block - mid + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-5 block - mid + + eor3 $res1b, $ctr_t1b, $ctr3b, $t1.16b @ AES final-4 block - result + pmull $rk3q1, $res0.1d, $h6.1d @ GHASH final-5 block - low + movi $t0.8b, #0 @ surpress further partial tag feed in + + pmull2 $rk4v.1q, $rk4v.2d, $h56k.2d @ GHASH final-5 block - mid + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-5 block - low + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-5 block - mid +.L128_enc_blocks_more_than_4: @ blocks left > 4 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-4 block - store result + + rev64 $res0b, $res1b @ GHASH final-4 block + + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-3 block - load plaintext + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-4 block - mid + movi $t0.8b, #0 @ surpress further partial tag feed in + pmull2 $rk2q1, $res0.2d, $h5.2d @ GHASH final-4 block - high + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-4 block - mid + + pmull $rk3q1, $res0.1d, $h5.1d @ GHASH final-4 block - low + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-4 block - high + pmull $rk4v.1q, $rk4v.1d, $h56k.1d @ GHASH final-4 block - mid + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-4 block - low + + eor3 $res1b, $ctr_t1b, $ctr4b, $t1.16b @ AES final-3 block - result + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-4 block - mid +.L128_enc_blocks_more_than_3: @ blocks left > 3 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-3 block - store result + + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + + rev64 $res0b, $res1b @ GHASH final-3 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + movi $t0.8b, #0 @ surpress further partial tag feed in + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-3 block - mid + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + pmull $rk3q1, $res0.1d, $h4.1d @ GHASH final-3 block - low + + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-2 block - load plaintext + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-3 block - mid + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-3 block - mid + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-3 block - low + + eor3 $res1b, $ctr_t1b, $ctr5b, $t1.16b @ AES final-2 block - result + + pmull2 $rk4v.1q, $rk4v.2d, $h34k.2d @ GHASH final-3 block - mid + pmull2 $rk2q1, $res0.2d, $h4.2d @ GHASH final-3 block - high + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-3 block - mid + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-3 block - high +.L128_enc_blocks_more_than_2: @ blocks left > 2 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-2 block - store result + + rev64 $res0b, $res1b @ GHASH final-2 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-1 block - load plaintext + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-2 block - mid + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + movi $t0.8b, #0 @ surpress further partial tag feed in + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-2 block - mid + eor3 $res1b, $ctr_t1b, $ctr6b, $t1.16b @ AES final-1 block - result + + pmull2 $rk2q1, $res0.2d, $h3.2d @ GHASH final-2 block - high + + pmull $rk3q1, $res0.1d, $h3.1d @ GHASH final-2 block - low + pmull $rk4v.1q, $rk4v.1d, $h34k.1d @ GHASH final-2 block - mid + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-2 block - high + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-2 block - mid + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-2 block - low +.L128_enc_blocks_more_than_1: @ blocks left > 1 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-1 block - store result + + ldr $h2q, [$current_tag, #64] @ load h2l | h2h + ext $h2.16b, $h2.16b, $h2.16b, #8 + rev64 $res0b, $res1b @ GHASH final-1 block + ldr $ctr_t1q, [$input_ptr], #16 @ AES final block - load plaintext + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + movi $t0.8b, #0 @ surpress further partial tag feed in + ins $rk4v.d[0], $res0.d[1] @ GHASH final-1 block - mid + eor3 $res1b, $ctr_t1b, $ctr7b, $t1.16b @ AES final block - result + + pmull2 $rk2q1, $res0.2d, $h2.2d @ GHASH final-1 block - high + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-1 block - mid + + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-1 block - mid + + pmull $rk3q1, $res0.1d, $h2.1d @ GHASH final-1 block - low + pmull2 $rk4v.1q, $rk4v.2d, $h12k.2d @ GHASH final-1 block - mid + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-1 block - high + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-1 block - mid + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-1 block - low +.L128_enc_blocks_less_than_1: @ blocks left <= 1 + + rev32 $rtmp_ctr.16b, $rtmp_ctr.16b + str $rtmp_ctrq, [$counter] @ store the updated counter + and $bit_length, $bit_length, #127 @ bit_length %= 128 + + sub $bit_length, $bit_length, #128 @ bit_length -= 128 + + neg $bit_length, $bit_length @ bit_length = 128 - #bits in input (in range [1,128]) + + mvn $temp0_x, xzr @ temp0_x = 0xffffffffffffffff + ld1 { $rk0}, [$output_ptr] @ load existing bytes where the possibly partial last block is to be stored + and $bit_length, $bit_length, #127 @ bit_length %= 128 + + lsr $temp0_x, $temp0_x, $bit_length @ temp0_x is mask for top 64b of last block + mvn $temp1_x, xzr @ temp1_x = 0xffffffffffffffff + cmp $bit_length, #64 + + csel $temp2_x, $temp1_x, $temp0_x, lt + csel $temp3_x, $temp0_x, xzr, lt + + mov $ctr0.d[1], $temp3_x + mov $ctr0.d[0], $temp2_x @ ctr0b is mask for last block + + and $res1b, $res1b, $ctr0b @ possibly partial last block has zeroes in highest bits + + rev64 $res0b, $res1b @ GHASH final block + + bif $res1b, $rk0, $ctr0b @ insert existing bytes in top end of result before storing + st1 { $res1b}, [$output_ptr] @ store all 16B + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $t0.d[0], $res0.d[1] @ GHASH final block - mid + + eor $t0.8b, $t0.8b, $res0.8b @ GHASH final block - mid + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + + pmull $t0.1q, $t0.1d, $h12k.1d @ GHASH final block - mid + + pmull2 $rk2q1, $res0.2d, $h1.2d @ GHASH final block - high + eor $acc_mb, $acc_mb, $t0.16b @ GHASH final block - mid + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + + pmull $rk3q1, $res0.1d, $h1.1d @ GHASH final block - low + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final block - high + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final block - low + + ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + + eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low + ext $acc_lb, $acc_lb, $acc_lb, #8 + rev64 $acc_lb, $acc_lb + st1 { $acc_l.16b }, [$current_tag] + lsr x0, $bit_length, #3 @ return sizes + + ldp d10, d11, [sp, #16] + ldp d12, d13, [sp, #32] + ldp d14, d15, [sp, #48] + ldp d8, d9, [sp], #80 + ret + +.L128_enc_ret: + mov w0, #0x0 + ret +.size unroll8_eor3_aes_gcm_enc_128_kernel,.-unroll8_eor3_aes_gcm_enc_128_kernel +___ + +######################################################################################### +# size_t unroll8_eor3_aes_gcm_dec_128_kernel(const unsigned char *in, +# size_t len, +# unsigned char *out, +# u64 *Xi, +# unsigned char ivec[16], +# const void *key); +# +$code.=<<___; +.global unroll8_eor3_aes_gcm_dec_128_kernel +.type unroll8_eor3_aes_gcm_dec_128_kernel,%function +.align 4 +unroll8_eor3_aes_gcm_dec_128_kernel: + AARCH64_VALID_CALL_TARGET + cbz x1, .L128_dec_ret + stp d8, d9, [sp, #-80]! + mov $counter, x4 + mov $cc, x5 + stp d10, d11, [sp, #16] + stp d12, d13, [sp, #32] + stp d14, d15, [sp, #48] + mov x5, #0xc200000000000000 + stp x5, xzr, [sp, #64] + add $modulo_constant, sp, #64 + + lsr $main_end_input_ptr, $bit_length, #3 @ byte_len + ld1 { $ctr0b}, [$counter] @ CTR block 0 + + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + sub $main_end_input_ptr, $main_end_input_ptr, #1 @ byte_len - 1 + + mov $constant_temp, #0x100000000 @ set up counter increment + movi $rctr_inc.16b, #0x0 + mov $rctr_inc.d[1], $constant_temp + ld1 { $acc_lb}, [$current_tag] + ext $acc_lb, $acc_lb, $acc_lb, #8 + rev64 $acc_lb, $acc_lb + + rev32 $rtmp_ctr.16b, $ctr0.16b @ set up reversed counter + + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 0 + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 0 + + rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 1 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 1 + + and $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffff80 @ number of bytes to be processed in main loop (at least 1 byte must be handled by tail) + + rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 2 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 2 + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 0 + + rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 3 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 3 + + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 1 + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 1 + + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 4 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 4 + + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 5 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 5 + + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 0 + + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 6 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 6 + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 0 + + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 0 + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 0 + + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 7 + + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 0 + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 1 + + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 0 + + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 1 + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 1 + + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 1 + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 1 + + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 2 + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 2 + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 1 + + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 2 + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 2 + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 2 + + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 2 + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 2 + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 2 + + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 3 + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 3 + + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 3 + + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 3 + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 3 + + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 3 + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 3 + + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 4 + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 4 + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 3 + + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 4 + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 4 + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 4 + + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 4 + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 4 + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 4 + + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 5 + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 5 + + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 5 + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 5 + + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 5 + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 5 + + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 5 + + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 6 + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 6 + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 5 + + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 6 + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 6 + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 6 + + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 6 + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 6 + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 6 + + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 7 + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 7 + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 7 + + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 7 + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 7 + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 7 + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 7 + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 7 + + add $main_end_input_ptr, $main_end_input_ptr, $input_ptr + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 7 + + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 8 + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 8 + + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 8 + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 8 + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 8 + + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 8 + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 8 + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 8 + + aese $ctr0b, $rk9 @ AES block 0 - round 9 + aese $ctr1b, $rk9 @ AES block 1 - round 9 + aese $ctr6b, $rk9 @ AES block 6 - round 9 + + ldr $rk10q, [$cc, #160] @ load rk10 + aese $ctr4b, $rk9 @ AES block 4 - round 9 + aese $ctr3b, $rk9 @ AES block 3 - round 9 + + aese $ctr2b, $rk9 @ AES block 2 - round 9 + aese $ctr5b, $rk9 @ AES block 5 - round 9 + aese $ctr7b, $rk9 @ AES block 7 - round 9 + + add $end_input_ptr, $input_ptr, $bit_length, lsr #3 @ end_input_ptr + cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks + b.ge .L128_dec_tail @ handle tail + + ldp $res0q, $res1q, [$input_ptr], #32 @ AES block 0, 1 - load ciphertext + + eor3 $ctr0b, $res0b, $ctr0b, $rk10 @ AES block 0 - result + eor3 $ctr1b, $res1b, $ctr1b, $rk10 @ AES block 1 - result + stp $ctr0q, $ctr1q, [$output_ptr], #32 @ AES block 0, 1 - store result + + rev32 $ctr0.16b, $rtmp_ctr.16b @ CTR block 8 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8 + ldp $res2q, $res3q, [$input_ptr], #32 @ AES block 2, 3 - load ciphertext + + ldp $res4q, $res5q, [$input_ptr], #32 @ AES block 4, 5 - load ciphertext + + rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 9 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 9 + ldp $res6q, $res7q, [$input_ptr], #32 @ AES block 6, 7 - load ciphertext + + eor3 $ctr3b, $res3b, $ctr3b, $rk10 @ AES block 3 - result + eor3 $ctr2b, $res2b, $ctr2b, $rk10 @ AES block 2 - result + stp $ctr2q, $ctr3q, [$output_ptr], #32 @ AES block 2, 3 - store result + + rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 10 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 10 + + eor3 $ctr6b, $res6b, $ctr6b, $rk10 @ AES block 6 - result + + rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 11 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 11 + + eor3 $ctr4b, $res4b, $ctr4b, $rk10 @ AES block 4 - result + eor3 $ctr5b, $res5b, $ctr5b, $rk10 @ AES block 5 - result + stp $ctr4q, $ctr5q, [$output_ptr], #32 @ AES block 4, 5 - store result + + eor3 $ctr7b, $res7b, $ctr7b, $rk10 @ AES block 7 - result + stp $ctr6q, $ctr7q, [$output_ptr], #32 @ AES block 6, 7 - store result + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 12 + + cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 12 + b.ge .L128_dec_prepretail @ do prepretail + +.L128_dec_main_loop: @ main loop start + ldr $h7q, [$current_tag, #176] @ load h7l | h7h + ext $h7.16b, $h7.16b, $h7.16b, #8 + ldr $h8q, [$current_tag, #208] @ load h7l | h7h + ext $h8.16b, $h8.16b, $h8.16b, #8 + + rev64 $res1b, $res1b @ GHASH block 8k+1 + rev64 $res0b, $res0b @ GHASH block 8k + ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0 + + rev64 $res6b, $res6b @ GHASH block 8k+6 + ldr $h5q, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + ldr $h6q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + + eor $res0b, $res0b, $acc_lb @ PRE 1 + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13 + + rev64 $res2b, $res2b @ GHASH block 8k+2 + rev64 $res4b, $res4b @ GHASH block 8k+4 + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14 + ldr $h56kq, [$current_tag, #144] @ load h6k | h5k + ldr $h78kq, [$current_tag, #192] @ load h6k | h5k + + pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high + rev64 $res3b, $res3b @ GHASH block 8k+3 + + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15 + trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + rev64 $res5b, $res5b @ GHASH block 8k+5 + + pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low + trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + + pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0 + pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high + + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0 + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0 + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0 + + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0 + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0 + eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high + + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0 + eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0 + + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1 + eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low + eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high + + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1 + + pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low + trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid + + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1 + + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1 + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1 + pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low + + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1 + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1 + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1 + + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2 + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2 + eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low + + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2 + eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + ldr $h2q, [$current_tag, #64] @ load h1l | h1h + ext $h2.16b, $h2.16b, $h2.16b, #8 + + eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2 + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2 + + trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2 + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2 + + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2 + pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid + pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid + + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3 + rev64 $res7b, $res7b @ GHASH block 8k+7 + pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high + + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low + eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3 + trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3 + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3 + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3 + + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3 + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3 + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3 + + pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high + pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low + pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high + + pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4 + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4 + + eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4 + + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4 + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4 + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4 + + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4 + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4 + trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5 + pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid + + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5 + eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5 + + pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5 + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5 + + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5 + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5 + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5 + + pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high + eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low + + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6 + eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6 + + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6 + pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6 + + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6 + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6 + pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low + + pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15 + + eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6 + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7 + + rev32 $h1.16b, $rtmp_ctr.16b @ CTR block 8k+16 + eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+16 + + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7 + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7 + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7 + + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7 + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7 + rev32 $h2.16b, $rtmp_ctr.16b @ CTR block 8k+17 + + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7 + ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+17 + + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8 + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8 + ldp $res0q, $res1q, [$input_ptr], #32 @ AES block 8k+8, 8k+9 - load ciphertext + + ldp $res2q, $res3q, [$input_ptr], #32 @ AES block 8k+10, 8k+11 - load ciphertext + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8 + rev32 $h3.16b, $rtmp_ctr.16b @ CTR block 8k+18 + + ldp $res4q, $res5q, [$input_ptr], #32 @ AES block 8k+12, 8k+13 - load ciphertext + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8 + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + + ldp $res6q, $res7q, [$input_ptr], #32 @ AES block 8k+14, 8k+15 - load ciphertext + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+18 + + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8 + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8 + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8 + + aese $ctr0b, $rk9 @ AES block 8k+8 - round 9 + aese $ctr1b, $rk9 @ AES block 8k+9 - round 9 + ldr $rk10q, [$cc, #160] @ load rk10 + + aese $ctr6b, $rk9 @ AES block 8k+14 - round 9 + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + aese $ctr2b, $rk9 @ AES block 8k+10 - round 9 + + aese $ctr7b, $rk9 @ AES block 8k+15 - round 9 + aese $ctr4b, $rk9 @ AES block 8k+12 - round 9 + ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + + rev32 $h4.16b, $rtmp_ctr.16b @ CTR block 8k+19 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+19 + + aese $ctr3b, $rk9 @ AES block 8k+11 - round 9 + aese $ctr5b, $rk9 @ AES block 8k+13 - round 9 + eor3 $ctr1b, $res1b, $ctr1b, $rk10 @ AES block 8k+9 - result + + eor3 $ctr0b, $res0b, $ctr0b, $rk10 @ AES block 8k+8 - result + eor3 $ctr7b, $res7b, $ctr7b, $rk10 @ AES block 8k+15 - result + eor3 $ctr6b, $res6b, $ctr6b, $rk10 @ AES block 8k+14 - result + + eor3 $ctr2b, $res2b, $ctr2b, $rk10 @ AES block 8k+10 - result + stp $ctr0q, $ctr1q, [$output_ptr], #32 @ AES block 8k+8, 8k+9 - store result + mov $ctr1.16b, $h2.16b @ CTR block 8k+17 + + eor3 $ctr4b, $res4b, $ctr4b, $rk10 @ AES block 8k+12 - result + eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low + mov $ctr0.16b, $h1.16b @ CTR block 8k+16 + + eor3 $ctr3b, $res3b, $ctr3b, $rk10 @ AES block 8k+11 - result + cmp $input_ptr, $main_end_input_ptr @ LOOP CONTROL + stp $ctr2q, $ctr3q, [$output_ptr], #32 @ AES block 8k+10, 8k+11 - store result + + eor3 $ctr5b, $res5b, $ctr5b, $rk10 @ AES block 8k+13 - result + mov $ctr2.16b, $h3.16b @ CTR block 8k+18 + + stp $ctr4q, $ctr5q, [$output_ptr], #32 @ AES block 8k+12, 8k+13 - store result + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 8k+20 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+20 + + stp $ctr6q, $ctr7q, [$output_ptr], #32 @ AES block 8k+14, 8k+15 - store result + mov $ctr3.16b, $h4.16b @ CTR block 8k+19 + b.lt .L128_dec_main_loop + +.L128_dec_prepretail: @ PREPRETAIL + rev64 $res3b, $res3b @ GHASH block 8k+3 + ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0 + rev64 $res0b, $res0b @ GHASH block 8k + + rev64 $res2b, $res2b @ GHASH block 8k+2 + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13 + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + + ldr $h7q, [$current_tag, #176] @ load h7l | h7h + ext $h7.16b, $h7.16b, $h7.16b, #8 + ldr $h8q, [$current_tag, #208] @ load h8l | h8h + ext $h8.16b, $h8.16b, $h8.16b, #8 + eor $res0b, $res0b, $acc_lb @ PRE 1 + rev64 $res1b, $res1b @ GHASH block 8k+1 + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13 + ldr $h5q, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + ldr $h6q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + rev64 $res5b, $res5b @ GHASH block 8k+5 + + rev64 $res4b, $res4b @ GHASH block 8k+4 + + rev64 $res6b, $res6b @ GHASH block 8k+6 + + ldr $h56kq, [$current_tag, #144] @ load h6k | h5k + ldr $h78kq, [$current_tag, #192] @ load h6k | h5k + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14 + + pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high + + trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high + + pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low + pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0 + + eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0 + eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid + + pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15 + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0 + + eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high + trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0 + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0 + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0 + + pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid + pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid + pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low + + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1 + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0 + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0 + + eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low + eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid + + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1 + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1 + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1 + + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low + pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid + + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1 + pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid + + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1 + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1 + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1 + + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + ldr $h2q, [$current_tag, #64] @ load h1l | h1h + ext $h2.16b, $h2.16b, $h2.16b, #8 + eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2 + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2 + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2 + + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2 + trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2 + + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2 + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2 + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2 + + pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high + pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low + trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + rev64 $res7b, $res7b @ GHASH block 8k+7 + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3 + + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high + pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low + + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3 + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3 + trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + + pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high + pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low + trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3 + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3 + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3 + + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3 + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3 + eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + + eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4 + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4 + + eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4 + pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid + + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4 + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4 + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4 + + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4 + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4 + pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid + + pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high + pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid + pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid + + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5 + + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low + eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low + + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5 + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5 + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5 + + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5 + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5 + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5 + + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5 + eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low + + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6 + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6 + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6 + + eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6 + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6 + + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6 + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6 + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6 + + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7 + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + + pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7 + ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7 + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7 + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7 + + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7 + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7 + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7 + + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + ldr $rk10q, [$cc, #160] @ load rk10 + + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8 + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8 + + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8 + ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8 + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8 + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8 + + aese $ctr6b, $rk9 @ AES block 8k+14 - round 9 + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8 + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8 + + eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15 + aese $ctr2b, $rk9 @ AES block 8k+10 - round 9 + + aese $ctr3b, $rk9 @ AES block 8k+11 - round 9 + aese $ctr5b, $rk9 @ AES block 8k+13 - round 9 + aese $ctr0b, $rk9 @ AES block 8k+8 - round 9 + + aese $ctr4b, $rk9 @ AES block 8k+12 - round 9 + aese $ctr1b, $rk9 @ AES block 8k+9 - round 9 + aese $ctr7b, $rk9 @ AES block 8k+15 - round 9 + +.L128_dec_tail: @ TAIL + + mov $t1.16b, $rk10 + sub $main_end_input_ptr, $end_input_ptr, $input_ptr @ main_end_input_ptr is number of bytes left to process + + cmp $main_end_input_ptr, #112 + + ldp $h78kq, $h8q, [$current_tag, #192] @ load h7l | h7h + ext $h8.16b, $h8.16b, $h8.16b, #8 + ldr $res1q, [$input_ptr], #16 @ AES block 8k+8 - load ciphertext + + ldp $h5q, $h56kq, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + ext $t0.16b, $acc_lb, $acc_lb, #8 @ prepare final partial tag + + ldp $h6q, $h7q, [$current_tag, #160] @ load h6k | h5k + ext $h6.16b, $h6.16b, $h6.16b, #8 + ext $h7.16b, $h7.16b, $h7.16b, #8 + + eor3 $res4b, $res1b, $ctr0b, $t1.16b @ AES block 8k+8 - result + b.gt .L128_dec_blocks_more_than_7 + + cmp $main_end_input_ptr, #96 + mov $ctr7b, $ctr6b + movi $acc_l.8b, #0 + + movi $acc_h.8b, #0 + mov $ctr6b, $ctr5b + mov $ctr5b, $ctr4b + + mov $ctr4b, $ctr3b + mov $ctr3b, $ctr2b + mov $ctr2b, $ctr1b + + movi $acc_m.8b, #0 + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + b.gt .L128_dec_blocks_more_than_6 + + cmp $main_end_input_ptr, #80 + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr5b + mov $ctr5b, $ctr4b + + mov $ctr4b, $ctr3b + mov $ctr3b, $ctr1b + b.gt .L128_dec_blocks_more_than_5 + + cmp $main_end_input_ptr, #64 + + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr5b + mov $ctr5b, $ctr4b + + mov $ctr4b, $ctr1b + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + b.gt .L128_dec_blocks_more_than_4 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr5b + + mov $ctr5b, $ctr1b + cmp $main_end_input_ptr, #48 + b.gt .L128_dec_blocks_more_than_3 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + mov $ctr7b, $ctr6b + cmp $main_end_input_ptr, #32 + + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + mov $ctr6b, $ctr1b + b.gt .L128_dec_blocks_more_than_2 + + cmp $main_end_input_ptr, #16 + + mov $ctr7b, $ctr1b + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + b.gt L128_dec_blocks_more_than_1 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + b .L128_dec_blocks_less_than_1 +.L128_dec_blocks_more_than_7: @ blocks left > 7 + rev64 $res0b, $res1b @ GHASH final-7 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $acc_m.d[0], $h78k.d[1] @ GHASH final-7 block - mid + + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH final-7 block - low + ins $rk4v.d[0], $res0.d[1] @ GHASH final-7 block - mid + + movi $t0.8b, #0 @ surpress further partial tag feed in + ldr $res1q, [$input_ptr], #16 @ AES final-6 block - load ciphertext + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-7 block - mid + + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH final-7 block - high + st1 { $res4b}, [$output_ptr], #16 @ AES final-7 block - store result + eor3 $res4b, $res1b, $ctr1b, $t1.16b @ AES final-6 block - result + + pmull $acc_m.1q, $rk4v.1d, $acc_m.1d @ GHASH final-7 block - mid +.L128_dec_blocks_more_than_6: @ blocks left > 6 + + rev64 $res0b, $res1b @ GHASH final-6 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-6 block - mid + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-6 block - mid + + pmull $rk3q1, $res0.1d, $h7.1d @ GHASH final-6 block - low + ldr $res1q, [$input_ptr], #16 @ AES final-5 block - load ciphertext + movi $t0.8b, #0 @ surpress further partial tag feed in + + pmull $rk4v.1q, $rk4v.1d, $h78k.1d @ GHASH final-6 block - mid + st1 { $res4b}, [$output_ptr], #16 @ AES final-6 block - store result + pmull2 $rk2q1, $res0.2d, $h7.2d @ GHASH final-6 block - high + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-6 block - low + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-6 block - high + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-6 block - mid + eor3 $res4b, $res1b, $ctr2b, $t1.16b @ AES final-5 block - result +.L128_dec_blocks_more_than_5: @ blocks left > 5 + + rev64 $res0b, $res1b @ GHASH final-5 block + + ldr $res1q, [$input_ptr], #16 @ AES final-4 block - load ciphertext + st1 { $res4b}, [$output_ptr], #16 @ AES final-5 block - store result + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-5 block - mid + + eor3 $res4b, $res1b, $ctr3b, $t1.16b @ AES final-4 block - result + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-5 block - mid + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-5 block - mid + pmull $rk3q1, $res0.1d, $h6.1d @ GHASH final-5 block - low + movi $t0.8b, #0 @ surpress further partial tag feed in + + pmull2 $rk4v.1q, $rk4v.2d, $h56k.2d @ GHASH final-5 block - mid + pmull2 $rk2q1, $res0.2d, $h6.2d @ GHASH final-5 block - high + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-5 block - low + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-5 block - mid + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-5 block - high +.L128_dec_blocks_more_than_4: @ blocks left > 4 + + rev64 $res0b, $res1b @ GHASH final-4 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + ldr $res1q, [$input_ptr], #16 @ AES final-3 block - load ciphertext + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-4 block - mid + movi $t0.8b, #0 @ surpress further partial tag feed in + pmull2 $rk2q1, $res0.2d, $h5.2d @ GHASH final-4 block - high + + pmull $rk3q1, $res0.1d, $h5.1d @ GHASH final-4 block - low + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-4 block - high + + st1 { $res4b}, [$output_ptr], #16 @ AES final-4 block - store result + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-4 block - mid + + eor3 $res4b, $res1b, $ctr4b, $t1.16b @ AES final-3 block - result + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-4 block - low + + pmull $rk4v.1q, $rk4v.1d, $h56k.1d @ GHASH final-4 block - mid + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-4 block - mid +.L128_dec_blocks_more_than_3: @ blocks left > 3 + + st1 { $res4b}, [$output_ptr], #16 @ AES final-3 block - store result + rev64 $res0b, $res1b @ GHASH final-3 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-3 block - mid + + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-3 block - mid + + ldr $res1q, [$input_ptr], #16 @ AES final-2 block - load ciphertext + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-3 block - mid + pmull $rk3q1, $res0.1d, $h4.1d @ GHASH final-3 block - low + pmull2 $rk2q1, $res0.2d, $h4.2d @ GHASH final-3 block - high + + movi $t0.8b, #0 @ surpress further partial tag feed in + eor3 $res4b, $res1b, $ctr5b, $t1.16b @ AES final-2 block - result + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-3 block - low + + pmull2 $rk4v.1q, $rk4v.2d, $h34k.2d @ GHASH final-3 block - mid + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-3 block - high + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-3 block - mid +.L128_dec_blocks_more_than_2: @ blocks left > 2 + + rev64 $res0b, $res1b @ GHASH final-2 block + + st1 { $res4b}, [$output_ptr], #16 @ AES final-2 block - store result + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + movi $t0.8b, #0 @ surpress further partial tag feed in + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-2 block - mid + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-2 block - mid + + pmull $rk3q1, $res0.1d, $h3.1d @ GHASH final-2 block - low + + pmull2 $rk2q1, $res0.2d, $h3.2d @ GHASH final-2 block - high + pmull $rk4v.1q, $rk4v.1d, $h34k.1d @ GHASH final-2 block - mid + ldr $res1q, [$input_ptr], #16 @ AES final-1 block - load ciphertext + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-2 block - mid + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-2 block - low + + eor3 $res4b, $res1b, $ctr6b, $t1.16b @ AES final-1 block - result + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-2 block - high +.L128_dec_blocks_more_than_1: @ blocks left > 1 + + st1 { $res4b}, [$output_ptr], #16 @ AES final-1 block - store result + rev64 $res0b, $res1b @ GHASH final-1 block + + ldr $h2q, [$current_tag, #64] @ load h1l | h1h + ext $h2.16b, $h2.16b, $h2.16b, #8 + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + movi $t0.8b, #0 @ surpress further partial tag feed in + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-1 block - mid + + ldr $res1q, [$input_ptr], #16 @ AES final block - load ciphertext + pmull2 $rk2q1, $res0.2d, $h2.2d @ GHASH final-1 block - high + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-1 block - mid + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-1 block - high + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-1 block - mid + eor3 $res4b, $res1b, $ctr7b, $t1.16b @ AES final block - result + + pmull $rk3q1, $res0.1d, $h2.1d @ GHASH final-1 block - low + + pmull2 $rk4v.1q, $rk4v.2d, $h12k.2d @ GHASH final-1 block - mid + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-1 block - low + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-1 block - mid +.L128_dec_blocks_less_than_1: @ blocks left <= 1 + + and $bit_length, $bit_length, #127 @ bit_length %= 128 + + sub $bit_length, $bit_length, #128 @ bit_length -= 128 + + neg $bit_length, $bit_length @ bit_length = 128 - #bits in input (in range [1,128]) + + mvn $temp0_x, xzr @ temp0_x = 0xffffffffffffffff + and $bit_length, $bit_length, #127 @ bit_length %= 128 + + lsr $temp0_x, $temp0_x, $bit_length @ temp0_x is mask for top 64b of last block + cmp $bit_length, #64 + mvn $temp1_x, xzr @ temp1_x = 0xffffffffffffffff + + csel $temp2_x, $temp1_x, $temp0_x, lt + csel $temp3_x, $temp0_x, xzr, lt + + mov $ctr0.d[1], $temp3_x + mov $ctr0.d[0], $temp2_x @ ctr0b is mask for last block + + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + ld1 { $rk0}, [$output_ptr] @ load existing bytes where the possibly partial last block is to be stored + + and $res1b, $res1b, $ctr0b @ possibly partial last block has zeroes in highest bits + + rev64 $res0b, $res1b @ GHASH final block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + pmull2 $rk2q1, $res0.2d, $h1.2d @ GHASH final block - high + ins $t0.d[0], $res0.d[1] @ GHASH final block - mid + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final block - high + eor $t0.8b, $t0.8b, $res0.8b @ GHASH final block - mid + + bif $res4b, $rk0, $ctr0b @ insert existing bytes in top end of result before storing + + pmull $t0.1q, $t0.1d, $h12k.1d @ GHASH final block - mid + st1 { $res4b}, [$output_ptr] @ store all 16B + + pmull $rk3q1, $res0.1d, $h1.1d @ GHASH final block - low + + eor $acc_mb, $acc_mb, $t0.16b @ GHASH final block - mid + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final block - low + + eor $t10.16b, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + + pmull $t11.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + ext $acc_hb, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + + eor $acc_mb, $acc_mb, $t10.16b @ MODULO - karatsuba tidy up + + eor3 $acc_mb, $acc_mb, $acc_hb, $t11.16b @ MODULO - fold into mid + + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + ext $acc_mb, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + + eor3 $acc_lb, $acc_lb, $acc_mb, $acc_hb @ MODULO - fold into low + ext $acc_lb, $acc_lb, $acc_lb, #8 + rev64 $acc_lb, $acc_lb + st1 { $acc_l.16b }, [$current_tag] + rev32 $rtmp_ctr.16b, $rtmp_ctr.16b + + str $rtmp_ctrq, [$counter] @ store the updated counter + + lsr x0, $bit_length, #3 + + ldp d10, d11, [sp, #16] + ldp d12, d13, [sp, #32] + ldp d14, d15, [sp, #48] + ldp d8, d9, [sp], #80 + ret +.L128_dec_ret: + mov w0, #0x0 + ret +.size unroll8_eor3_aes_gcm_dec_128_kernel,.-unroll8_eor3_aes_gcm_dec_128_kernel +___ +} + +{ +my ($end_input_ptr,$main_end_input_ptr,$temp0_x,$temp1_x)=map("x$_",(4..7)); +my ($temp2_x,$temp3_x)=map("x$_",(13..14)); +my ($ctr0b,$ctr1b,$ctr2b,$ctr3b,$ctr4b,$ctr5b,$ctr6b,$ctr7b,$res0b,$res1b,$res2b,$res3b,$res4b,$res5b,$res6b,$res7b)=map("v$_.16b",(0..15)); +my ($ctr0,$ctr1,$ctr2,$ctr3,$ctr4,$ctr5,$ctr6,$ctr7,$res0,$res1,$res2,$res3,$res4,$res5,$res6,$res7)=map("v$_",(0..15)); +my ($ctr0d,$ctr1d,$ctr2d,$ctr3d,$ctr4d,$ctr5d,$ctr6d,$ctr7d)=map("d$_",(0..7)); +my ($ctr0q,$ctr1q,$ctr2q,$ctr3q,$ctr4q,$ctr5q,$ctr6q,$ctr7q)=map("q$_",(0..7)); +my ($res0q,$res1q,$res2q,$res3q,$res4q,$res5q,$res6q,$res7q)=map("q$_",(8..15)); + +my ($ctr_t0,$ctr_t1,$ctr_t2,$ctr_t3,$ctr_t4,$ctr_t5,$ctr_t6,$ctr_t7)=map("v$_",(8..15)); +my ($ctr_t0b,$ctr_t1b,$ctr_t2b,$ctr_t3b,$ctr_t4b,$ctr_t5b,$ctr_t6b,$ctr_t7b)=map("v$_.16b",(8..15)); +my ($ctr_t0q,$ctr_t1q,$ctr_t2q,$ctr_t3q,$ctr_t4q,$ctr_t5q,$ctr_t6q,$ctr_t7q)=map("q$_",(8..15)); + +my ($acc_hb,$acc_mb,$acc_lb)=map("v$_.16b",(17..19)); +my ($acc_h,$acc_m,$acc_l)=map("v$_",(17..19)); + +my ($h1,$h12k,$h2,$h3,$h34k,$h4)=map("v$_",(20..25)); +my ($h5,$h56k,$h6,$h7,$h78k,$h8)=map("v$_",(20..25)); +my ($h1q,$h12kq,$h2q,$h3q,$h34kq,$h4q)=map("q$_",(20..25)); +my ($h5q,$h56kq,$h6q,$h7q,$h78kq,$h8q)=map("q$_",(20..25)); + +my $t0="v16"; +my $t0d="d16"; + +my $t1="v29"; +my $t2=$res1; +my $t3=$t1; + +my $t4=$res0; +my $t5=$res2; +my $t6=$t0; + +my $t7=$res3; +my $t8=$res4; +my $t9=$res5; + +my $t10=$res6; +my $t11="v21"; +my $t12=$t1; + +my $rtmp_ctr="v30"; +my $rtmp_ctrq="q30"; +my $rctr_inc="v31"; +my $rctr_incd="d31"; + +my $mod_constantd=$t0d; +my $mod_constant=$t0; + +my ($rk0,$rk1,$rk2)=map("v$_.16b",(26..28)); +my ($rk3,$rk4,$rk5)=map("v$_.16b",(26..28)); +my ($rk6,$rk7,$rk8)=map("v$_.16b",(26..28)); +my ($rk9,$rk10,$rk11)=map("v$_.16b",(26..28)); +my ($rk12,$rk13,$rk14)=map("v$_.16b",(26..28)); +my ($rk0q,$rk1q,$rk2q)=map("q$_",(26..28)); +my ($rk3q,$rk4q,$rk5q)=map("q$_",(26..28)); +my ($rk6q,$rk7q,$rk8q)=map("q$_",(26..28)); +my ($rk9q,$rk10q,$rk11q)=map("q$_",(26..28)); +my ($rk12q,$rk13q,$rk14q)=map("q$_",(26..28)); +my $rk2q1="v28.1q"; +my $rk3q1="v26.1q"; +my $rk4v="v27"; + +######################################################################################### +# size_t unroll8_eor3_aes_gcm_enc_192_kernel(const unsigned char *in, +# size_t len, +# unsigned char *out, +# const void *key, +# unsigned char ivec[16], +# u64 *Xi); +# +$code.=<<___; +.global unroll8_eor3_aes_gcm_enc_192_kernel +.type unroll8_eor3_aes_gcm_enc_192_kernel,%function +.align 4 +unroll8_eor3_aes_gcm_enc_192_kernel: + AARCH64_VALID_CALL_TARGET + cbz x1, .L192_enc_ret + stp d8, d9, [sp, #-80]! + mov $counter, x4 + mov $cc, x5 + stp d10, d11, [sp, #16] + stp d12, d13, [sp, #32] + stp d14, d15, [sp, #48] + mov x5, #0xc200000000000000 + stp x5, xzr, [sp, #64] + add $modulo_constant, sp, #64 + + lsr $main_end_input_ptr, $bit_length, #3 @ byte_len + ld1 { $ctr0b}, [$counter] @ CTR block 0 + + mov $constant_temp, #0x100000000 @ set up counter increment + movi $rctr_inc.16b, #0x0 + mov $rctr_inc.d[1], $constant_temp + + rev32 $rtmp_ctr.16b, $ctr0.16b @ set up reversed counter + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 0 + + rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 1 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 1 + + rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 2 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 2 + + rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 3 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 3 + + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 4 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 4 + sub $main_end_input_ptr, $main_end_input_ptr, #1 @ byte_len - 1 + + and $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffff80 @ number of bytes to be processed in main loop (at least 1 byte must be handled by tail) + + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 5 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 5 + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + + add $main_end_input_ptr, $main_end_input_ptr, $input_ptr + + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 6 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 6 + + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 7 + + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 0 + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 0 + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 0 + + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 0 + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 0 + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 0 + + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 0 + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 0 + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 1 + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 1 + + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 1 + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 1 + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 1 + + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 2 + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 1 + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 1 + + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 1 + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 2 + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 2 + + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 2 + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 2 + + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 2 + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 2 + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 2 + + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 3 + + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 3 + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 3 + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 3 + + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 3 + + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 3 + + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 3 + + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 4 + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 4 + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 3 + + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 4 + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 4 + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 4 + + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 4 + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 4 + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 4 + + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 5 + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 5 + + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 5 + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 5 + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 5 + + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 5 + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 5 + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 5 + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 7 + + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 6 + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 6 + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 6 + + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 6 + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 6 + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 6 + + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 6 + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 6 + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 7 + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 7 + + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 7 + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 7 + + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 7 + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 7 + + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 7 + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 7 + + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 8 + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 8 + + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 8 + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 8 + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 8 + + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 8 + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 8 + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 8 + + add $end_input_ptr, $input_ptr, $bit_length, lsr #3 @ end_input_ptr + cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks + aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 9 + + ld1 { $acc_lb}, [$current_tag] + ext $acc_lb, $acc_lb, $acc_lb, #8 + rev64 $acc_lb, $acc_lb + ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11 + + aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 9 + aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 9 + + aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 9 + aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 9 + + aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 9 + aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 9 + + aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 14 - round 10 + aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 9 + aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 11 - round 10 + + aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 9 - round 10 + aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 13 - round 10 + aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 12 - round 10 + + aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 8 - round 10 + aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 10 - round 10 + aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 15 - round 10 + + aese $ctr6b, $rk11 @ AES block 14 - round 11 + aese $ctr3b, $rk11 @ AES block 11 - round 11 + + aese $ctr4b, $rk11 @ AES block 12 - round 11 + aese $ctr7b, $rk11 @ AES block 15 - round 11 + ldr $rk12q, [$cc, #192] @ load rk12 + + aese $ctr1b, $rk11 @ AES block 9 - round 11 + aese $ctr5b, $rk11 @ AES block 13 - round 11 + + aese $ctr2b, $rk11 @ AES block 10 - round 11 + aese $ctr0b, $rk11 @ AES block 8 - round 11 + b.ge .L192_enc_tail @ handle tail + + ldp $ctr_t0q, $ctr_t1q, [$input_ptr], #32 @ AES block 0, 1 - load plaintext + + ldp $ctr_t2q, $ctr_t3q, [$input_ptr], #32 @ AES block 2, 3 - load plaintext + + ldp $ctr_t4q, $ctr_t5q, [$input_ptr], #32 @ AES block 4, 5 - load plaintext + + ldp $ctr_t6q, $ctr_t7q, [$input_ptr], #32 @ AES block 6, 7 - load plaintext + + eor3 $res0b, $ctr_t0b, $ctr0b, $rk12 @ AES block 0 - result + rev32 $ctr0.16b, $rtmp_ctr.16b @ CTR block 8 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8 + + eor3 $res3b, $ctr_t3b, $ctr3b, $rk12 @ AES block 3 - result + eor3 $res1b, $ctr_t1b, $ctr1b, $rk12 @ AES block 1 - result + + rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 9 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 9 + eor3 $res4b, $ctr_t4b, $ctr4b, $rk12 @ AES block 4 - result + + eor3 $res5b, $ctr_t5b, $ctr5b, $rk12 @ AES block 5 - result + eor3 $res7b, $ctr_t7b, $ctr7b, $rk12 @ AES block 7 - result + stp $res0q, $res1q, [$output_ptr], #32 @ AES block 0, 1 - store result + + eor3 $res2b, $ctr_t2b, $ctr2b, $rk12 @ AES block 2 - result + rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 10 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 10 + + stp $res2q, $res3q, [$output_ptr], #32 @ AES block 2, 3 - store result + cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks + + rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 11 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 11 + eor3 $res6b, $ctr_t6b, $ctr6b, $rk12 @ AES block 6 - result + + stp $res4q, $res5q, [$output_ptr], #32 @ AES block 4, 5 - store result + + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 12 + stp $res6q, $res7q, [$output_ptr], #32 @ AES block 6, 7 - store result + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 12 + + b.ge .L192_enc_prepretail @ do prepretail + +.L192_enc_main_loop: @ main loop start + rev64 $res4b, $res4b @ GHASH block 8k+4 (t0, t1, and t2 free) + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + rev64 $res2b, $res2b @ GHASH block 8k+2 + + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13 + ldr $h7q, [$current_tag, #176] @ load h7l | h7h + ext $h7.16b, $h7.16b, $h7.16b, #8 + ldr $h8q, [$current_tag, #208] @ load h8l | h8h + ext $h8.16b, $h8.16b, $h8.16b, #8 + + ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0 + rev64 $res0b, $res0b @ GHASH block 8k + ldr $h5q, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + ldr $h6q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + + rev64 $res1b, $res1b @ GHASH block 8k+1 + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14 + + eor $res0b, $res0b, $acc_lb @ PRE 1 + rev64 $res3b, $res3b @ GHASH block 8k+3 + rev64 $res5b, $res5b @ GHASH block 8k+5 (t0, t1, t2 and t3 free) + + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0 + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15 + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0 + + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0 + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0 + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0 + + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0 + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0 + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0 + + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1 + + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1 + pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high + pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low + + trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1 + ldr $h56kq, [$current_tag, #144] @ load h6k | h5k + ldr $h78kq, [$current_tag, #192] @ load h8k | h7k + + pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low + trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1 + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1 + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1 + + eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1 + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1 + + pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high + eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2 + + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2 + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2 + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2 + + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2 + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3 + eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high + + pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2 + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3 + + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2 + trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2 + + trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3 + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3 + eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + + pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid + pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid + pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low + + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3 + eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + + eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3 + eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low + + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4 + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4 + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3 + + pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4 + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3 + + pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4 + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4 + + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4 + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4 + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4 + + eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5 + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + ldr $h2q, [$current_tag, #64] @ load h1l | h1h + ext $h2.16b, $h2.16b, $h2.16b, #8 + + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5 + rev64 $res7b, $res7b @ GHASH block 8k+7 (t0, t1, t2 and t3 free) + + rev64 $res6b, $res6b @ GHASH block 8k+6 (t0, t1, and t2 free) + pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high + pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low + + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5 + trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5 + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5 + pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high + eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5 + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5 + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5 + + pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6 + trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6 + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6 + pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high + + pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low + trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6 + + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6 + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6 + + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6 + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7 + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6 + + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7 + eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + + pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid + + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7 + pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7 + + eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15 + + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7 + + pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid + pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7 + + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8 + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8 + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8 + + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8 + eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7 + + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8 + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8 + pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid + + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8 + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8 + ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11 + + eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low + rev32 $h1.16b, $rtmp_ctr.16b @ CTR block 8k+16 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+16 + + aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 9 + eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high + + aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 9 + aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 9 + ldp $ctr_t0q, $ctr_t1q, [$input_ptr], #32 @ AES block 8k+8, 8k+9 - load plaintext + + pmull $t11.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + rev32 $h2.16b, $rtmp_ctr.16b @ CTR block 8k+17 + aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 9 + + aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 9 + aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 9 + aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 9 + + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 9 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+17 + + aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 10 + aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 10 + ldr $rk12q, [$cc, #192] @ load rk12 + ext $t12.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + + aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 10 + aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 10 + ldp $ctr_t2q, $ctr_t3q, [$input_ptr], #32 @ AES block 8k+10, 8k+11 - load plaintext + + aese $ctr4b, $rk11 @ AES block 8k+12 - round 11 + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + ldp $ctr_t4q, $ctr_t5q, [$input_ptr], #32 @ AES block 8k+12, 8k+13 - load plaintext + + ldp $ctr_t6q, $ctr_t7q, [$input_ptr], #32 @ AES block 8k+14, 8k+15 - load plaintext + aese $ctr2b, $rk11 @ AES block 8k+10 - round 11 + aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 10 + + rev32 $h3.16b, $rtmp_ctr.16b @ CTR block 8k+18 + aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 10 + + aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 10 + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + + aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 10 + aese $ctr5b, $rk11 @ AES block 8k+13 - round 11 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+18 + + aese $ctr7b, $rk11 @ AES block 8k+15 - round 11 + aese $ctr0b, $rk11 @ AES block 8k+8 - round 11 + eor3 $res4b, $ctr_t4b, $ctr4b, $rk12 @ AES block 4 - result + + aese $ctr6b, $rk11 @ AES block 8k+14 - round 11 + aese $ctr3b, $rk11 @ AES block 8k+11 - round 11 + aese $ctr1b, $rk11 @ AES block 8k+9 - round 11 + + rev32 $h4.16b, $rtmp_ctr.16b @ CTR block 8k+19 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+19 + eor3 $res7b, $ctr_t7b, $ctr7b, $rk12 @ AES block 7 - result + + eor3 $res2b, $ctr_t2b, $ctr2b, $rk12 @ AES block 8k+10 - result + eor3 $res0b, $ctr_t0b, $ctr0b, $rk12 @ AES block 8k+8 - result + mov $ctr2.16b, $h3.16b @ CTR block 8k+18 + + eor3 $res1b, $ctr_t1b, $ctr1b, $rk12 @ AES block 8k+9 - result + mov $ctr1.16b, $h2.16b @ CTR block 8k+17 + stp $res0q, $res1q, [$output_ptr], #32 @ AES block 8k+8, 8k+9 - store result + ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + + eor3 $res6b, $ctr_t6b, $ctr6b, $rk12 @ AES block 6 - result + mov $ctr0.16b, $h1.16b @ CTR block 8k+16 + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 8k+20 + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+20 + eor3 $res5b, $ctr_t5b, $ctr5b, $rk12 @ AES block 5 - result + eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low + + eor3 $res3b, $ctr_t3b, $ctr3b, $rk12 @ AES block 8k+11 - result + mov $ctr3.16b, $h4.16b @ CTR block 8k+19 + + stp $res2q, $res3q, [$output_ptr], #32 @ AES block 8k+10, 8k+11 - store result + + stp $res4q, $res5q, [$output_ptr], #32 @ AES block 8k+12, 8k+13 - store result + + cmp $input_ptr, $main_end_input_ptr @ LOOP CONTROL + stp $res6q, $res7q, [$output_ptr], #32 @ AES block 8k+14, 8k+15 - store result + b.lt .L192_enc_main_loop + +.L192_enc_prepretail: @ PREPRETAIL + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13 + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13 + + ldr $h7q, [$current_tag, #176] @ load h7l | h7h + ext $h7.16b, $h7.16b, $h7.16b, #8 + ldr $h8q, [$current_tag, #208] @ load h8l | h8h + ext $h8.16b, $h8.16b, $h8.16b, #8 + rev64 $res0b, $res0b @ GHASH block 8k + ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0 + + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14 + ldr $h56kq, [$current_tag, #144] @ load h6k | h5k + ldr $h78kq, [$current_tag, #192] @ load h8k | h7k + + rev64 $res3b, $res3b @ GHASH block 8k+3 + rev64 $res2b, $res2b @ GHASH block 8k+2 + ldr $h5q, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + ldr $h6q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + + eor $res0b, $res0b, $acc_lb @ PRE 1 + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15 + rev64 $res1b, $res1b @ GHASH block 8k+1 + + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0 + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0 + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0 + + pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0 + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0 + + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0 + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0 + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high + + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1 + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low + trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + + trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0 + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + + pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low + eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1 + + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1 + eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1 + + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1 + pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high + pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high + + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1 + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1 + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1 + + pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2 + eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low + + pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2 + eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high + + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3 + trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2 + + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2 + pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid + trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2 + rev64 $res5b, $res5b @ GHASH block 8k+5 (t0, t1, t2 and t3 free) + rev64 $res6b, $res6b @ GHASH block 8k+6 (t0, t1, and t2 free) + + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2 + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2 + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2 + + eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3 + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3 + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3 + + eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid + eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3 + + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3 + pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid + + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + ldr $h2q, [$current_tag, #64] @ load h1l | h1h + ext $h2.16b, $h2.16b, $h2.16b, #8 + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3 + rev64 $res4b, $res4b @ GHASH block 8k+4 (t0, t1, and t2 free) + + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3 + pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4 + + trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4 + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4 + + eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4 + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4 + + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4 + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4 + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4 + + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5 + rev64 $res7b, $res7b @ GHASH block 8k+7 (t0, t1, t2 and t3 free) + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5 + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5 + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + + pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high + pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high + pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low + + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5 + trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + + pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high + pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low + pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low + + trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5 + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6 + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5 + + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5 + eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5 + + pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid + pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid + + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6 + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6 + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7 + + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6 + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6 + eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6 + eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7 + + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6 + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6 + + pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7 + eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low + + pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high + pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid + pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low + + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7 + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7 + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7 + eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + + eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low + eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high + + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + ext $t12.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7 + pmull $t11.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8 + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8 + + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7 + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8 + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8 + aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 9 + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8 + + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8 + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8 + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8 + + aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 9 + ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11 + aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 9 + + aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 9 + aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 9 + + ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 9 + aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 9 + aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 9 + + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + ldr $rk12q, [$cc, #192] @ load rk12 + + aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 10 + aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 10 + aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 10 + + eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low + aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 10 + aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 10 + + aese $ctr1b, $rk11 @ AES block 8k+9 - round 11 + aese $ctr7b, $rk11 @ AES block 8k+15 - round 11 + + aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 10 + aese $ctr3b, $rk11 @ AES block 8k+11 - round 11 + + aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 10 + aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 10 + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15 + aese $ctr2b, $rk11 @ AES block 8k+10 - round 11 + aese $ctr0b, $rk11 @ AES block 8k+8 - round 11 + + aese $ctr6b, $rk11 @ AES block 8k+14 - round 11 + aese $ctr4b, $rk11 @ AES block 8k+12 - round 11 + aese $ctr5b, $rk11 @ AES block 8k+13 - round 11 + +.L192_enc_tail: @ TAIL + + ldp $h5q, $h56kq, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + sub $main_end_input_ptr, $end_input_ptr, $input_ptr @ main_end_input_ptr is number of bytes left to process + + ldr $ctr_t0q, [$input_ptr], #16 @ AES block 8k+8 - l3ad plaintext + + ldp $h78kq, $h8q, [$current_tag, #192] @ load h8l | h8h + ext $h8.16b, $h8.16b, $h8.16b, #8 + + mov $t1.16b, $rk12 + + ldp $h6q, $h7q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + ext $h7.16b, $h7.16b, $h7.16b, #8 + cmp $main_end_input_ptr, #112 + + eor3 $res1b, $ctr_t0b, $ctr0b, $t1.16b @ AES block 8k+8 - result + ext $t0.16b, $acc_lb, $acc_lb, #8 @ prepare final partial tag + b.gt .L192_enc_blocks_more_than_7 + + cmp $main_end_input_ptr, #96 + mov $ctr7b, $ctr6b + movi $acc_h.8b, #0 + + mov $ctr6b, $ctr5b + movi $acc_l.8b, #0 + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + + mov $ctr5b, $ctr4b + mov $ctr4b, $ctr3b + mov $ctr3b, $ctr2b + + mov $ctr2b, $ctr1b + movi $acc_m.8b, #0 + b.gt .L192_enc_blocks_more_than_6 + + mov $ctr7b, $ctr6b + cmp $main_end_input_ptr, #80 + + mov $ctr6b, $ctr5b + mov $ctr5b, $ctr4b + mov $ctr4b, $ctr3b + + mov $ctr3b, $ctr1b + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + b.gt .L192_enc_blocks_more_than_5 + + cmp $main_end_input_ptr, #64 + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr5b + mov $ctr5b, $ctr4b + + mov $ctr4b, $ctr1b + b.gt .L192_enc_blocks_more_than_4 + + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr5b + mov $ctr5b, $ctr1b + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + cmp $main_end_input_ptr, #48 + b.gt .L192_enc_blocks_more_than_3 + + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr1b + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + cmp $main_end_input_ptr, #32 + b.gt .L192_enc_blocks_more_than_2 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + + cmp $main_end_input_ptr, #16 + mov $ctr7b, $ctr1b + b.gt .L192_enc_blocks_more_than_1 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + b .L192_enc_blocks_less_than_1 +.L192_enc_blocks_more_than_7: @ blocks left > 7 + st1 { $res1b}, [$output_ptr], #16 @ AES final-7 block - store result + + rev64 $res0b, $res1b @ GHASH final-7 block + ins $acc_m.d[0], $h78k.d[1] @ GHASH final-7 block - mid + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-7 block - mid + + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-6 block - load plaintext + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-7 block - mid + movi $t0.8b, #0 @ surpress further partial tag feed in + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH final-7 block - low + + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH final-7 block - high + + pmull $acc_m.1q, $rk4v.1d, $acc_m.1d @ GHASH final-7 block - mid + eor3 $res1b, $ctr_t1b, $ctr1b, $t1.16b @ AES final-6 block - result +.L192_enc_blocks_more_than_6: @ blocks left > 6 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-6 block - store result + + rev64 $res0b, $res1b @ GHASH final-6 block + + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-5 block - load plaintext + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-6 block - mid + + pmull $rk3q1, $res0.1d, $h7.1d @ GHASH final-6 block - low + eor3 $res1b, $ctr_t1b, $ctr2b, $t1.16b @ AES final-5 block - result + + movi $t0.8b, #0 @ surpress further partial tag feed in + pmull2 $rk2q1, $res0.2d, $h7.2d @ GHASH final-6 block - high + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-6 block - mid + + pmull $rk4v.1q, $rk4v.1d, $h78k.1d @ GHASH final-6 block - mid + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-6 block - high + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-6 block - low + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-6 block - mid +.L192_enc_blocks_more_than_5: @ blocks left > 5 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-5 block - store result + + rev64 $res0b, $res1b @ GHASH final-5 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-5 block - mid + + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-4 block - load plaintext + pmull2 $rk2q1, $res0.2d, $h6.2d @ GHASH final-5 block - high + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-5 block - mid + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-5 block - high + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-5 block - mid + pmull $rk3q1, $res0.1d, $h6.1d @ GHASH final-5 block - low + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-5 block - low + pmull2 $rk4v.1q, $rk4v.2d, $h56k.2d @ GHASH final-5 block - mid + + eor3 $res1b, $ctr_t1b, $ctr3b, $t1.16b @ AES final-4 block - result + movi $t0.8b, #0 @ surpress further partial tag feed in + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-5 block - mid +.L192_enc_blocks_more_than_4: @ blocks left > 4 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-4 block - store result + + rev64 $res0b, $res1b @ GHASH final-4 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-3 block - load plaintext + pmull2 $rk2q1, $res0.2d, $h5.2d @ GHASH final-4 block - high + ins $rk4v.d[0], $res0.d[1] @ GHASH final-4 block - mid + + pmull $rk3q1, $res0.1d, $h5.1d @ GHASH final-4 block - low + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-4 block - high + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-4 block - mid + + movi $t0.8b, #0 @ surpress further partial tag feed in + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-4 block - low + + pmull $rk4v.1q, $rk4v.1d, $h56k.1d @ GHASH final-4 block - mid + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-4 block - mid + eor3 $res1b, $ctr_t1b, $ctr4b, $t1.16b @ AES final-3 block - result +.L192_enc_blocks_more_than_3: @ blocks left > 3 + + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + st1 { $res1b}, [$output_ptr], #16 @ AES final-3 block - store result + + rev64 $res0b, $res1b @ GHASH final-3 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + movi $t0.8b, #0 @ surpress further partial tag feed in + + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-2 block - load plaintext + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-3 block - mid + + eor3 $res1b, $ctr_t1b, $ctr5b, $t1.16b @ AES final-2 block - result + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-3 block - mid + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-3 block - mid + pmull $rk3q1, $res0.1d, $h4.1d @ GHASH final-3 block - low + + pmull2 $rk2q1, $res0.2d, $h4.2d @ GHASH final-3 block - high + pmull2 $rk4v.1q, $rk4v.2d, $h34k.2d @ GHASH final-3 block - mid + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-3 block - low + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-3 block - mid + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-3 block - high +.L192_enc_blocks_more_than_2: @ blocks left > 2 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-2 block - store result + + rev64 $res0b, $res1b @ GHASH final-2 block + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-1 block - load plaintext + ins $rk4v.d[0], $res0.d[1] @ GHASH final-2 block - mid + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-2 block - mid + + pmull $rk3q1, $res0.1d, $h3.1d @ GHASH final-2 block - low + pmull2 $rk2q1, $res0.2d, $h3.2d @ GHASH final-2 block - high + movi $t0.8b, #0 @ surpress further partial tag feed in + + pmull $rk4v.1q, $rk4v.1d, $h34k.1d @ GHASH final-2 block - mid + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-2 block - low + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-2 block - high + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-2 block - mid + eor3 $res1b, $ctr_t1b, $ctr6b, $t1.16b @ AES final-1 block - result +.L192_enc_blocks_more_than_1: @ blocks left > 1 + + ldr $h2q, [$current_tag, #64] @ load h1l | h1h + ext $h2.16b, $h2.16b, $h2.16b, #8 + st1 { $res1b}, [$output_ptr], #16 @ AES final-1 block - store result + + rev64 $res0b, $res1b @ GHASH final-1 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-1 block - mid + pmull $rk3q1, $res0.1d, $h2.1d @ GHASH final-1 block - low + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-1 block - low + pmull2 $rk2q1, $res0.2d, $h2.2d @ GHASH final-1 block - high + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-1 block - mid + + ldr $ctr_t1q, [$input_ptr], #16 @ AES final block - load plaintext + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-1 block - mid + + eor3 $res1b, $ctr_t1b, $ctr7b, $t1.16b @ AES final block - result + pmull2 $rk4v.1q, $rk4v.2d, $h12k.2d @ GHASH final-1 block - mid + + movi $t0.8b, #0 @ surpress further partial tag feed in + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-1 block - mid + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-1 block - high +.L192_enc_blocks_less_than_1: @ blocks left <= 1 + + mvn $temp0_x, xzr @ temp0_x = 0xffffffffffffffff + and $bit_length, $bit_length, #127 @ bit_length %= 128 + + sub $bit_length, $bit_length, #128 @ bit_length -= 128 + + neg $bit_length, $bit_length @ bit_length = 128 - #bits in input (in range [1,128]) + + and $bit_length, $bit_length, #127 @ bit_length %= 128 + + lsr $temp0_x, $temp0_x, $bit_length @ temp0_x is mask for top 64b of last block + cmp $bit_length, #64 + mvn $temp1_x, xzr @ temp1_x = 0xffffffffffffffff + + csel $temp2_x, $temp1_x, $temp0_x, lt + csel $temp3_x, $temp0_x, xzr, lt + + mov $ctr0.d[1], $temp3_x + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + + ld1 { $rk0}, [$output_ptr] @ load existing bytes where the possibly partial last block is to be stored + mov $ctr0.d[0], $temp2_x @ ctr0b is mask for last block + + and $res1b, $res1b, $ctr0b @ possibly partial last block has zeroes in highest bits + + rev64 $res0b, $res1b @ GHASH final block + bif $res1b, $rk0, $ctr0b @ insert existing bytes in top end of result before storing + + st1 { $res1b}, [$output_ptr] @ store all 16B + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $t0.d[0], $res0.d[1] @ GHASH final block - mid + pmull2 $rk2q1, $res0.2d, $h1.2d @ GHASH final block - high + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final block - high + pmull $rk3q1, $res0.1d, $h1.1d @ GHASH final block - low + + eor $t0.8b, $t0.8b, $res0.8b @ GHASH final block - mid + + pmull $t0.1q, $t0.1d, $h12k.1d @ GHASH final block - mid + + eor $acc_mb, $acc_mb, $t0.16b @ GHASH final block - mid + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final block - low + ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + + rev32 $rtmp_ctr.16b, $rtmp_ctr.16b + + str $rtmp_ctrq, [$counter] @ store the updated counter + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + + pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + + eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low + ext $acc_lb, $acc_lb, $acc_lb, #8 + rev64 $acc_lb, $acc_lb + st1 { $acc_l.16b }, [$current_tag] + + lsr x0, $bit_length, #3 @ return sizes + + ldp d10, d11, [sp, #16] + ldp d12, d13, [sp, #32] + ldp d14, d15, [sp, #48] + ldp d8, d9, [sp], #80 + ret + +.L192_enc_ret: + mov w0, #0x0 + ret +.size unroll8_eor3_aes_gcm_enc_192_kernel,.-unroll8_eor3_aes_gcm_enc_192_kernel +___ + +######################################################################################### +# size_t unroll8_eor3_aes_gcm_dec_192_kernel(const unsigned char *in, +# size_t len, +# unsigned char *out, +# const void *key, +# unsigned char ivec[16], +# u64 *Xi); +# +$code.=<<___; +.global unroll8_eor3_aes_gcm_dec_192_kernel +.type unroll8_eor3_aes_gcm_dec_192_kernel,%function +.align 4 +unroll8_eor3_aes_gcm_dec_192_kernel: + AARCH64_VALID_CALL_TARGET + cbz x1, .L192_dec_ret + stp d8, d9, [sp, #-80]! + mov $counter, x4 + mov $cc, x5 + stp d10, d11, [sp, #16] + stp d12, d13, [sp, #32] + stp d14, d15, [sp, #48] + mov x5, #0xc200000000000000 + stp x5, xzr, [sp, #64] + add $modulo_constant, sp, #64 + + lsr $main_end_input_ptr, $bit_length, #3 @ byte_len + ld1 { $ctr0b}, [$counter] @ CTR block 0 + ld1 { $acc_lb}, [$current_tag] + + mov $constant_temp, #0x100000000 @ set up counter increment + movi $rctr_inc.16b, #0x0 + mov $rctr_inc.d[1], $constant_temp + + rev32 $rtmp_ctr.16b, $ctr0.16b @ set up reversed counter + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 0 + + rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 1 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 1 + + rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 2 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 2 + + rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 3 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 3 + + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 4 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 4 + + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 5 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 5 + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 6 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 6 + + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 7 + + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 0 + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 0 + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 0 + + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 0 + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 0 + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 0 + + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 0 + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 0 + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 1 + + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 1 + + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 1 + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 1 + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 1 + + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 1 + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 1 + + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 2 + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 2 + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 1 + + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 2 + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 2 + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 2 + + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 2 + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 2 + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 2 + + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 3 + + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 3 + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 3 + + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 3 + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 3 + + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 3 + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 3 + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 3 + + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 4 + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 4 + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 4 + + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 4 + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 4 + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 4 + + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 4 + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 5 + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 4 + + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 5 + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 5 + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 5 + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 5 + + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 5 + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 5 + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 5 + + sub $main_end_input_ptr, $main_end_input_ptr, #1 @ byte_len - 1 + + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 6 + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 6 + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 6 + + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 6 + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 6 + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 6 + + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 6 + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 6 + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 7 + + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 7 + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 7 + + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 7 + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 7 + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 7 + + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 7 + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 7 + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 7 + + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 8 + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 8 + and $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffff80 @ number of bytes to be processed in main loop (at least 1 byte must be handled by tail) + + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 8 + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 8 + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 8 + + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 8 + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 8 + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 8 + + add $end_input_ptr, $input_ptr, $bit_length, lsr #3 @ end_input_ptr + aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 9 + + ld1 { $acc_lb}, [$current_tag] + ext $acc_lb, $acc_lb, $acc_lb, #8 + rev64 $acc_lb, $acc_lb + + ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11 + + aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 9 + add $main_end_input_ptr, $main_end_input_ptr, $input_ptr + + aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 9 + aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 9 + aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 9 + + cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks + aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 9 + + aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 9 + aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 9 + + aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 10 + aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 10 + aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 10 + + aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 10 + aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 10 + aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 10 + + aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 10 + aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 10 + ldr $rk12q, [$cc, #192] @ load rk12 + + aese $ctr0b, $rk11 @ AES block 0 - round 11 + aese $ctr1b, $rk11 @ AES block 1 - round 11 + aese $ctr4b, $rk11 @ AES block 4 - round 11 + + aese $ctr6b, $rk11 @ AES block 6 - round 11 + aese $ctr5b, $rk11 @ AES block 5 - round 11 + aese $ctr7b, $rk11 @ AES block 7 - round 11 + + aese $ctr2b, $rk11 @ AES block 2 - round 11 + aese $ctr3b, $rk11 @ AES block 3 - round 11 + b.ge .L192_dec_tail @ handle tail + + ldp $res0q, $res1q, [$input_ptr], #32 @ AES block 0, 1 - load ciphertext + + ldp $res2q, $res3q, [$input_ptr], #32 @ AES block 2, 3 - load ciphertext + + ldp $res4q, $res5q, [$input_ptr], #32 @ AES block 4, 5 - load ciphertext + + eor3 $ctr1b, $res1b, $ctr1b, $rk12 @ AES block 1 - result + eor3 $ctr0b, $res0b, $ctr0b, $rk12 @ AES block 0 - result + stp $ctr0q, $ctr1q, [$output_ptr], #32 @ AES block 0, 1 - store result + + rev32 $ctr0.16b, $rtmp_ctr.16b @ CTR block 8 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8 + + rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 9 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 9 + eor3 $ctr3b, $res3b, $ctr3b, $rk12 @ AES block 3 - result + + eor3 $ctr2b, $res2b, $ctr2b, $rk12 @ AES block 2 - result + stp $ctr2q, $ctr3q, [$output_ptr], #32 @ AES block 2, 3 - store result + ldp $res6q, $res7q, [$input_ptr], #32 @ AES block 6, 7 - load ciphertext + + rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 10 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 10 + + eor3 $ctr4b, $res4b, $ctr4b, $rk12 @ AES block 4 - result + + rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 11 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 11 + + eor3 $ctr5b, $res5b, $ctr5b, $rk12 @ AES block 5 - result + stp $ctr4q, $ctr5q, [$output_ptr], #32 @ AES block 4, 5 - store result + cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks + + eor3 $ctr6b, $res6b, $ctr6b, $rk12 @ AES block 6 - result + eor3 $ctr7b, $res7b, $ctr7b, $rk12 @ AES block 7 - result + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 12 + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 12 + stp $ctr6q, $ctr7q, [$output_ptr], #32 @ AES block 6, 7 - store result + b.ge .L192_dec_prepretail @ do prepretail + +.L192_dec_main_loop: @ main loop start + rev64 $res1b, $res1b @ GHASH block 8k+1 + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0 + + rev64 $res0b, $res0b @ GHASH block 8k + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13 + + ldr $h7q, [$current_tag, #176] @ load h7l | h7h + ext $h7.16b, $h7.16b, $h7.16b, #8 + ldr $h8q, [$current_tag, #208] @ load h8l | h8h + ext $h8.16b, $h8.16b, $h8.16b, #8 + rev64 $res4b, $res4b @ GHASH block 8k+4 + rev64 $res3b, $res3b @ GHASH block 8k+3 + + eor $res0b, $res0b, $acc_lb @ PRE 1 + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14 + + rev64 $res5b, $res5b @ GHASH block 8k+5 + + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15 + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0 + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0 + + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0 + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0 + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0 + + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0 + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0 + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0 + + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low + pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1 + pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low + ldr $h5q, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + ldr $h6q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1 + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1 + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1 + + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1 + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1 + + trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + rev64 $res2b, $res2b @ GHASH block 8k+2 + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1 + + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1 + ldr $h56kq, [$current_tag, #144] @ load h6k | h5k + ldr $h78kq, [$current_tag, #192] @ load h8k | h7k + trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + + eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high + pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high + pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high + + eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid + eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2 + + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2 + pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low + eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high + + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2 + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3 + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2 + + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2 + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2 + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2 + + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2 + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3 + + pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low + trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3 + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3 + + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3 + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3 + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + + eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3 + + trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15 + + pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid + pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid + pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid + + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3 + pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid + pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high + + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4 + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4 + eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid + + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4 + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4 + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4 + + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4 + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4 + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4 + + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + ldr $h2q, [$current_tag, #64] @ load h1l | h1h + ext $h2.16b, $h2.16b, $h2.16b, #8 + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5 + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5 + + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5 + rev64 $res7b, $res7b @ GHASH block 8k+7 + + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5 + eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5 + + pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low + trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5 + + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5 + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5 + rev64 $res6b, $res6b @ GHASH block 8k+6 + + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high + pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low + + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6 + eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6 + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6 + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6 + + pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6 + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6 + + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7 + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7 + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6 + + pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid + eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high + eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low + + pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low + trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6 + + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7 + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7 + + eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7 + + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7 + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7 + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7 + + eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid + pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high + + pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low + + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8 + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8 + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8 + + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8 + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8 + eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low + + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8 + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8 + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8 + + eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high + rev32 $h1.16b, $rtmp_ctr.16b @ CTR block 8k+16 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+16 + + aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 9 + eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 9 + + aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 9 + aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 9 + ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11 + + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + ldp $res0q, $res1q, [$input_ptr], #32 @ AES block 8k+8, 8k+9 - load ciphertext + + aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 9 + aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 9 + ldp $res2q, $res3q, [$input_ptr], #32 @ AES block 8k+10, 8k+11 - load ciphertext + + rev32 $h2.16b, $rtmp_ctr.16b @ CTR block 8k+17 + pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+17 + + aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 9 + aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 9 + ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + + aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 10 + aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 10 + ldp $res4q, $res5q, [$input_ptr], #32 @ AES block 8k+12, 8k+13 - load ciphertext + + rev32 $h3.16b, $rtmp_ctr.16b @ CTR block 8k+18 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+18 + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + + aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 10 + aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 10 + ldr $rk12q, [$cc, #192] @ load rk12 + + ldp $res6q, $res7q, [$input_ptr], #32 @ AES block 8k+14, 8k+15 - load ciphertext + aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 10 + aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 10 + + aese $ctr0b, $rk11 @ AES block 8k+8 - round 11 + ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + aese $ctr1b, $rk11 @ AES block 8k+9 - round 11 + + aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 10 + aese $ctr6b, $rk11 @ AES block 8k+14 - round 11 + aese $ctr3b, $rk11 @ AES block 8k+11 - round 11 + + eor3 $ctr0b, $res0b, $ctr0b, $rk12 @ AES block 8k+8 - result + rev32 $h4.16b, $rtmp_ctr.16b @ CTR block 8k+19 + aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 10 + + aese $ctr4b, $rk11 @ AES block 8k+12 - round 11 + aese $ctr2b, $rk11 @ AES block 8k+10 - round 11 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+19 + + aese $ctr7b, $rk11 @ AES block 8k+15 - round 11 + aese $ctr5b, $rk11 @ AES block 8k+13 - round 11 + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + + eor3 $ctr1b, $res1b, $ctr1b, $rk12 @ AES block 8k+9 - result + stp $ctr0q, $ctr1q, [$output_ptr], #32 @ AES block 8k+8, 8k+9 - store result + eor3 $ctr3b, $res3b, $ctr3b, $rk12 @ AES block 8k+11 - result + + eor3 $ctr2b, $res2b, $ctr2b, $rk12 @ AES block 8k+10 - result + eor3 $ctr7b, $res7b, $ctr7b, $rk12 @ AES block 8k+15 - result + stp $ctr2q, $ctr3q, [$output_ptr], #32 @ AES block 8k+10, 8k+11 - store result + + eor3 $ctr5b, $res5b, $ctr5b, $rk12 @ AES block 8k+13 - result + eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low + mov $ctr3.16b, $h4.16b @ CTR block 8k+19 + + eor3 $ctr4b, $res4b, $ctr4b, $rk12 @ AES block 8k+12 - result + stp $ctr4q, $ctr5q, [$output_ptr], #32 @ AES block 8k+12, 8k+13 - store result + cmp $input_ptr, $main_end_input_ptr @ LOOP CONTROL + + eor3 $ctr6b, $res6b, $ctr6b, $rk12 @ AES block 8k+14 - result + stp $ctr6q, $ctr7q, [$output_ptr], #32 @ AES block 8k+14, 8k+15 - store result + mov $ctr0.16b, $h1.16b @ CTR block 8k+16 + + mov $ctr1.16b, $h2.16b @ CTR block 8k+17 + mov $ctr2.16b, $h3.16b @ CTR block 8k+18 + + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 8k+20 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+20 + b.lt .L192_dec_main_loop + +.L192_dec_prepretail: @ PREPRETAIL + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13 + + ldr $h7q, [$current_tag, #176] @ load h7l | h7h + ext $h7.16b, $h7.16b, $h7.16b, #8 + ldr $h8q, [$current_tag, #208] @ load h8l | h8h + ext $h8.16b, $h8.16b, $h8.16b, #8 + rev64 $res0b, $res0b @ GHASH block 8k + ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0 + + rev64 $res3b, $res3b @ GHASH block 8k+3 + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14 + + eor $res0b, $res0b, $acc_lb @ PRE 1 + rev64 $res2b, $res2b @ GHASH block 8k+2 + rev64 $res1b, $res1b @ GHASH block 8k+1 + + ldr $h5q, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + ldr $h6q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15 + + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0 + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0 + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0 + + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0 + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0 + pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high + + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0 + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0 + + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1 + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0 + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1 + pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high + pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low + + pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low + eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1 + + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1 + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1 + + trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high + + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1 + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1 + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1 + + ldr $h56kq, [$current_tag, #144] @ load h6k | h5k + ldr $h78kq, [$current_tag, #192] @ load h8k | h7k + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2 + eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid + + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2 + rev64 $res5b, $res5b @ GHASH block 8k+5 + pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low + + eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2 + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2 + + trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3 + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2 + + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2 + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2 + trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + + pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2 + pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid + + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3 + eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low + + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3 + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3 + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3 + + eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3 + + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid + pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid + + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + ldr $h2q, [$current_tag, #64] @ load h1l | h1h + ext $h2.16b, $h2.16b, $h2.16b, #8 + eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3 + + rev64 $res7b, $res7b @ GHASH block 8k+7 + + eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + rev64 $res4b, $res4b @ GHASH block 8k+4 + + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4 + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4 + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3 + + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4 + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4 + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4 + + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4 + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4 + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4 + + rev64 $res6b, $res6b @ GHASH block 8k+6 + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5 + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5 + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5 + + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5 + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5 + + pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high + pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high + pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low + + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5 + + pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low + trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high + + pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low + trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5 + + trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5 + eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6 + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6 + + eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6 + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6 + + pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid + pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6 + + pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6 + pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high + + eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7 + eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low + + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6 + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6 + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7 + + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7 + + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high + pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low + + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7 + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7 + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7 + + eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high + eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low + eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7 + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7 + + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8 + + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8 + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8 + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8 + + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8 + pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8 + + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8 + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8 + ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11 + + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 9 + aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 9 + + aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 9 + aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 9 + aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 9 + + aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 9 + aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 9 + aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 9 + + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + ldr $rk12q, [$cc, #192] @ load rk12 + ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + + aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 10 + aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 10 + aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 10 + + aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 10 + aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 10 + aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 10 + + aese $ctr0b, $rk11 @ AES block 8k+8 - round 11 + eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low + aese $ctr5b, $rk11 @ AES block 8k+13 - round 11 + + aese $ctr2b, $rk11 @ AES block 8k+10 - round 11 + aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 10 + aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 10 + + aese $ctr6b, $rk11 @ AES block 8k+14 - round 11 + aese $ctr4b, $rk11 @ AES block 8k+12 - round 11 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15 + + aese $ctr3b, $rk11 @ AES block 8k+11 - round 11 + aese $ctr1b, $rk11 @ AES block 8k+9 - round 11 + aese $ctr7b, $rk11 @ AES block 8k+15 - round 11 + +.L192_dec_tail: @ TAIL + + sub $main_end_input_ptr, $end_input_ptr, $input_ptr @ main_end_input_ptr is number of bytes left to process + + ldp $h5q, $h56kq, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + ldr $res1q, [$input_ptr], #16 @ AES block 8k+8 - load ciphertext + + ldp $h78kq, $h8q, [$current_tag, #192] @ load h8l | h8h + ext $h8.16b, $h8.16b, $h8.16b, #8 + + mov $t1.16b, $rk12 + + ldp $h6q, $h7q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + ext $h7.16b, $h7.16b, $h7.16b, #8 + ext $t0.16b, $acc_lb, $acc_lb, #8 @ prepare final partial tag + + eor3 $res4b, $res1b, $ctr0b, $t1.16b @ AES block 8k+8 - result + cmp $main_end_input_ptr, #112 + b.gt .L192_dec_blocks_more_than_7 + + mov $ctr7b, $ctr6b + movi $acc_h.8b, #0 + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + + mov $ctr6b, $ctr5b + mov $ctr5b, $ctr4b + mov $ctr4b, $ctr3b + + cmp $main_end_input_ptr, #96 + movi $acc_l.8b, #0 + mov $ctr3b, $ctr2b + + mov $ctr2b, $ctr1b + movi $acc_m.8b, #0 + b.gt .L192_dec_blocks_more_than_6 + + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr5b + mov $ctr5b, $ctr4b + + mov $ctr4b, $ctr3b + mov $ctr3b, $ctr1b + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + cmp $main_end_input_ptr, #80 + b.gt .L192_dec_blocks_more_than_5 + + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr5b + + mov $ctr5b, $ctr4b + mov $ctr4b, $ctr1b + cmp $main_end_input_ptr, #64 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + b.gt .L192_dec_blocks_more_than_4 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr5b + + mov $ctr5b, $ctr1b + cmp $main_end_input_ptr, #48 + b.gt .L192_dec_blocks_more_than_3 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + mov $ctr7b, $ctr6b + cmp $main_end_input_ptr, #32 + + mov $ctr6b, $ctr1b + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + b.gt .L192_dec_blocks_more_than_2 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + + mov $ctr7b, $ctr1b + cmp $main_end_input_ptr, #16 + b.gt .L192_dec_blocks_more_than_1 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + b .L192_dec_blocks_less_than_1 +.L192_dec_blocks_more_than_7: @ blocks left > 7 + rev64 $res0b, $res1b @ GHASH final-7 block + + ins $acc_m.d[0], $h78k.d[1] @ GHASH final-7 block - mid + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH final-7 block - high + ins $rk4v.d[0], $res0.d[1] @ GHASH final-7 block - mid + ldr $res1q, [$input_ptr], #16 @ AES final-6 block - load ciphertext + + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH final-7 block - low + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-7 block - mid + st1 { $res4b}, [$output_ptr], #16 @ AES final-7 block - store result + + eor3 $res4b, $res1b, $ctr1b, $t1.16b @ AES final-6 block - result + + pmull $acc_m.1q, $rk4v.1d, $acc_m.1d @ GHASH final-7 block - mid + movi $t0.8b, #0 @ surpress further partial tag feed in +.L192_dec_blocks_more_than_6: @ blocks left > 6 + + rev64 $res0b, $res1b @ GHASH final-6 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ldr $res1q, [$input_ptr], #16 @ AES final-5 block - load ciphertext + ins $rk4v.d[0], $res0.d[1] @ GHASH final-6 block - mid + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-6 block - mid + movi $t0.8b, #0 @ surpress further partial tag feed in + pmull2 $rk2q1, $res0.2d, $h7.2d @ GHASH final-6 block - high + + st1 { $res4b}, [$output_ptr], #16 @ AES final-6 block - store result + eor3 $res4b, $res1b, $ctr2b, $t1.16b @ AES final-5 block - result + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-6 block - high + pmull $rk4v.1q, $rk4v.1d, $h78k.1d @ GHASH final-6 block - mid + pmull $rk3q1, $res0.1d, $h7.1d @ GHASH final-6 block - low + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-6 block - mid + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-6 block - low +.L192_dec_blocks_more_than_5: @ blocks left > 5 + + rev64 $res0b, $res1b @ GHASH final-5 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-5 block - mid + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-5 block - mid + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-5 block - mid + pmull2 $rk2q1, $res0.2d, $h6.2d @ GHASH final-5 block - high + + ldr $res1q, [$input_ptr], #16 @ AES final-4 block - load ciphertext + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-5 block - high + pmull $rk3q1, $res0.1d, $h6.1d @ GHASH final-5 block - low + + pmull2 $rk4v.1q, $rk4v.2d, $h56k.2d @ GHASH final-5 block - mid + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-5 block - low + movi $t0.8b, #0 @ surpress further partial tag feed in + st1 { $res4b}, [$output_ptr], #16 @ AES final-5 block - store result + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-5 block - mid + eor3 $res4b, $res1b, $ctr3b, $t1.16b @ AES final-4 block - result +.L192_dec_blocks_more_than_4: @ blocks left > 4 + + rev64 $res0b, $res1b @ GHASH final-4 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + movi $t0.8b, #0 @ surpress further partial tag feed in + + ldr $res1q, [$input_ptr], #16 @ AES final-3 block - load ciphertext + ins $rk4v.d[0], $res0.d[1] @ GHASH final-4 block - mid + pmull $rk3q1, $res0.1d, $h5.1d @ GHASH final-4 block - low + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-4 block - mid + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-4 block - low + + pmull $rk4v.1q, $rk4v.1d, $h56k.1d @ GHASH final-4 block - mid + st1 { $res4b}, [$output_ptr], #16 @ AES final-4 block - store result + pmull2 $rk2q1, $res0.2d, $h5.2d @ GHASH final-4 block - high + + eor3 $res4b, $res1b, $ctr4b, $t1.16b @ AES final-3 block - result + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-4 block - mid + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-4 block - high +.L192_dec_blocks_more_than_3: @ blocks left > 3 + + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + rev64 $res0b, $res1b @ GHASH final-3 block + ldr $res1q, [$input_ptr], #16 @ AES final-2 block - load ciphertext + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-3 block - mid + pmull2 $rk2q1, $res0.2d, $h4.2d @ GHASH final-3 block - high + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-3 block - high + movi $t0.8b, #0 @ surpress further partial tag feed in + pmull $rk3q1, $res0.1d, $h4.1d @ GHASH final-3 block - low + + st1 { $res4b}, [$output_ptr], #16 @ AES final-3 block - store result + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-3 block - mid + eor3 $res4b, $res1b, $ctr5b, $t1.16b @ AES final-2 block - result + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-3 block - low + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-3 block - mid + + pmull2 $rk4v.1q, $rk4v.2d, $h34k.2d @ GHASH final-3 block - mid + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-3 block - mid +.L192_dec_blocks_more_than_2: @ blocks left > 2 + + rev64 $res0b, $res1b @ GHASH final-2 block + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-2 block - mid + ldr $res1q, [$input_ptr], #16 @ AES final-1 block - load ciphertext + + pmull2 $rk2q1, $res0.2d, $h3.2d @ GHASH final-2 block - high + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-2 block - mid + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-2 block - high + pmull $rk3q1, $res0.1d, $h3.1d @ GHASH final-2 block - low + + pmull $rk4v.1q, $rk4v.1d, $h34k.1d @ GHASH final-2 block - mid + movi $t0.8b, #0 @ surpress further partial tag feed in + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-2 block - low + st1 { $res4b}, [$output_ptr], #16 @ AES final-2 block - store result + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-2 block - mid + eor3 $res4b, $res1b, $ctr6b, $t1.16b @ AES final-1 block - result +.L192_dec_blocks_more_than_1: @ blocks left > 1 + + rev64 $res0b, $res1b @ GHASH final-1 block + ldr $res1q, [$input_ptr], #16 @ AES final block - load ciphertext + ldr $h2q, [$current_tag, #64] @ load h1l | h1h + ext $h2.16b, $h2.16b, $h2.16b, #8 + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + movi $t0.8b, #0 @ surpress further partial tag feed in + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + + pmull $rk3q1, $res0.1d, $h2.1d @ GHASH final-1 block - low + ins $rk4v.d[0], $res0.d[1] @ GHASH final-1 block - mid + st1 { $res4b}, [$output_ptr], #16 @ AES final-1 block - store result + + pmull2 $rk2q1, $res0.2d, $h2.2d @ GHASH final-1 block - high + + eor3 $res4b, $res1b, $ctr7b, $t1.16b @ AES final block - result + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-1 block - mid + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-1 block - mid + + pmull2 $rk4v.1q, $rk4v.2d, $h12k.2d @ GHASH final-1 block - mid + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-1 block - low + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-1 block - mid + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-1 block - high +.L192_dec_blocks_less_than_1: @ blocks left <= 1 + + rev32 $rtmp_ctr.16b, $rtmp_ctr.16b + and $bit_length, $bit_length, #127 @ bit_length %= 128 + + sub $bit_length, $bit_length, #128 @ bit_length -= 128 + str $rtmp_ctrq, [$counter] @ store the updated counter + + neg $bit_length, $bit_length @ bit_length = 128 - #bits in input (in range [1,128]) + mvn $temp0_x, xzr @ temp0_x = 0xffffffffffffffff + + and $bit_length, $bit_length, #127 @ bit_length %= 128 + + mvn $temp1_x, xzr @ temp1_x = 0xffffffffffffffff + lsr $temp0_x, $temp0_x, $bit_length @ temp0_x is mask for top 64b of last block + cmp $bit_length, #64 + + csel $temp2_x, $temp1_x, $temp0_x, lt + csel $temp3_x, $temp0_x, xzr, lt + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + + mov $ctr0.d[1], $temp3_x + ld1 { $rk0}, [$output_ptr] @ load existing bytes where the possibly partial last block is to be stored + + mov $ctr0.d[0], $temp2_x @ ctr0b is mask for last block + + and $res1b, $res1b, $ctr0b @ possibly partial last block has zeroes in highest bits + bif $res4b, $rk0, $ctr0b @ insert existing bytes in top end of result before storing + + rev64 $res0b, $res1b @ GHASH final block + + st1 { $res4b}, [$output_ptr] @ store all 16B + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $t0.d[0], $res0.d[1] @ GHASH final block - mid + pmull $rk3q1, $res0.1d, $h1.1d @ GHASH final block - low + + eor $t0.8b, $t0.8b, $res0.8b @ GHASH final block - mid + pmull2 $rk2q1, $res0.2d, $h1.2d @ GHASH final block - high + eor $acc_lb, $acc_lb, $rk3 @ GHASH final block - low + + pmull $t0.1q, $t0.1d, $h12k.1d @ GHASH final block - mid + eor $acc_hb, $acc_hb, $rk2 @ GHASH final block - high + + eor $t10.16b, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + eor $acc_mb, $acc_mb, $t0.16b @ GHASH final block - mid + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + + pmull $t11.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + ext $acc_hb, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + + eor $acc_mb, $acc_mb, $t10.16b @ MODULO - karatsuba tidy up + + eor3 $acc_mb, $acc_mb, $acc_hb, $t11.16b @ MODULO - fold into mid + + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + ext $acc_mb, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + + eor3 $acc_lb, $acc_lb, $acc_mb, $acc_hb @ MODULO - fold into low + ext $acc_lb, $acc_lb, $acc_lb, #8 + rev64 $acc_lb, $acc_lb + st1 { $acc_l.16b }, [$current_tag] + + ldp d10, d11, [sp, #16] + ldp d12, d13, [sp, #32] + ldp d14, d15, [sp, #48] + ldp d8, d9, [sp], #80 + ret + +.L192_dec_ret: + mov w0, #0x0 + ret +.size unroll8_eor3_aes_gcm_dec_192_kernel,.-unroll8_eor3_aes_gcm_dec_192_kernel +___ +} + +{ + +my ($end_input_ptr,$main_end_input_ptr,$temp0_x,$temp1_x)=map("x$_",(4..7)); +my ($temp2_x,$temp3_x)=map("x$_",(13..14)); +my ($ctr0b,$ctr1b,$ctr2b,$ctr3b,$ctr4b,$ctr5b,$ctr6b,$ctr7b,$res0b,$res1b,$res2b,$res3b,$res4b,$res5b,$res6b,$res7b)=map("v$_.16b",(0..15)); +my ($ctr0,$ctr1,$ctr2,$ctr3,$ctr4,$ctr5,$ctr6,$ctr7,$res0,$res1,$res2,$res3,$res4,$res5,$res6,$res7)=map("v$_",(0..15)); +my ($ctr0d,$ctr1d,$ctr2d,$ctr3d,$ctr4d,$ctr5d,$ctr6d,$ctr7d)=map("d$_",(0..7)); +my ($ctr0q,$ctr1q,$ctr2q,$ctr3q,$ctr4q,$ctr5q,$ctr6q,$ctr7q)=map("q$_",(0..7)); +my ($res0q,$res1q,$res2q,$res3q,$res4q,$res5q,$res6q,$res7q)=map("q$_",(8..15)); + +my ($ctr_t0,$ctr_t1,$ctr_t2,$ctr_t3,$ctr_t4,$ctr_t5,$ctr_t6,$ctr_t7)=map("v$_",(8..15)); +my ($ctr_t0b,$ctr_t1b,$ctr_t2b,$ctr_t3b,$ctr_t4b,$ctr_t5b,$ctr_t6b,$ctr_t7b)=map("v$_.16b",(8..15)); +my ($ctr_t0q,$ctr_t1q,$ctr_t2q,$ctr_t3q,$ctr_t4q,$ctr_t5q,$ctr_t6q,$ctr_t7q)=map("q$_",(8..15)); + +my ($acc_hb,$acc_mb,$acc_lb)=map("v$_.16b",(17..19)); +my ($acc_h,$acc_m,$acc_l)=map("v$_",(17..19)); + +my ($h1,$h12k,$h2,$h3,$h34k,$h4)=map("v$_",(20..25)); +my ($h5,$h56k,$h6,$h7,$h78k,$h8)=map("v$_",(20..25)); +my ($h1q,$h12kq,$h2q,$h3q,$h34kq,$h4q)=map("q$_",(20..25)); +my ($h5q,$h56kq,$h6q,$h7q,$h78kq,$h8q)=map("q$_",(20..25)); + +my $t0="v16"; +my $t0d="d16"; + +my $t1="v29"; +my $t2=$res1; +my $t3=$t1; + +my $t4=$res0; +my $t5=$res2; +my $t6=$t0; + +my $t7=$res3; +my $t8=$res4; +my $t9=$res5; + +my $t10=$res6; +my $t11="v21"; +my $t12=$t1; + +my $rtmp_ctr="v30"; +my $rtmp_ctrq="q30"; +my $rctr_inc="v31"; +my $rctr_incd="d31"; + +my $mod_constantd=$t0d; +my $mod_constant=$t0; + +my ($rk0,$rk1,$rk2)=map("v$_.16b",(26..28)); +my ($rk3,$rk4,$rk5)=map("v$_.16b",(26..28)); +my ($rk6,$rk7,$rk8)=map("v$_.16b",(26..28)); +my ($rk9,$rk10,$rk11)=map("v$_.16b",(26..28)); +my ($rk12,$rk13,$rk14)=map("v$_.16b",(26..28)); +my ($rk0q,$rk1q,$rk2q)=map("q$_",(26..28)); +my ($rk3q,$rk4q,$rk5q)=map("q$_",(26..28)); +my ($rk6q,$rk7q,$rk8q)=map("q$_",(26..28)); +my ($rk9q,$rk10q,$rk11q)=map("q$_",(26..28)); +my ($rk12q,$rk13q,$rk14q)=map("q$_",(26..28)); +my $rk2q1="v28.1q"; +my $rk3q1="v26.1q"; +my $rk4v="v27"; +######################################################################################### +# size_t unroll8_eor3_aes_gcm_enc_256_kernel(const unsigned char *in, +# size_t len, +# unsigned char *out, +# const void *key, +# unsigned char ivec[16], +# u64 *Xi); +# +$code.=<<___; +.global unroll8_eor3_aes_gcm_enc_256_kernel +.type unroll8_eor3_aes_gcm_enc_256_kernel,%function +.align 4 +unroll8_eor3_aes_gcm_enc_256_kernel: + AARCH64_VALID_CALL_TARGET + cbz x1, .L256_enc_ret + stp d8, d9, [sp, #-80]! + mov $counter, x4 + mov $cc, x5 + stp d10, d11, [sp, #16] + stp d12, d13, [sp, #32] + stp d14, d15, [sp, #48] + mov x5, #0xc200000000000000 + stp x5, xzr, [sp, #64] + add $modulo_constant, sp, #64 + + ld1 { $ctr0b}, [$counter] @ CTR block 0 + + lsr $main_end_input_ptr, $bit_length, #3 @ byte_len + + mov $constant_temp, #0x100000000 @ set up counter increment + movi $rctr_inc.16b, #0x0 + mov $rctr_inc.d[1], $constant_temp + sub $main_end_input_ptr, $main_end_input_ptr, #1 @ byte_len - 1 + + and $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffff80 @ number of bytes to be processed in main loop (at least 1 byte must be handled by tail) + + add $main_end_input_ptr, $main_end_input_ptr, $input_ptr + + rev32 $rtmp_ctr.16b, $ctr0.16b @ set up reversed counter + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 0 + + rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 1 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 1 + + rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 2 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 2 + + rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 3 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 3 + + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 4 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 4 + + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 5 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 5 + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 6 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 6 + + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 7 + + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 0 + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 0 + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 0 + + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 0 + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 0 + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 0 + + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 0 + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 0 + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 1 + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 1 + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 1 + + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 1 + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 1 + + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 1 + + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 1 + + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 2 + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 2 + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 1 + + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 2 + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 2 + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 2 + + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 2 + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 2 + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 2 + + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 3 + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 3 + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 3 + + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 3 + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 3 + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 3 + + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 3 + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 3 + + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 4 + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 4 + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 4 + + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 4 + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 4 + + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 4 + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 4 + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 4 + + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 5 + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 5 + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 5 + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 5 + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 5 + + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 5 + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 5 + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 5 + + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 6 + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 6 + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 6 + + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 6 + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 6 + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 6 + + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 6 + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 6 + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 7 + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 7 + + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 7 + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 7 + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 7 + + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 7 + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 7 + + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 7 + + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 8 + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 8 + + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 8 + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 8 + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 8 + + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 8 + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 8 + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 8 + + ld1 { $acc_lb}, [$current_tag] + ext $acc_lb, $acc_lb, $acc_lb, #8 + rev64 $acc_lb, $acc_lb + ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11 + + aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 9 + aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 9 + aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 9 + + aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 9 + aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 9 + aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 9 + + aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 9 + + aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 10 + aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 10 + aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 9 + + aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 10 + aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 10 + aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 10 + + aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 10 + aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 10 + aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 10 + + aese $ctr4b, $rk11 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 11 + ldp $rk12q, $rk13q, [$cc, #192] @ load rk12, rk13 + aese $ctr5b, $rk11 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 11 + + aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 11 + aese $ctr6b, $rk11 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 11 + aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 11 + + aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 11 + aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 11 + aese $ctr7b, $rk11 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 11 + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 7 + ldr $rk14q, [$cc, #224] @ load rk14 + + aese $ctr4b, $rk12 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 12 + aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 12 + aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 12 + + aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 12 + aese $ctr5b, $rk12 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 12 + aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 12 + + aese $ctr2b, $rk13 @ AES block 2 - round 13 + aese $ctr1b, $rk13 @ AES block 1 - round 13 + aese $ctr4b, $rk13 @ AES block 4 - round 13 + + aese $ctr6b, $rk12 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 12 + aese $ctr7b, $rk12 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 12 + + aese $ctr0b, $rk13 @ AES block 0 - round 13 + aese $ctr5b, $rk13 @ AES block 5 - round 13 + + aese $ctr6b, $rk13 @ AES block 6 - round 13 + aese $ctr7b, $rk13 @ AES block 7 - round 13 + aese $ctr3b, $rk13 @ AES block 3 - round 13 + + add $end_input_ptr, $input_ptr, $bit_length, lsr #3 @ end_input_ptr + cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks + b.ge .L256_enc_tail @ handle tail + + ldp $ctr_t0q, $ctr_t1q, [$input_ptr], #32 @ AES block 0, 1 - load plaintext + + ldp $ctr_t2q, $ctr_t3q, [$input_ptr], #32 @ AES block 2, 3 - load plaintext + + eor3 $res0b, $ctr_t0b, $ctr0b, $rk14 @ AES block 0 - result + rev32 $ctr0.16b, $rtmp_ctr.16b @ CTR block 8 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8 + + eor3 $res1b, $ctr_t1b, $ctr1b, $rk14 @ AES block 1 - result + eor3 $res3b, $ctr_t3b, $ctr3b, $rk14 @ AES block 3 - result + + rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 9 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 9 + ldp $ctr_t4q, $ctr_t5q, [$input_ptr], #32 @ AES block 4, 5 - load plaintext + + ldp $ctr_t6q, $ctr_t7q, [$input_ptr], #32 @ AES block 6, 7 - load plaintext + eor3 $res2b, $ctr_t2b, $ctr2b, $rk14 @ AES block 2 - result + cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks + + rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 10 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 10 + stp $res0q, $res1q, [$output_ptr], #32 @ AES block 0, 1 - store result + + stp $res2q, $res3q, [$output_ptr], #32 @ AES block 2, 3 - store result + + rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 11 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 11 + + eor3 $res4b, $ctr_t4b, $ctr4b, $rk14 @ AES block 4 - result + + eor3 $res7b, $ctr_t7b, $ctr7b, $rk14 @ AES block 7 - result + eor3 $res6b, $ctr_t6b, $ctr6b, $rk14 @ AES block 6 - result + eor3 $res5b, $ctr_t5b, $ctr5b, $rk14 @ AES block 5 - result + + stp $res4q, $res5q, [$output_ptr], #32 @ AES block 4, 5 - store result + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 12 + + stp $res6q, $res7q, [$output_ptr], #32 @ AES block 6, 7 - store result + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 12 + b.ge .L256_enc_prepretail @ do prepretail + +.L256_enc_main_loop: @ main loop start + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13 + ldr $h56kq, [$current_tag, #144] @ load h6k | h5k + ldr $h78kq, [$current_tag, #192] @ load h8k | h7k + + rev64 $res3b, $res3b @ GHASH block 8k+3 + ldr $h5q, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + ldr $h6q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + rev64 $res1b, $res1b @ GHASH block 8k+1 + + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14 + rev64 $res0b, $res0b @ GHASH block 8k + + rev64 $res4b, $res4b @ GHASH block 8k+4 + ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0 + ldr $h7q, [$current_tag, #176] @ load h7l | h7h + ext $h7.16b, $h7.16b, $h7.16b, #8 + ldr $h8q, [$current_tag, #208] @ load h8l | h8h + ext $h8.16b, $h8.16b, $h8.16b, #8 + + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0 + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0 + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15 + + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0 + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0 + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0 + + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0 + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0 + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0 + + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + eor $res0b, $res0b, $acc_lb @ PRE 1 + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1 + + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1 + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1 + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1 + + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1 + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1 + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1 + + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low + pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high + + trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1 + + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2 + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2 + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2 + + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2 + pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2 + + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3 + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3 + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2 + + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3 + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2 + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2 + + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3 + rev64 $res6b, $res6b @ GHASH block 8k+6 + pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high + + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3 + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + rev64 $res2b, $res2b @ GHASH block 8k+2 + + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3 + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3 + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3 + + eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high + pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high + rev64 $res5b, $res5b @ GHASH block 8k+5 + + pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low + eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + + trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high + pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low + + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4 + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4 + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4 + + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4 + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4 + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4 + + trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4 + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4 + + trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5 + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5 + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5 + + eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5 + rev64 $res7b, $res7b @ GHASH block 8k+7 + + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5 + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5 + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5 + + pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid + pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5 + + pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6 + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6 + + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6 + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6 + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6 + + eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid + pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6 + + eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6 + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6 + + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7 + + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + ldr $h2q, [$current_tag, #64] @ load h1l | h1h + ext $h2.16b, $h2.16b, $h2.16b, #8 + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7 + eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7 + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7 + + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7 + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7 + pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low + + trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7 + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7 + + pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8 + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8 + + pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low + trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8 + aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 9 + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8 + + pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid + pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8 + + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8 + pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high + pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low + + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8 + trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8 + + eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 9 + aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 9 + + eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 9 + aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 9 + + ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11 + aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 9 + aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 9 + + pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high + eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low + pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low + + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid + pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid + + aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 9 + + eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low + eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high + + aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 10 + aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 10 + aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 10 + + aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 10 + aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 10 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15 + + aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 10 + aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 10 + aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 10 + + eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high + + ldp $rk12q, $rk13q, [$cc, #192] @ load rk12, rk13 + rev32 $h1.16b, $rtmp_ctr.16b @ CTR block 8k+16 + + ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + ldp $ctr_t0q, $ctr_t1q, [$input_ptr], #32 @ AES block 8k+8, 8k+9 - load plaintext + aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 11 + + aese $ctr6b, $rk11 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 11 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+16 + aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 11 + + aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 11 + aese $ctr7b, $rk11 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 11 + + pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 11 + + aese $ctr7b, $rk12 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 12 + aese $ctr5b, $rk11 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 11 + + aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 12 + aese $ctr6b, $rk12 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 12 + rev32 $h2.16b, $rtmp_ctr.16b @ CTR block 8k+17 + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+17 + aese $ctr4b, $rk11 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 11 + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + + aese $ctr5b, $rk12 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 12 + ldr $rk14q, [$cc, #224] @ load rk14 + aese $ctr7b, $rk13 @ AES block 8k+15 - round 13 + + ldp $ctr_t2q, $ctr_t3q, [$input_ptr], #32 @ AES block 8k+10, 8k+11 - load plaintext + aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 12 + aese $ctr4b, $rk12 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 12 + + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 12 + ldp $ctr_t4q, $ctr_t5q, [$input_ptr], #32 @ AES block 4, 5 - load plaintext + + ldp $ctr_t6q, $ctr_t7q, [$input_ptr], #32 @ AES block 6, 7 - load plaintext + aese $ctr2b, $rk13 @ AES block 8k+10 - round 13 + aese $ctr4b, $rk13 @ AES block 8k+12 - round 13 + + rev32 $h3.16b, $rtmp_ctr.16b @ CTR block 8k+18 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+18 + aese $ctr5b, $rk13 @ AES block 8k+13 - round 13 + + aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 12 + aese $ctr3b, $rk13 @ AES block 8k+11 - round 13 + cmp $input_ptr, $main_end_input_ptr @ LOOP CONTROL + + eor3 $res2b, $ctr_t2b, $ctr2b, $rk14 @ AES block 8k+10 - result + rev32 $h4.16b, $rtmp_ctr.16b @ CTR block 8k+19 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+19 + + aese $ctr0b, $rk13 @ AES block 8k+8 - round 13 + aese $ctr6b, $rk13 @ AES block 8k+14 - round 13 + eor3 $res5b, $ctr_t5b, $ctr5b, $rk14 @ AES block 5 - result + + ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + aese $ctr1b, $rk13 @ AES block 8k+9 - round 13 + + eor3 $res4b, $ctr_t4b, $ctr4b, $rk14 @ AES block 4 - result + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 8k+20 + eor3 $res3b, $ctr_t3b, $ctr3b, $rk14 @ AES block 8k+11 - result + + mov $ctr3.16b, $h4.16b @ CTR block 8k+19 + eor3 $res1b, $ctr_t1b, $ctr1b, $rk14 @ AES block 8k+9 - result + eor3 $res0b, $ctr_t0b, $ctr0b, $rk14 @ AES block 8k+8 - result + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+20 + stp $res0q, $res1q, [$output_ptr], #32 @ AES block 8k+8, 8k+9 - store result + mov $ctr2.16b, $h3.16b @ CTR block 8k+18 + + eor3 $res7b, $ctr_t7b, $ctr7b, $rk14 @ AES block 7 - result + eor3 $acc_lb, $acc_lb, $t11.16b, $acc_hb @ MODULO - fold into low + stp $res2q, $res3q, [$output_ptr], #32 @ AES block 8k+10, 8k+11 - store result + + eor3 $res6b, $ctr_t6b, $ctr6b, $rk14 @ AES block 6 - result + mov $ctr1.16b, $h2.16b @ CTR block 8k+17 + stp $res4q, $res5q, [$output_ptr], #32 @ AES block 4, 5 - store result + + stp $res6q, $res7q, [$output_ptr], #32 @ AES block 6, 7 - store result + mov $ctr0.16b, $h1.16b @ CTR block 8k+16 + b.lt .L256_enc_main_loop + +.L256_enc_prepretail: @ PREPRETAIL + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13 + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13 + + rev64 $res2b, $res2b @ GHASH block 8k+2 + + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14 + + rev64 $res5b, $res5b @ GHASH block 8k+5 + ldr $h56kq, [$current_tag, #144] @ load h6k | h5k + ldr $h78kq, [$current_tag, #192] @ load h8k | h7k + + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15 + + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0 + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0 + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0 + + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0 + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0 + + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0 + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0 + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0 + + ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0 + rev64 $res0b, $res0b @ GHASH block 8k + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1 + + rev64 $res1b, $res1b @ GHASH block 8k+1 + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1 + + ldr $h7q, [$current_tag, #176] @ load h7l | h7h + ext $h7.16b, $h7.16b, $h7.16b, #8 + ldr $h8q, [$current_tag, #208] @ load h8l | h8h + ext $h8.16b, $h8.16b, $h8.16b, #8 + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1 + + ldr $h5q, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + ldr $h6q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1 + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1 + + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1 + eor $res0b, $res0b, $acc_lb @ PRE 1 + + rev64 $res3b, $res3b @ GHASH block 8k+3 + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1 + + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2 + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2 + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1 + + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2 + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2 + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2 + + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2 + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2 + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2 + + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high + + rev64 $res6b, $res6b @ GHASH block 8k+6 + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3 + pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high + + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3 + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low + trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + + pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3 + + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3 + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3 + eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high + + pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low + pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3 + + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3 + eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3 + + pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4 + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4 + + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4 + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4 + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4 + + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5 + pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid + eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high + + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4 + trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4 + eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4 + + pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low + pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid + eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + + rev64 $res4b, $res4b @ GHASH block 8k+4 + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5 + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5 + + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5 + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5 + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid + pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid + + eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low + eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid + + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5 + rev64 $res7b, $res7b @ GHASH block 8k+7 + trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5 + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5 + eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6 + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6 + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6 + + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6 + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6 + + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6 + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6 + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6 + + pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high + pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + ldr $h2q, [$current_tag, #64] @ load h1l | h1h + ext $h2.16b, $h2.16b, $h2.16b, #8 + + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7 + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7 + + pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high + trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7 + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7 + pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low + + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7 + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7 + eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + + pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high + pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7 + + trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7 + + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8 + eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8 + + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8 + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8 + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8 + + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8 + eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8 + + pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid + pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8 + + pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high + pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid + pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid + + pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low + eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high + + ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11 + aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 9 + aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 9 + + eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high + eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + + eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low + + aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 9 + aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 9 + aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 9 + + aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 9 + aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 9 + + aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 10 + aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 10 + aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 9 + + aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 10 + aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 10 + aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 10 + + aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 10 + aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 10 + aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 10 + + pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + aese $ctr7b, $rk11 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 11 + + ldp $rk12q, $rk13q, [$cc, #192] @ load rk12, rk13 + ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 11 + + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 11 + aese $ctr6b, $rk11 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 11 + + aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 11 + aese $ctr4b, $rk11 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 11 + aese $ctr5b, $rk11 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 11 + + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 11 + ldr $rk14q, [$cc, #224] @ load rk14 + + aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 12 + aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 12 + aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 12 + + aese $ctr6b, $rk12 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 12 + aese $ctr5b, $rk12 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 12 + ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + + aese $ctr4b, $rk12 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 12 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15 + + aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 12 + aese $ctr7b, $rk12 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 12 + aese $ctr0b, $rk13 @ AES block 8k+8 - round 13 + + eor3 $acc_lb, $acc_lb, $t11.16b, $acc_hb @ MODULO - fold into low + aese $ctr5b, $rk13 @ AES block 8k+13 - round 13 + aese $ctr1b, $rk13 @ AES block 8k+9 - round 13 + + aese $ctr3b, $rk13 @ AES block 8k+11 - round 13 + aese $ctr4b, $rk13 @ AES block 8k+12 - round 13 + aese $ctr7b, $rk13 @ AES block 8k+15 - round 13 + + aese $ctr2b, $rk13 @ AES block 8k+10 - round 13 + aese $ctr6b, $rk13 @ AES block 8k+14 - round 13 +.L256_enc_tail: @ TAIL + + ldp $h78kq, $h8q, [$current_tag, #192] @ load h8l | h8h + ext $h8.16b, $h8.16b, $h8.16b, #8 + sub $main_end_input_ptr, $end_input_ptr, $input_ptr @ main_end_input_ptr is number of bytes left to process + + ldr $ctr_t0q, [$input_ptr], #16 @ AES block 8k+8 - load plaintext + + ldp $h5q, $h56kq, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + + ext $t0.16b, $acc_lb, $acc_lb, #8 @ prepare final partial tag + ldp $h6q, $h7q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + ext $h7.16b, $h7.16b, $h7.16b, #8 + mov $t1.16b, $rk14 + + cmp $main_end_input_ptr, #112 + eor3 $res1b, $ctr_t0b, $ctr0b, $t1.16b @ AES block 8k+8 - result + b.gt .L256_enc_blocks_more_than_7 + + movi $acc_l.8b, #0 + mov $ctr7b, $ctr6b + movi $acc_h.8b, #0 + + mov $ctr6b, $ctr5b + mov $ctr5b, $ctr4b + mov $ctr4b, $ctr3b + + mov $ctr3b, $ctr2b + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + mov $ctr2b, $ctr1b + + movi $acc_m.8b, #0 + cmp $main_end_input_ptr, #96 + b.gt .L256_enc_blocks_more_than_6 + + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr5b + cmp $main_end_input_ptr, #80 + + mov $ctr5b, $ctr4b + mov $ctr4b, $ctr3b + mov $ctr3b, $ctr1b + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + b.gt .L256_enc_blocks_more_than_5 + + mov $ctr7b, $ctr6b + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + + mov $ctr6b, $ctr5b + mov $ctr5b, $ctr4b + + cmp $main_end_input_ptr, #64 + mov $ctr4b, $ctr1b + b.gt .L256_enc_blocks_more_than_4 + + cmp $main_end_input_ptr, #48 + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr5b + + mov $ctr5b, $ctr1b + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + b.gt .L256_enc_blocks_more_than_3 + + cmp $main_end_input_ptr, #32 + mov $ctr7b, $ctr6b + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + + mov $ctr6b, $ctr1b + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + b.gt .L256_enc_blocks_more_than_2 + + mov $ctr7b, $ctr1b + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + cmp $main_end_input_ptr, #16 + b.gt .L256_enc_blocks_more_than_1 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + b .L256_enc_blocks_less_than_1 +.L256_enc_blocks_more_than_7: @ blocks left > 7 + st1 { $res1b}, [$output_ptr], #16 @ AES final-7 block - store result + + rev64 $res0b, $res1b @ GHASH final-7 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-6 block - load plaintext + + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH final-7 block - high + ins $rk4v.d[0], $res0.d[1] @ GHASH final-7 block - mid + ins $acc_m.d[0], $h78k.d[1] @ GHASH final-7 block - mid + + movi $t0.8b, #0 @ surpress further partial tag feed in + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-7 block - mid + eor3 $res1b, $ctr_t1b, $ctr1b, $t1.16b @ AES final-6 block - result + + pmull $acc_m.1q, $rk4v.1d, $acc_m.1d @ GHASH final-7 block - mid + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH final-7 block - low +.L256_enc_blocks_more_than_6: @ blocks left > 6 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-6 block - store result + + rev64 $res0b, $res1b @ GHASH final-6 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + pmull $rk3q1, $res0.1d, $h7.1d @ GHASH final-6 block - low + ins $rk4v.d[0], $res0.d[1] @ GHASH final-6 block - mid + pmull2 $rk2q1, $res0.2d, $h7.2d @ GHASH final-6 block - high + + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-5 block - load plaintext + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-6 block - low + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-6 block - mid + + pmull $rk4v.1q, $rk4v.1d, $h78k.1d @ GHASH final-6 block - mid + eor3 $res1b, $ctr_t1b, $ctr2b, $t1.16b @ AES final-5 block - result + + movi $t0.8b, #0 @ surpress further partial tag feed in + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-6 block - mid + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-6 block - high +.L256_enc_blocks_more_than_5: @ blocks left > 5 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-5 block - store result + + rev64 $res0b, $res1b @ GHASH final-5 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-5 block - mid + + pmull2 $rk2q1, $res0.2d, $h6.2d @ GHASH final-5 block - high + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-5 block - high + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-5 block - mid + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-5 block - mid + + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-4 block - load plaintext + pmull $rk3q1, $res0.1d, $h6.1d @ GHASH final-5 block - low + + pmull2 $rk4v.1q, $rk4v.2d, $h56k.2d @ GHASH final-5 block - mid + movi $t0.8b, #0 @ surpress further partial tag feed in + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-5 block - low + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-5 block - mid + eor3 $res1b, $ctr_t1b, $ctr3b, $t1.16b @ AES final-4 block - result +.L256_enc_blocks_more_than_4: @ blocks left > 4 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-4 block - store result + + rev64 $res0b, $res1b @ GHASH final-4 block + + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-3 block - load plaintext + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-4 block - mid + pmull2 $rk2q1, $res0.2d, $h5.2d @ GHASH final-4 block - high + + eor3 $res1b, $ctr_t1b, $ctr4b, $t1.16b @ AES final-3 block - result + pmull $rk3q1, $res0.1d, $h5.1d @ GHASH final-4 block - low + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-4 block - mid + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-4 block - low + + pmull $rk4v.1q, $rk4v.1d, $h56k.1d @ GHASH final-4 block - mid + + movi $t0.8b, #0 @ surpress further partial tag feed in + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-4 block - mid + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-4 block - high +.L256_enc_blocks_more_than_3: @ blocks left > 3 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-3 block - store result + + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + rev64 $res0b, $res1b @ GHASH final-3 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-3 block - mid + pmull2 $rk2q1, $res0.2d, $h4.2d @ GHASH final-3 block - high + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-3 block - high + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-3 block - mid + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-3 block - mid + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-2 block - load plaintext + + pmull2 $rk4v.1q, $rk4v.2d, $h34k.2d @ GHASH final-3 block - mid + pmull $rk3q1, $res0.1d, $h4.1d @ GHASH final-3 block - low + + eor3 $res1b, $ctr_t1b, $ctr5b, $t1.16b @ AES final-2 block - result + movi $t0.8b, #0 @ surpress further partial tag feed in + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-3 block - mid + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-3 block - low +.L256_enc_blocks_more_than_2: @ blocks left > 2 + + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-2 block - store result + + rev64 $res0b, $res1b @ GHASH final-2 block + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-1 block - load plaintext + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-2 block - mid + + movi $t0.8b, #0 @ surpress further partial tag feed in + + pmull2 $rk2q1, $res0.2d, $h3.2d @ GHASH final-2 block - high + eor3 $res1b, $ctr_t1b, $ctr6b, $t1.16b @ AES final-1 block - result + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-2 block - mid + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-2 block - high + + pmull $rk4v.1q, $rk4v.1d, $h34k.1d @ GHASH final-2 block - mid + pmull $rk3q1, $res0.1d, $h3.1d @ GHASH final-2 block - low + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-2 block - mid + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-2 block - low +.L256_enc_blocks_more_than_1: @ blocks left > 1 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-1 block - store result + + ldr $h2q, [$current_tag, #64] @ load h1l | h1h + ext $h2.16b, $h2.16b, $h2.16b, #8 + rev64 $res0b, $res1b @ GHASH final-1 block + ldr $ctr_t1q, [$input_ptr], #16 @ AES final block - load plaintext + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + movi $t0.8b, #0 @ surpress further partial tag feed in + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-1 block - mid + pmull2 $rk2q1, $res0.2d, $h2.2d @ GHASH final-1 block - high + + eor3 $res1b, $ctr_t1b, $ctr7b, $t1.16b @ AES final block - result + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-1 block - high + + pmull $rk3q1, $res0.1d, $h2.1d @ GHASH final-1 block - low + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-1 block - mid + + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-1 block - low + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-1 block - mid + + pmull2 $rk4v.1q, $rk4v.2d, $h12k.2d @ GHASH final-1 block - mid + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-1 block - mid +.L256_enc_blocks_less_than_1: @ blocks left <= 1 + + and $bit_length, $bit_length, #127 @ bit_length %= 128 + + sub $bit_length, $bit_length, #128 @ bit_length -= 128 + + neg $bit_length, $bit_length @ bit_length = 128 - #bits in input (in range [1,128]) + + mvn $temp0_x, xzr @ temp0_x = 0xffffffffffffffff + and $bit_length, $bit_length, #127 @ bit_length %= 128 + + lsr $temp0_x, $temp0_x, $bit_length @ temp0_x is mask for top 64b of last block + cmp $bit_length, #64 + mvn $temp1_x, xzr @ temp1_x = 0xffffffffffffffff + + csel $temp3_x, $temp0_x, xzr, lt + csel $temp2_x, $temp1_x, $temp0_x, lt + + mov $ctr0.d[0], $temp2_x @ ctr0b is mask for last block + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + + ld1 { $rk0}, [$output_ptr] @ load existing bytes where the possibly partial last block is to be stored + mov $ctr0.d[1], $temp3_x + + and $res1b, $res1b, $ctr0b @ possibly partial last block has zeroes in highest bits + + rev64 $res0b, $res1b @ GHASH final block + + rev32 $rtmp_ctr.16b, $rtmp_ctr.16b + bif $res1b, $rk0, $ctr0b @ insert existing bytes in top end of result before storing + str $rtmp_ctrq, [$counter] @ store the updated counter + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + st1 { $res1b}, [$output_ptr] @ store all 16B + + ins $t0.d[0], $res0.d[1] @ GHASH final block - mid + pmull2 $rk2q1, $res0.2d, $h1.2d @ GHASH final block - high + pmull $rk3q1, $res0.1d, $h1.1d @ GHASH final block - low + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final block - high + eor $acc_lb, $acc_lb, $rk3 @ GHASH final block - low + + eor $t0.8b, $t0.8b, $res0.8b @ GHASH final block - mid + + pmull $t0.1q, $t0.1d, $h12k.1d @ GHASH final block - mid + + eor $acc_mb, $acc_mb, $t0.16b @ GHASH final block - mid + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + + ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + + eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low + ext $acc_lb, $acc_lb, $acc_lb, #8 + rev64 $acc_lb, $acc_lb + st1 { $acc_l.16b }, [$current_tag] + lsr x0, $bit_length, #3 @ return sizes + + ldp d10, d11, [sp, #16] + ldp d12, d13, [sp, #32] + ldp d14, d15, [sp, #48] + ldp d8, d9, [sp], #80 + ret + +.L256_enc_ret: + mov w0, #0x0 + ret +.size unroll8_eor3_aes_gcm_enc_256_kernel,.-unroll8_eor3_aes_gcm_enc_256_kernel +___ + +{ +######################################################################################### +# size_t unroll8_eor3_aes_gcm_dec_256_kernel(const unsigned char *in, +# size_t len, +# unsigned char *out, +# const void *key, +# unsigned char ivec[16], +# u64 *Xi); +# +$code.=<<___; +.global unroll8_eor3_aes_gcm_dec_256_kernel +.type unroll8_eor3_aes_gcm_dec_256_kernel,%function +.align 4 +unroll8_eor3_aes_gcm_dec_256_kernel: + AARCH64_VALID_CALL_TARGET + cbz x1, .L256_dec_ret + stp d8, d9, [sp, #-80]! + mov $counter, x4 + mov $cc, x5 + stp d10, d11, [sp, #16] + stp d12, d13, [sp, #32] + stp d14, d15, [sp, #48] + mov x5, #0xc200000000000000 + stp x5, xzr, [sp, #64] + add $modulo_constant, sp, #64 + + ld1 { $ctr0b}, [$counter] @ CTR block 0 + + mov $constant_temp, #0x100000000 @ set up counter increment + movi $rctr_inc.16b, #0x0 + mov $rctr_inc.d[1], $constant_temp + lsr $main_end_input_ptr, $bit_length, #3 @ byte_len + + sub $main_end_input_ptr, $main_end_input_ptr, #1 @ byte_len - 1 + + rev32 $rtmp_ctr.16b, $ctr0.16b @ set up reversed counter + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 0 + + rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 1 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 1 + + rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 2 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 2 + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + + rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 3 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 3 + + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 4 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 4 + + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 0 + + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 5 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 5 + + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 0 + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 0 + + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 6 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 6 + + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 7 + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 0 + + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 0 + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 0 + + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 0 + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 0 + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 1 + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 1 + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 1 + + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 1 + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 1 + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 1 + + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 1 + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 1 + + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 2 + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 2 + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 2 + + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 2 + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 2 + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 2 + + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 2 + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 2 + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 3 + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 3 + + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 3 + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 3 + + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 3 + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 3 + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 3 + + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 3 + + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 4 + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 4 + + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 4 + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 4 + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 4 + + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 4 + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 4 + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 4 + + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 5 + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 5 + + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 5 + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 5 + + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 5 + + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 5 + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 5 + + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 5 + + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 6 + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 6 + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 6 + + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 6 + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 6 + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 6 + + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 6 + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 6 + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 7 + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 7 + + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 7 + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 7 + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 7 + + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 7 + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 7 + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 7 + + and $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffff80 @ number of bytes to be processed in main loop (at least 1 byte must be handled by tail) + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 8 + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 8 + + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 8 + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 8 + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 8 + + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 8 + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 8 + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 8 + + aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 9 + + ld1 { $acc_lb}, [$current_tag] + ext $acc_lb, $acc_lb, $acc_lb, #8 + rev64 $acc_lb, $acc_lb + ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11 + add $end_input_ptr, $input_ptr, $bit_length, lsr #3 @ end_input_ptr + add $main_end_input_ptr, $main_end_input_ptr, $input_ptr + + aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 9 + aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 9 + + aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 9 + aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 9 + + aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 9 + + aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 9 + aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 9 + + aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 10 + aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 10 + aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 10 + + aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 10 + aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 10 + aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 10 + + aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 10 + aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 10 + ldp $rk12q, $rk13q, [$cc, #192] @ load rk12, rk13 + + aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 11 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 7 + + aese $ctr7b, $rk11 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 11 + aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 11 + aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 11 + + aese $ctr5b, $rk11 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 11 + aese $ctr4b, $rk11 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 11 + aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 11 + + aese $ctr6b, $rk11 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 11 + ldr $rk14q, [$cc, #224] @ load rk14 + + aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 12 + aese $ctr4b, $rk12 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 12 + aese $ctr5b, $rk12 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 12 + + cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks + aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 12 + aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 12 + + aese $ctr6b, $rk12 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 12 + aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 12 + aese $ctr7b, $rk12 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 12 + + aese $ctr5b, $rk13 @ AES block 5 - round 13 + aese $ctr1b, $rk13 @ AES block 1 - round 13 + aese $ctr2b, $rk13 @ AES block 2 - round 13 + + aese $ctr0b, $rk13 @ AES block 0 - round 13 + aese $ctr4b, $rk13 @ AES block 4 - round 13 + aese $ctr6b, $rk13 @ AES block 6 - round 13 + + aese $ctr3b, $rk13 @ AES block 3 - round 13 + aese $ctr7b, $rk13 @ AES block 7 - round 13 + b.ge .L256_dec_tail @ handle tail + + ldp $res0q, $res1q, [$input_ptr], #32 @ AES block 0, 1 - load ciphertext + + ldp $res2q, $res3q, [$input_ptr], #32 @ AES block 2, 3 - load ciphertext + + ldp $res4q, $res5q, [$input_ptr], #32 @ AES block 4, 5 - load ciphertext + + ldp $res6q, $res7q, [$input_ptr], #32 @ AES block 6, 7 - load ciphertext + cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks + + eor3 $ctr1b, $res1b, $ctr1b, $rk14 @ AES block 1 - result + eor3 $ctr0b, $res0b, $ctr0b, $rk14 @ AES block 0 - result + stp $ctr0q, $ctr1q, [$output_ptr], #32 @ AES block 0, 1 - store result + + rev32 $ctr0.16b, $rtmp_ctr.16b @ CTR block 8 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8 + eor3 $ctr3b, $res3b, $ctr3b, $rk14 @ AES block 3 - result + + eor3 $ctr5b, $res5b, $ctr5b, $rk14 @ AES block 5 - result + + eor3 $ctr4b, $res4b, $ctr4b, $rk14 @ AES block 4 - result + rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 9 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 9 + + eor3 $ctr2b, $res2b, $ctr2b, $rk14 @ AES block 2 - result + stp $ctr2q, $ctr3q, [$output_ptr], #32 @ AES block 2, 3 - store result + + rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 10 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 10 + + eor3 $ctr6b, $res6b, $ctr6b, $rk14 @ AES block 6 - result + + rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 11 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 11 + stp $ctr4q, $ctr5q, [$output_ptr], #32 @ AES block 4, 5 - store result + + eor3 $ctr7b, $res7b, $ctr7b, $rk14 @ AES block 7 - result + stp $ctr6q, $ctr7q, [$output_ptr], #32 @ AES block 6, 7 - store result + + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 12 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 12 + b.ge .L256_dec_prepretail @ do prepretail + +.L256_dec_main_loop: @ main loop start + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13 + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13 + + rev64 $res1b, $res1b @ GHASH block 8k+1 + ldr $h7q, [$current_tag, #176] @ load h7l | h7h + ext $h7.16b, $h7.16b, $h7.16b, #8 + ldr $h8q, [$current_tag, #208] @ load h8l | h8h + ext $h8.16b, $h8.16b, $h8.16b, #8 + + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14 + rev64 $res0b, $res0b @ GHASH block 8k + + ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0 + rev64 $res4b, $res4b @ GHASH block 8k+4 + rev64 $res3b, $res3b @ GHASH block 8k+3 + + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15 + rev64 $res7b, $res7b @ GHASH block 8k+7 + + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0 + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0 + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0 + + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0 + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0 + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0 + + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0 + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0 + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + + eor $res0b, $res0b, $acc_lb @ PRE 1 + ldr $h5q, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + ldr $h6q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1 + + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1 + rev64 $res2b, $res2b @ GHASH block 8k+2 + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1 + + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1 + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1 + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1 + + trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1 + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1 + + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2 + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2 + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2 + + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2 + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2 + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low + + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2 + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2 + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2 + + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3 + + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3 + pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high + pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low + + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3 + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3 + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high + + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3 + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3 + trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + + pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3 + eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high + + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4 + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3 + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4 + + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4 + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4 + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4 + + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4 + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4 + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4 + + ldr $h56kq, [$current_tag, #144] @ load h6k | h5k + ldr $h78kq, [$current_tag, #192] @ load h8k | h7k + eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid + pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low + + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5 + eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low + + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5 + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5 + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5 + + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5 + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5 + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5 + + eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high + trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + rev64 $res5b, $res5b @ GHASH block 8k+5 + + pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid + pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid + trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6 + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6 + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5 + + trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6 + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6 + + eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6 + + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6 + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6 + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6 + + pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid + pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid + eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low + + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + rev64 $res6b, $res6b @ GHASH block 8k+6 + eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid + + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7 + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7 + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + ldr $h2q, [$current_tag, #64] @ load h1l | h1h + ext $h2.16b, $h2.16b, $h2.16b, #8 + eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7 + + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7 + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7 + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7 + + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7 + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7 + + pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high + pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low + trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8 + pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8 + + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8 + pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8 + + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8 + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8 + pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high + + trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8 + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8 + + ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11 + pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low + trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15 + eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high + aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 9 + + aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 9 + eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 9 + + ldp $res0q, $res1q, [$input_ptr], #32 @ AES block 8k+8, 8k+9 - load ciphertext + eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 9 + + pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid + aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 9 + aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 9 + + pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid + pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid + pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high + + pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low + aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 10 + aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 10 + + pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid + aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 9 + eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low + + aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 9 + eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high + + aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 10 + aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 10 + aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 10 + + aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 10 + aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 10 + aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 10 + + eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low + rev32 $h1.16b, $rtmp_ctr.16b @ CTR block 8k+16 + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+16 + aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 11 + ldp $rk12q, $rk13q, [$cc, #192] @ load rk12, rk13 + + aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 11 + aese $ctr6b, $rk11 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 11 + + eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + rev32 $h2.16b, $rtmp_ctr.16b @ CTR block 8k+17 + aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 11 + + ldp $res2q, $res3q, [$input_ptr], #32 @ AES block 8k+10, 8k+11 - load ciphertext + aese $ctr7b, $rk11 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 11 + ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + + aese $ctr5b, $rk11 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 11 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+17 + aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 11 + + aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 12 + aese $ctr7b, $rk12 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 12 + aese $ctr6b, $rk12 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 12 + + rev32 $h3.16b, $rtmp_ctr.16b @ CTR block 8k+18 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+18 + pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 12 + aese $ctr4b, $rk11 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 11 + + ldr $rk14q, [$cc, #224] @ load rk14 + aese $ctr5b, $rk12 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 12 + aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 12 + + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 12 + aese $ctr4b, $rk12 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 12 + + ldp $res4q, $res5q, [$input_ptr], #32 @ AES block 8k+12, 8k+13 - load ciphertext + aese $ctr1b, $rk13 @ AES block 8k+9 - round 13 + aese $ctr2b, $rk13 @ AES block 8k+10 - round 13 + + ldp $res6q, $res7q, [$input_ptr], #32 @ AES block 8k+14, 8k+15 - load ciphertext + aese $ctr0b, $rk13 @ AES block 8k+8 - round 13 + aese $ctr5b, $rk13 @ AES block 8k+13 - round 13 + + rev32 $h4.16b, $rtmp_ctr.16b @ CTR block 8k+19 + eor3 $ctr2b, $res2b, $ctr2b, $rk14 @ AES block 8k+10 - result + eor3 $ctr1b, $res1b, $ctr1b, $rk14 @ AES block 8k+9 - result + + ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + aese $ctr7b, $rk13 @ AES block 8k+15 - round 13 + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+19 + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + aese $ctr4b, $rk13 @ AES block 8k+12 - round 13 + + eor3 $ctr5b, $res5b, $ctr5b, $rk14 @ AES block 8k+13 - result + eor3 $ctr0b, $res0b, $ctr0b, $rk14 @ AES block 8k+8 - result + aese $ctr3b, $rk13 @ AES block 8k+11 - round 13 + + stp $ctr0q, $ctr1q, [$output_ptr], #32 @ AES block 8k+8, 8k+9 - store result + mov $ctr0.16b, $h1.16b @ CTR block 8k+16 + eor3 $ctr4b, $res4b, $ctr4b, $rk14 @ AES block 8k+12 - result + + eor3 $acc_lb, $acc_lb, $t11.16b, $acc_hb @ MODULO - fold into low + eor3 $ctr3b, $res3b, $ctr3b, $rk14 @ AES block 8k+11 - result + stp $ctr2q, $ctr3q, [$output_ptr], #32 @ AES block 8k+10, 8k+11 - store result + + mov $ctr3.16b, $h4.16b @ CTR block 8k+19 + mov $ctr2.16b, $h3.16b @ CTR block 8k+18 + aese $ctr6b, $rk13 @ AES block 8k+14 - round 13 + + mov $ctr1.16b, $h2.16b @ CTR block 8k+17 + stp $ctr4q, $ctr5q, [$output_ptr], #32 @ AES block 8k+12, 8k+13 - store result + eor3 $ctr7b, $res7b, $ctr7b, $rk14 @ AES block 8k+15 - result + + eor3 $ctr6b, $res6b, $ctr6b, $rk14 @ AES block 8k+14 - result + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 8k+20 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+20 + + cmp $input_ptr, $main_end_input_ptr @ LOOP CONTROL + stp $ctr6q, $ctr7q, [$output_ptr], #32 @ AES block 8k+14, 8k+15 - store result + b.lt .L256_dec_main_loop + +.L256_dec_prepretail: @ PREPRETAIL + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13 + + rev64 $res4b, $res4b @ GHASH block 8k+4 + ldr $h56kq, [$current_tag, #144] @ load h6k | h5k + ldr $h78kq, [$current_tag, #192] @ load h8k | h7k + + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14 + rev64 $res0b, $res0b @ GHASH block 8k + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14 + + ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0 + ldr $h7q, [$current_tag, #176] @ load h7l | h7h + ext $h7.16b, $h7.16b, $h7.16b, #8 + ldr $h8q, [$current_tag, #208] @ load h8l | h8h + ext $h8.16b, $h8.16b, $h8.16b, #8 + rev64 $res1b, $res1b @ GHASH block 8k+1 + + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15 + rev64 $res2b, $res2b @ GHASH block 8k+2 + ldr $h5q, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + ldr $h6q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0 + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0 + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0 + + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0 + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0 + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0 + + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1 + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0 + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0 + + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1 + eor $res0b, $res0b, $acc_lb @ PRE 1 + + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1 + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1 + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1 + + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1 + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1 + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1 + + pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high + trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low + + rev64 $res3b, $res3b @ GHASH block 8k+3 + pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low + + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2 + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2 + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2 + + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2 + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2 + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high + + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2 + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3 + + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3 + rev64 $res6b, $res6b @ GHASH block 8k+6 + + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3 + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2 + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3 + + pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high + trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2 + + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3 + pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high + + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3 + eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high + eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid + + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3 + pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3 + + eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high + trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + + pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid + pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low + eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low + + pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4 + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4 + + eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + ldr $h2q, [$current_tag, #64] @ load h1l | h1h + ext $h2.16b, $h2.16b, $h2.16b, #8 + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4 + + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4 + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4 + eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid + + eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5 + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4 + + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5 + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4 + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4 + + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5 + pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5 + + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5 + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5 + pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid + + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5 + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5 + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + rev64 $res7b, $res7b @ GHASH block 8k+7 + rev64 $res5b, $res5b @ GHASH block 8k+5 + + eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + + trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6 + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6 + + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6 + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6 + + pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high + pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high + pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low + + trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low + trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7 + pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6 + + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6 + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6 + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6 + + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7 + + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7 + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7 + + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7 + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7 + eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high + + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7 + trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7 + + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8 + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8 + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8 + + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8 + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8 + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8 + + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8 + aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 9 + eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + + aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 9 + aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 9 + eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + + aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 9 + aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 9 + pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid + + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8 + pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid + pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high + + pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid + pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid + pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low + + ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11 + eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low + eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + + aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 9 + aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 9 + aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 9 + + eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high + eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + + eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + + aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 10 + aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 10 + aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 10 + + aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 10 + aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 10 + aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 10 + + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + + aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 10 + aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 10 + ldp $rk12q, $rk13q, [$cc, #192] @ load rk12, rk13 + + ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + + aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 11 + aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 11 + aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 11 + + pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 11 + + aese $ctr7b, $rk11 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 11 + aese $ctr6b, $rk11 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 11 + aese $ctr4b, $rk11 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 11 + + aese $ctr5b, $rk11 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 11 + aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 12 + + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + + aese $ctr3b, $rk13 @ AES block 8k+11 - round 13 + aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 12 + aese $ctr6b, $rk12 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 12 + + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + aese $ctr4b, $rk12 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 12 + aese $ctr7b, $rk12 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 12 + + aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 12 + ldr $rk14q, [$cc, #224] @ load rk14 + aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 12 + + aese $ctr4b, $rk13 @ AES block 8k+12 - round 13 + ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + aese $ctr5b, $rk12 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 12 + + aese $ctr6b, $rk13 @ AES block 8k+14 - round 13 + aese $ctr2b, $rk13 @ AES block 8k+10 - round 13 + aese $ctr1b, $rk13 @ AES block 8k+9 - round 13 + + aese $ctr5b, $rk13 @ AES block 8k+13 - round 13 + eor3 $acc_lb, $acc_lb, $t11.16b, $acc_hb @ MODULO - fold into low + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15 + + aese $ctr7b, $rk13 @ AES block 8k+15 - round 13 + aese $ctr0b, $rk13 @ AES block 8k+8 - round 13 +.L256_dec_tail: @ TAIL + + ext $t0.16b, $acc_lb, $acc_lb, #8 @ prepare final partial tag + sub $main_end_input_ptr, $end_input_ptr, $input_ptr @ main_end_input_ptr is number of bytes left to process + cmp $main_end_input_ptr, #112 + + ldr $res1q, [$input_ptr], #16 @ AES block 8k+8 - load ciphertext + + ldp $h78kq, $h8q, [$current_tag, #192] @ load h8l | h8h + ext $h8.16b, $h8.16b, $h8.16b, #8 + mov $t1.16b, $rk14 + + ldp $h5q, $h56kq, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + + eor3 $res4b, $res1b, $ctr0b, $t1.16b @ AES block 8k+8 - result + ldp $h6q, $h7q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + ext $h7.16b, $h7.16b, $h7.16b, #8 + b.gt .L256_dec_blocks_more_than_7 + + mov $ctr7b, $ctr6b + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + mov $ctr6b, $ctr5b + + mov $ctr5b, $ctr4b + mov $ctr4b, $ctr3b + movi $acc_l.8b, #0 + + movi $acc_h.8b, #0 + movi $acc_m.8b, #0 + mov $ctr3b, $ctr2b + + cmp $main_end_input_ptr, #96 + mov $ctr2b, $ctr1b + b.gt .L256_dec_blocks_more_than_6 + + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr5b + + mov $ctr5b, $ctr4b + cmp $main_end_input_ptr, #80 + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + + mov $ctr4b, $ctr3b + mov $ctr3b, $ctr1b + b.gt .L256_dec_blocks_more_than_5 + + cmp $main_end_input_ptr, #64 + mov $ctr7b, $ctr6b + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + + mov $ctr6b, $ctr5b + + mov $ctr5b, $ctr4b + mov $ctr4b, $ctr1b + b.gt .L256_dec_blocks_more_than_4 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + mov $ctr7b, $ctr6b + cmp $main_end_input_ptr, #48 + + mov $ctr6b, $ctr5b + mov $ctr5b, $ctr1b + b.gt .L256_dec_blocks_more_than_3 + + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + mov $ctr7b, $ctr6b + + cmp $main_end_input_ptr, #32 + mov $ctr6b, $ctr1b + b.gt .L256_dec_blocks_more_than_2 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + + mov $ctr7b, $ctr1b + cmp $main_end_input_ptr, #16 + b.gt .L256_dec_blocks_more_than_1 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + b .L256_dec_blocks_less_than_1 +.L256_dec_blocks_more_than_7: @ blocks left > 7 + rev64 $res0b, $res1b @ GHASH final-7 block + ldr $res1q, [$input_ptr], #16 @ AES final-6 block - load ciphertext + st1 { $res4b}, [$output_ptr], #16 @ AES final-7 block - store result + + ins $acc_m.d[0], $h78k.d[1] @ GHASH final-7 block - mid + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-7 block - mid + eor3 $res4b, $res1b, $ctr1b, $t1.16b @ AES final-6 block - result + + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH final-7 block - high + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-7 block - mid + movi $t0.8b, #0 @ surpress further partial tag feed in + + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH final-7 block - low + pmull $acc_m.1q, $rk4v.1d, $acc_m.1d @ GHASH final-7 block - mid +.L256_dec_blocks_more_than_6: @ blocks left > 6 + + rev64 $res0b, $res1b @ GHASH final-6 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + ldr $res1q, [$input_ptr], #16 @ AES final-5 block - load ciphertext + movi $t0.8b, #0 @ surpress further partial tag feed in + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-6 block - mid + st1 { $res4b}, [$output_ptr], #16 @ AES final-6 block - store result + pmull2 $rk2q1, $res0.2d, $h7.2d @ GHASH final-6 block - high + + pmull $rk3q1, $res0.1d, $h7.1d @ GHASH final-6 block - low + + eor3 $res4b, $res1b, $ctr2b, $t1.16b @ AES final-5 block - result + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-6 block - low + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-6 block - mid + + pmull $rk4v.1q, $rk4v.1d, $h78k.1d @ GHASH final-6 block - mid + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-6 block - mid + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-6 block - high +.L256_dec_blocks_more_than_5: @ blocks left > 5 + + rev64 $res0b, $res1b @ GHASH final-5 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + pmull2 $rk2q1, $res0.2d, $h6.2d @ GHASH final-5 block - high + ins $rk4v.d[0], $res0.d[1] @ GHASH final-5 block - mid + + ldr $res1q, [$input_ptr], #16 @ AES final-4 block - load ciphertext + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-5 block - mid + st1 { $res4b}, [$output_ptr], #16 @ AES final-5 block - store result + + pmull $rk3q1, $res0.1d, $h6.1d @ GHASH final-5 block - low + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-5 block - mid + + pmull2 $rk4v.1q, $rk4v.2d, $h56k.2d @ GHASH final-5 block - mid + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-5 block - high + eor3 $res4b, $res1b, $ctr3b, $t1.16b @ AES final-4 block - result + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-5 block - low + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-5 block - mid + movi $t0.8b, #0 @ surpress further partial tag feed in +.L256_dec_blocks_more_than_4: @ blocks left > 4 + + rev64 $res0b, $res1b @ GHASH final-4 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-4 block - mid + ldr $res1q, [$input_ptr], #16 @ AES final-3 block - load ciphertext + + movi $t0.8b, #0 @ surpress further partial tag feed in + + pmull $rk3q1, $res0.1d, $h5.1d @ GHASH final-4 block - low + pmull2 $rk2q1, $res0.2d, $h5.2d @ GHASH final-4 block - high + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-4 block - mid + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-4 block - high + + pmull $rk4v.1q, $rk4v.1d, $h56k.1d @ GHASH final-4 block - mid + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-4 block - low + st1 { $res4b}, [$output_ptr], #16 @ AES final-4 block - store result + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-4 block - mid + eor3 $res4b, $res1b, $ctr4b, $t1.16b @ AES final-3 block - result +.L256_dec_blocks_more_than_3: @ blocks left > 3 + + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + rev64 $res0b, $res1b @ GHASH final-3 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + ldr $res1q, [$input_ptr], #16 @ AES final-2 block - load ciphertext + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-3 block - mid + st1 { $res4b}, [$output_ptr], #16 @ AES final-3 block - store result + + eor3 $res4b, $res1b, $ctr5b, $t1.16b @ AES final-2 block - result + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-3 block - mid + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-3 block - mid + pmull $rk3q1, $res0.1d, $h4.1d @ GHASH final-3 block - low + pmull2 $rk2q1, $res0.2d, $h4.2d @ GHASH final-3 block - high + + movi $t0.8b, #0 @ surpress further partial tag feed in + pmull2 $rk4v.1q, $rk4v.2d, $h34k.2d @ GHASH final-3 block - mid + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-3 block - low + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-3 block - high + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-3 block - mid +.L256_dec_blocks_more_than_2: @ blocks left > 2 + + rev64 $res0b, $res1b @ GHASH final-2 block + + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + ldr $res1q, [$input_ptr], #16 @ AES final-1 block - load ciphertext + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-2 block - mid + + pmull $rk3q1, $res0.1d, $h3.1d @ GHASH final-2 block - low + st1 { $res4b}, [$output_ptr], #16 @ AES final-2 block - store result + eor3 $res4b, $res1b, $ctr6b, $t1.16b @ AES final-1 block - result + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-2 block - mid + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-2 block - low + movi $t0.8b, #0 @ surpress further partial tag feed in + + pmull $rk4v.1q, $rk4v.1d, $h34k.1d @ GHASH final-2 block - mid + pmull2 $rk2q1, $res0.2d, $h3.2d @ GHASH final-2 block - high + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-2 block - mid + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-2 block - high +.L256_dec_blocks_more_than_1: @ blocks left > 1 + + rev64 $res0b, $res1b @ GHASH final-1 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-1 block - mid + ldr $h2q, [$current_tag, #64] @ load h1l | h1h + ext $h2.16b, $h2.16b, $h2.16b, #8 + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-1 block - mid + ldr $res1q, [$input_ptr], #16 @ AES final block - load ciphertext + st1 { $res4b}, [$output_ptr], #16 @ AES final-1 block - store result + + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + pmull $rk3q1, $res0.1d, $h2.1d @ GHASH final-1 block - low + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-1 block - mid + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-1 block - low + + eor3 $res4b, $res1b, $ctr7b, $t1.16b @ AES final block - result + pmull2 $rk2q1, $res0.2d, $h2.2d @ GHASH final-1 block - high + + pmull2 $rk4v.1q, $rk4v.2d, $h12k.2d @ GHASH final-1 block - mid + + movi $t0.8b, #0 @ surpress further partial tag feed in + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-1 block - high + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-1 block - mid +.L256_dec_blocks_less_than_1: @ blocks left <= 1 + + ld1 { $rk0}, [$output_ptr] @ load existing bytes where the possibly partial last block is to be stored + mvn $temp0_x, xzr @ temp0_x = 0xffffffffffffffff + and $bit_length, $bit_length, #127 @ bit_length %= 128 + + sub $bit_length, $bit_length, #128 @ bit_length -= 128 + rev32 $rtmp_ctr.16b, $rtmp_ctr.16b + str $rtmp_ctrq, [$counter] @ store the updated counter + + neg $bit_length, $bit_length @ bit_length = 128 - #bits in input (in range [1,128]) + + and $bit_length, $bit_length, #127 @ bit_length %= 128 + + lsr $temp0_x, $temp0_x, $bit_length @ temp0_x is mask for top 64b of last block + cmp $bit_length, #64 + mvn $temp1_x, xzr @ temp1_x = 0xffffffffffffffff + + csel $temp3_x, $temp0_x, xzr, lt + csel $temp2_x, $temp1_x, $temp0_x, lt + + mov $ctr0.d[0], $temp2_x @ ctr0b is mask for last block + mov $ctr0.d[1], $temp3_x + + and $res1b, $res1b, $ctr0b @ possibly partial last block has zeroes in highest bits + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + bif $res4b, $rk0, $ctr0b @ insert existing bytes in top end of result before storing + + rev64 $res0b, $res1b @ GHASH final block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $t0.d[0], $res0.d[1] @ GHASH final block - mid + pmull2 $rk2q1, $res0.2d, $h1.2d @ GHASH final block - high + + eor $t0.8b, $t0.8b, $res0.8b @ GHASH final block - mid + + pmull $rk3q1, $res0.1d, $h1.1d @ GHASH final block - low + eor $acc_hb, $acc_hb, $rk2 @ GHASH final block - high + + pmull $t0.1q, $t0.1d, $h12k.1d @ GHASH final block - mid + + eor $acc_mb, $acc_mb, $t0.16b @ GHASH final block - mid + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + eor $acc_lb, $acc_lb, $rk3 @ GHASH final block - low + + pmull $t11.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + eor $t10.16b, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + + ext $acc_hb, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + st1 { $res4b}, [$output_ptr] @ store all 16B + + eor $acc_mb, $acc_mb, $t10.16b @ MODULO - karatsuba tidy up + + eor $t11.16b, $acc_hb, $t11.16b @ MODULO - fold into mid + eor $acc_mb, $acc_mb, $t11.16b @ MODULO - fold into mid + + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + + ext $acc_mb, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + eor $acc_lb, $acc_lb, $acc_hb @ MODULO - fold into low + + eor $acc_lb, $acc_lb, $acc_mb @ MODULO - fold into low + ext $acc_lb, $acc_lb, $acc_lb, #8 + rev64 $acc_lb, $acc_lb + st1 { $acc_l.16b }, [$current_tag] + lsr x0, $bit_length, #3 @ return sizes + + ldp d10, d11, [sp, #16] + ldp d12, d13, [sp, #32] + ldp d14, d15, [sp, #48] + ldp d8, d9, [sp], #80 + ret + +.L256_dec_ret: + mov w0, #0x0 + ret +.size unroll8_eor3_aes_gcm_dec_256_kernel,.-unroll8_eor3_aes_gcm_dec_256_kernel +___ +} +} + +$code.=<<___; +.asciz "AES GCM module for ARMv8, SPDX BSD-3-Clause by <xiaokang.qian\@arm.com>" +.align 2 +#endif +___ + +{ + my %opcode = ( + "rax1" => 0xce608c00, "eor3" => 0xce000000, + "bcax" => 0xce200000, "xar" => 0xce800000 ); + + sub unsha3 { + my ($mnemonic,$arg)=@_; + + $arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv#]([0-9\-]+))?)?/ + && + sprintf ".inst\t0x%08x\t//%s %s", + $opcode{$mnemonic}|$1|($2<<5)|($3<<16)|(eval($4)<<10), + $mnemonic,$arg; + } + sub unvmov { + my $arg=shift; + + $arg =~ m/q([0-9]+)#(lo|hi),\s*q([0-9]+)#(lo|hi)/o && + sprintf "ins v%d.d[%d],v%d.d[%d]",$1<8?$1:$1+8,($2 eq "lo")?0:1, + $3<8?$3:$3+8,($4 eq "lo")?0:1; + } + + foreach(split("\n",$code)) { + s/@\s/\/\//o; # old->new style commentary + s/\`([^\`]*)\`/eval($1)/ge; + + m/\bld1r\b/ and s/\.16b/.2d/g or + s/\b(eor3|rax1|xar|bcax)\s+(v.*)/unsha3($1,$2)/ge; + print $_,"\n"; + } +} + +close STDOUT or die "error closing STDOUT: $!"; # enforce flush Index: openssl-1.1.1l/crypto/modes/asm/ghashv8-armx.pl =================================================================== --- openssl-1.1.1l.orig/crypto/modes/asm/ghashv8-armx.pl +++ openssl-1.1.1l/crypto/modes/asm/ghashv8-armx.pl @@ -141,6 +141,7 @@ gcm_init_v8: ___ if ($flavour =~ /64/) { my ($t3,$Yl,$Ym,$Yh) = map("q$_",(4..7)); +my ($H3,$H34k,$H4,$H5,$H56k,$H6,$H7,$H78k,$H8) = map("q$_",(15..23)); $code.=<<___; @ calculate H^3 and H^4 @@ -175,15 +176,103 @@ $code.=<<___; vpmull.p64 $Yl,$Yl,$xC2 veor $t2,$t2,$Xh veor $t3,$t3,$Yh - veor $H, $Xl,$t2 @ H^3 - veor $H2,$Yl,$t3 @ H^4 + veor $H3, $Xl,$t2 @ H^3 + veor $H4,$Yl,$t3 @ H^4 - vext.8 $t0,$H, $H,#8 @ Karatsuba pre-processing - vext.8 $t1,$H2,$H2,#8 - veor $t0,$t0,$H - veor $t1,$t1,$H2 - vext.8 $Hhl,$t0,$t1,#8 @ pack Karatsuba pre-processed - vst1.64 {$H-$H2},[x0] @ store Htable[3..5] + vext.8 $t0,$H3, $H3,#8 @ Karatsuba pre-processing + vext.8 $t1,$H4,$H4,#8 + vext.8 $t2,$H2,$H2,#8 + veor $t0,$t0,$H3 + veor $t1,$t1,$H4 + veor $t2,$t2,$H2 + vext.8 $H34k,$t0,$t1,#8 @ pack Karatsuba pre-processed + vst1.64 {$H3-$H4},[x0],#48 @ store Htable[3..5] + + @ calculate H^5 and H^6 + vpmull.p64 $Xl,$H2, $H3 + vpmull.p64 $Yl,$H3,$H3 + vpmull2.p64 $Xh,$H2, $H3 + vpmull2.p64 $Yh,$H3,$H3 + vpmull.p64 $Xm,$t0,$t2 + vpmull.p64 $Ym,$t0,$t0 + + vext.8 $t0,$Xl,$Xh,#8 @ Karatsuba post-processing + vext.8 $t1,$Yl,$Yh,#8 + veor $t2,$Xl,$Xh + veor $Xm,$Xm,$t0 + veor $t3,$Yl,$Yh + veor $Ym,$Ym,$t1 + veor $Xm,$Xm,$t2 + vpmull.p64 $t2,$Xl,$xC2 @ 1st phase + veor $Ym,$Ym,$t3 + vpmull.p64 $t3,$Yl,$xC2 + + vmov $Xh#lo,$Xm#hi @ Xh|Xm - 256-bit result + vmov $Yh#lo,$Ym#hi + vmov $Xm#hi,$Xl#lo @ Xm is rotated Xl + vmov $Ym#hi,$Yl#lo + veor $Xl,$Xm,$t2 + veor $Yl,$Ym,$t3 + + vext.8 $t2,$Xl,$Xl,#8 @ 2nd phase + vext.8 $t3,$Yl,$Yl,#8 + vpmull.p64 $Xl,$Xl,$xC2 + vpmull.p64 $Yl,$Yl,$xC2 + veor $t2,$t2,$Xh + veor $t3,$t3,$Yh + veor $H5,$Xl,$t2 @ H^5 + veor $H6,$Yl,$t3 @ H^6 + + vext.8 $t0,$H5, $H5,#8 @ Karatsuba pre-processing + vext.8 $t1,$H6,$H6,#8 + vext.8 $t2,$H2,$H2,#8 + veor $t0,$t0,$H5 + veor $t1,$t1,$H6 + veor $t2,$t2,$H2 + vext.8 $H56k,$t0,$t1,#8 @ pack Karatsuba pre-processed + vst1.64 {$H5-$H6},[x0],#48 @ store Htable[6..8] + + @ calculate H^7 and H^8 + vpmull.p64 $Xl,$H2,$H5 + vpmull.p64 $Yl,$H2,$H6 + vpmull2.p64 $Xh,$H2,$H5 + vpmull2.p64 $Yh,$H2,$H6 + vpmull.p64 $Xm,$t0,$t2 + vpmull.p64 $Ym,$t1,$t2 + + vext.8 $t0,$Xl,$Xh,#8 @ Karatsuba post-processing + vext.8 $t1,$Yl,$Yh,#8 + veor $t2,$Xl,$Xh + veor $Xm,$Xm,$t0 + veor $t3,$Yl,$Yh + veor $Ym,$Ym,$t1 + veor $Xm,$Xm,$t2 + vpmull.p64 $t2,$Xl,$xC2 @ 1st phase + veor $Ym,$Ym,$t3 + vpmull.p64 $t3,$Yl,$xC2 + + vmov $Xh#lo,$Xm#hi @ Xh|Xm - 256-bit result + vmov $Yh#lo,$Ym#hi + vmov $Xm#hi,$Xl#lo @ Xm is rotated Xl + vmov $Ym#hi,$Yl#lo + veor $Xl,$Xm,$t2 + veor $Yl,$Ym,$t3 + + vext.8 $t2,$Xl,$Xl,#8 @ 2nd phase + vext.8 $t3,$Yl,$Yl,#8 + vpmull.p64 $Xl,$Xl,$xC2 + vpmull.p64 $Yl,$Yl,$xC2 + veor $t2,$t2,$Xh + veor $t3,$t3,$Yh + veor $H7,$Xl,$t2 @ H^7 + veor $H8,$Yl,$t3 @ H^8 + + vext.8 $t0,$H7,$H7,#8 @ Karatsuba pre-processing + vext.8 $t1,$H8,$H8,#8 + veor $t0,$t0,$H7 + veor $t1,$t1,$H8 + vext.8 $H78k,$t0,$t1,#8 @ pack Karatsuba pre-processed + vst1.64 {$H7-$H8},[x0] @ store Htable[9..11] ___ } $code.=<<___; Index: openssl-1.1.1l/crypto/modes/build.info =================================================================== --- openssl-1.1.1l.orig/crypto/modes/build.info +++ openssl-1.1.1l/crypto/modes/build.info @@ -20,6 +20,8 @@ GENERATE[ghash-armv4.S]=asm/ghash-armv4. INCLUDE[ghash-armv4.o]=.. GENERATE[ghashv8-armx.S]=asm/ghashv8-armx.pl $(PERLASM_SCHEME) INCLUDE[ghashv8-armx.o]=.. +GENERATE[aes-gcm-armv8-unroll8_64.S]=asm/aes-gcm-armv8-unroll8_64.pl $(PERLASM_SCHEME) +INCLUDE[aes-gcm-armv8-unroll8_64.o]=.. GENERATE[ghash-s390x.S]=asm/ghash-s390x.pl $(PERLASM_SCHEME) INCLUDE[ghash-s390x.o]=..
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor
