commit cae0536

working login, passwd, fdisk, ld, as, cc1, ip, ifconfig, b3sum

Signed-off-by: xplshn <anto@xplshn.com.ar>

 1@@ -29,12 +29,25 @@ cmd/dev/config.h
 2 cmd/dev/cc/config.h
 3 cmd/dev/version.h
 4 
 5-# build system generated files
 6+# build system generated files (genconfig.sh and ninqu)
 7 scripts/mk/config.mk
 8 scripts/mk/config.kv
 9+scripts/mk/config.set
10+scripts/mk/features.h
11 scripts/mk/rules.mk
12+shared/libutil/nofork_list.h
13 deps.mk
14 
15+# compiler dependency files
16+*.d
17+
18+# python bytecode
19+__pycache__/
20+*.pyc
21+
22+# generated man pages
23+man/man*/
24+
25 # ninja backend generator output and state
26 build.ninja
27 .ninja_log

1@@ -52,7 +52,7 @@ $(NINQU): $(NINQU_SRC) cmd/dev/ninqu/ninqu.h scripts/mk/config.set $(NINQU_LIB_S
2 FORCE:
3 
4 # ninqu generates build.ninja if ninja/samu is available and execs that
5-# instead of running its own backend, but you can use make BACKEND=internal
6+# instead of running its own backend, but you can use make backend=internal
7 # to force the internal one to be used anyways
8 %: $(NINQU) FORCE
9 	$(NINQU) $(JOBS) $@

  1@@ -1,11 +1,11 @@
  2 # posix: POSIX 2024
  3 PRESET_FEATURES=all
  4 
  5-# subset  of: posix xsi net pseudo extra linux dev
  6-PRESET_UTILS="posix xsi net pseudo extra linux"
  7+# subset of: posix xsi net pseudo extra linux dev
  8+PRESET_UTILS="posix xsi net pseudo extra linux dev"
  9 
 10 # installation paths
 11-VERSION=2026-06-08T05-53-UTC-03
 12+VERSION=2026-08-09T03-56-UTC-03
 13 PREFIX=/usr/local
 14 MANPREFIX=/usr/local/share/man
 15 
 16@@ -13,12 +13,12 @@ MANPREFIX=/usr/local/share/man
 17 CC=cc
 18 CFLAGS="-std=c99 -Wall -Wextra -pedantic"
 19 LDFLAGS=""
 20-LDLIBS="-lcrypt -lresolv"
 21+LDLIBS=""
 22 AR=ar
 23 ARFLAGS="rc"
 24 RANLIB=ranlib
 25 
 26-# --- preset expansion: do not edit below ---
 27+# preset expansion: do not edit below
 28 
 29 case "$PRESET_FEATURES" in
 30 posix)
 31@@ -52,6 +52,11 @@ posix)
 32     FEATURE_DEPMOD_ALIAS=0;           FEATURE_DEPMOD_SYMBOLS=0
 33     FEATURE_TAR_TTY_SAFE=0;           FEATURE_TAR_NOFOLLOW=0
 34     FEATURE_READLINK_REALPATH=0;      FEATURE_UMOUNT_OPTIONS=0
 35+    FEATURE_LN_RELATIVE=0
 36+    FEATURE_LN_NO_TARGET_DIR=0
 37+    FEATURE_BC_QUIET=0
 38+    FEATURE_BC_MULTICHAR=0
 39+    FEATURE_BC_ESCAPES=0
 40     FEATURE_TAR_TO_STDOUT=0;          FEATURE_TAR_KEEP_OLD=0
 41     FEATURE_TAR_STRIP_COMPONENTS=0;   FEATURE_TAR_FILES_FROM=0
 42     FEATURE_TAR_EXCLUDE_FROM=0;       FEATURE_IP_ROUTE_ADD_DEL=0
 43@@ -87,6 +92,11 @@ common)
 44     FEATURE_DEPMOD_ALIAS=1;           FEATURE_DEPMOD_SYMBOLS=1
 45     FEATURE_TAR_TTY_SAFE=1;           FEATURE_TAR_NOFOLLOW=1
 46     FEATURE_READLINK_REALPATH=1;      FEATURE_UMOUNT_OPTIONS=1
 47+    FEATURE_LN_RELATIVE=1
 48+    FEATURE_LN_NO_TARGET_DIR=1
 49+    FEATURE_BC_QUIET=1
 50+    FEATURE_BC_MULTICHAR=1
 51+    FEATURE_BC_ESCAPES=1
 52     FEATURE_TAR_TO_STDOUT=1;          FEATURE_TAR_KEEP_OLD=1
 53     FEATURE_TAR_STRIP_COMPONENTS=1;   FEATURE_TAR_FILES_FROM=1
 54     FEATURE_TAR_EXCLUDE_FROM=1;       FEATURE_IP_ROUTE_ADD_DEL=1
 55@@ -124,6 +134,11 @@ all)
 56     FEATURE_DEPMOD_ALIAS=1;           FEATURE_DEPMOD_SYMBOLS=1
 57     FEATURE_TAR_TTY_SAFE=1;           FEATURE_TAR_NOFOLLOW=1
 58     FEATURE_READLINK_REALPATH=1;      FEATURE_UMOUNT_OPTIONS=1
 59+    FEATURE_LN_RELATIVE=1
 60+    FEATURE_LN_NO_TARGET_DIR=1
 61+    FEATURE_BC_QUIET=1
 62+    FEATURE_BC_MULTICHAR=1
 63+    FEATURE_BC_ESCAPES=1
 64     FEATURE_TAR_TO_STDOUT=1;          FEATURE_TAR_KEEP_OLD=1
 65     FEATURE_TAR_STRIP_COMPONENTS=1;   FEATURE_TAR_FILES_FROM=1
 66     FEATURE_TAR_EXCLUDE_FROM=1;       FEATURE_IP_ROUTE_ADD_DEL=1
 67@@ -154,37 +169,37 @@ for _p in $PRESET_UTILS; do
 68 done
 69 unset _p
 70 
 71-# --- per-item overrides (after preset expansion) ---
 72-# BUILD_POSIX_SH=0       # disable a tool when its group is active
 73-# BUILD_DEV_AS=1         # enable a tool whose group isnt in PRESET_UTILS
 74+# per-item overrides (after preset expansion)
 75+# BUILD_POSIX_SH=0 # disable a tool when its group is active
 76+# BUILD_DEV_AS=1 # enable a tool whose group isnt in PRESET_UTILS
 77 FEATURE_SH_HISTEDIT=1
 78 
 79 # FEATURE_NOEXEC: when set to 1, wexec/wpopen/wsystem can execute registered
 80 # tool mains directly inside the forked child process (skipping execve)
 81-# This is equivalent to Busybox noexec applet support
 82-# This applies to ALL commands contained within aruu-box, you may also
 83+# this is equivalent to busybox noexec applet support
 84+# this applies to ALL commands contained within aruu-box, you may also
 85 # specify if some specific command is not an attack vector to also be nofork
 86 # via enabling FEATURE_NOFORK=1 and specifying those commands in NOFORK_LIST
 87 #
 88-#FEATURE_NOEXEC=1
 89+# FEATURE_NOEXEC=1
 90 #
 91 # FEATURE_NOEXEC_OVERRIDE: only meaningful when FEATURE_NOEXEC=1
 92 # when set to 1, ARUU_NOEXEC=0 in the environment forces fork+exec at runtime
 93 #
 94-#FEATURE_NOEXEC_OVERRIDE=1
 95+# FEATURE_NOEXEC_OVERRIDE=1
 96 #
 97 # FEATURE_NOFORK: when set to 1, wexecvp/wexecv can run simple commands
 98 # (like cat, echo, pwd, test) directly in the current process without spawning
 99-# a subprocess (noexec+nofork). This is equivalent to Busybox nofork applet support
100+# a subprocess (noexec+nofork). this is equivalent to busybox nofork applet support
101 #
102-#FEATURE_NOFORK=1
103+# FEATURE_NOFORK=1
104 #
105 # FEATURE_NOFORK_OVERRIDE: only meaningful when FEATURE_NOFORK=1
106 # when set to 1, ARUU_NOFORK=0 in the environment forces fork for nofork commands
107 # even tho they will still not do execvp() call at runtime
108 # unless FEATURE_NOEXEC_OVERRIDE was enabled at build time and ARUU_NOEXEC=0
109 #
110-#FEATURE_NOFORK_OVERRIDE=1
111+# FEATURE_NOFORK_OVERRIDE=1
112 #
113 # NOFORK_LIST: list of builtins that can run without fork (noexec+nofork)
114 #
115@@ -246,5 +261,5 @@ CPPFLAGS_TLS=$(echo "$CPPFLAGS_TLS" | xargs)
116 LDFLAGS_TLS=$(echo "$LDFLAGS_TLS" | xargs)
117 LDLIBS_TLS=$(echo "$LDLIBS_TLS" | xargs)
118 
119-# Makes Ninqu try them in this order:
120+# makes ninqu try them in this order:
121 NINJA_TRY="${NINJA_TRY:-samu,ninja}"

 1@@ -270,14 +270,14 @@ expand_rule(const char *name)
 2     int            r_literal = rule_is_literal(r);
 3 
 4     /* run a codegen producer before globbing only when its own
 5-     * output feeds this rules glob, so a step like POSIX_BC_C
 6-     * (produces cmd/posix/bc.c) finishes before glob_all() looks
 7-     * for cmd/posix/bc.c. anything else in extra_deps (a library to
 8-     * link, a header some other rule includes) never needs to run
 9-     * here, expand_rule() on it just registers its instances so
10-     * the resolve_input_token() pass in finalize_inst() can wire up
11-     * dep_rule_names for backend scheduling. skipped in
12-     * summary mode since -S must not execute */
13+ * output feeds this rules glob, so a step like POSIX_BC_C
14+ * (produces cmd/posix/bc.c) finishes before glob_all() looks
15+ * for cmd/posix/bc.c. anything else in extra_deps (a library to
16+ * link, a header some other rule includes) never needs to run
17+ * here, expand_rule() on it just registers its instances so
18+ * the resolve_input_token() pass in finalize_inst() can wire up
19+ * dep_rule_names for backend scheduling. skipped in
20+ * summary mode since -S must not execute */
21     if (!summary_mode) {
22       for (j = 0; j < r->extra_deps.n; j++) {
23         char *tok = kv_expand(r->extra_deps.v[j]);
24@@ -368,7 +368,7 @@ expand_rule(const char *name)
25     struct KvStore ov;
26 
27     /* non-glob rules have no MATCH/DIR/BASE to expose, so the
28-     * overlay only carries IN/OUT for build_cmd */
29+ * overlay only carries IN/OUT for build_cmd */
30     setup_inst(inst, r);
31     memset(&ov, 0, sizeof ov);
32     local_overlay = &ov;

1@@ -116,7 +116,7 @@ gate_eval(struct Gate *g)
2   return 0;
3 }
4 
5-/* expand every $(...) so a gate like build_$(BASESTEM) resolves
6+/* expand every $(...) so a gate like build_$(basestem) resolves
7  * against the active overlay before evaluation */
8 struct Gate *
9 gate_materialize(struct Gate *g)

1@@ -120,7 +120,7 @@ main(int argc, char *argv[])
2     return failed_any ? 1 : 0;
3   }
4 
5-  /* BACKEND=internal forces executor. otherwise generate file and exec backend binary */
6+  /* backend=internal forces executor. otherwise generate file and exec backend binary */
7   if (strcmp(kv_get_or("BACKEND", "auto"), "internal") != 0) {
8     char                 *bin;
9     const struct Backend *be = backend_resolve(&bin);

 1@@ -52,11 +52,11 @@ fprint_shell_word_plain(FILE *f, const char *s)
 2  * so neither lets a real newline survive inside the value. POSIX
 3  * single quotes cannot help either, there is no escape processing
 4  * inside them, so a backslash-n written between quotes stays two
 5- * literal characters, not a newline, once the shell reads it back.
 6+ * literal characters, not a newline, once the shell reads it back
 7  *
 8  * the fix is to not carry the newline in quoted text at all: emit
 9  * a command substitution that runs the word through `printf '%b'`,
10- * whose %b directive is POSIX-specified to expand backslash escapes
11+ * whose %b directive is posix-specified to expand backslash escapes
12  * (including \n) in its argument at runtime. the argument to printf
13  * is itself single-quoted using the same rules as the plain case,
14  * plus doubling any backslash so it survives as data rather than
15@@ -82,9 +82,9 @@ fprint_shell_word(FILE *f, const char *s)
16   }
17 
18   /* wrapped in double quotes so the shell treats the substitutions
19-   * result as one word instead of splitting it on whitespace, which
20-   * is exactly what an unquoted $(...) would do to a multi-line
21-   * result */
22+ * result as one word instead of splitting it on whitespace, which
23+ * is exactly what an unquoted $(...) would do to a multi-line
24+ * result */
25   fputs("\"$$(printf '%b' '", f);
26   for (; *s; s++) {
27     if (*s == '\'')
28@@ -220,17 +220,17 @@ emit_command(FILE *f, struct Inst *inst)
29  * inst_stale() never consults dep_inst for staleness either, only
30  * (in)/(stale_extra), so mirroring that as a plain (not order-only)
31  * prerequisite list is what makes ninjas own mtime check agree
32- * with ninqus.
33+ * with ninqus
34  *
35  * ninja only knows build outputs, never ninqus rule/group/meta
36- * names, so `ninja LIBUTIL` (what the Makefile forwards $@ as) has
37+ * names, so `ninja LIBUTIL` (what the makefile forwards $@ as) has
38  * nothing to resolve against on its own. wanted carries the literal
39  * command-line words ninqu itself was invoked with, one phony alias
40  * per word makes that name buildable too. when more than one target
41  * was requested in the same invocation every alias points at the
42  * whole combined closure rather than just its own slice, since
43  * insts[] no longer remembers which instance came from which wanted
44- * name, thats fine for the Makefiles own use (always one target
45+ * name, thats fine for the makefiles own use (always one target
46  * per invocation), just slightly conservative for a multi-target
47  * `ninqu -G ninja a b` run */
48 void
49@@ -292,8 +292,8 @@ emit_ninja(const char *path, struct StrList *wanted)
50         char         depbuf[1024];
51         const char  *depout = inst_target(dep, inst->dep_inst[k], depbuf, sizeof depbuf);
52         /* a rule ref that is also a real (dep ...) input already
53-         * forces both order and staleness, listing it again as
54-         * order-only is pure noise */
55+ * forces both order and staleness, listing it again as
56+ * order-only is pure noise */
57         if (sl_has(&inst->in, depout) || sl_has(&inst->stale_extra, depout))
58           continue;
59         if (!printed_bar) {

1@@ -95,7 +95,7 @@ void          lex_swallow_dollar(struct SLex *lx, char *buf, size_t *blen, size_
2 void          lex_string(struct SLex *lx, char *buf, size_t bsz);
3 const char   *find_dollar_close(const char *inner);
4 
5-/* gate: (not X) | (and X...) | (or X...) | bare var */
6+/* gate: (not x) | (and x...) | (or x...) | bare var */
7 enum GKind { G_TRUE, G_VAR, G_NOT, G_AND, G_OR };
8 
9 struct Gate {

 1@@ -27,7 +27,7 @@ sole_atom(struct SNode *node, const char *msg)
 2 }
 3 
 4 /* a dep operand is a bare atom, a (rule NAME) stored as @NAME, or
 5- * an (after X...) whose operands are order-only */
 6+ * an (after x...) whose operands are order-only */
 7 static void
 8 parse_dep_item(struct SNode *item, struct StrList *hard, struct StrList *ord, int allow_after)
 9 {
10@@ -158,10 +158,10 @@ process_rule_body(struct Rule *r, const char *name, struct SNode *form, int star
11     apply_rule_clause(r, name, form->kids[i]);
12 }
13 
14-/* (if EXPR clause...) splices its clauses in when EXPR is true.
15- * reuses the gate grammar so there is one condition language.
16+/* (if EXPR clause...) splices its clauses in when EXPR is true
17+ * reuses the gate grammar so there is one condition language
18  * a $(...) in the condition is rejected: splicing per glob match
19- * would mean deferring cmd/dep construction, which no rule needs.
20+ * would mean deferring cmd/dep construction, which no rule needs
21  * use (gate ...) on a separate rule for a per-instance condition */
22 static void
23 apply_if_clause(struct Rule *r, const char *name, struct SNode *clause)
24@@ -247,13 +247,13 @@ apply_rule_clause(struct Rule *r, const char *name, struct SNode *clause)
25     );
26   } else if (strcmp(h, "group") == 0) {
27     /* display-only: shows up in (query rules) output, no effect on
28-     * resolution. (member ...) below is the enforcing one. the two
29-     * are separate because several rules used (group ...) this way
30-     * before namespacing existed */
31+ * resolution. (member ...) below is the enforcing one. the two
32+ * are separate because several rules used (group ...) this way
33+ * before namespacing existed */
34     strlcpy(r->group_name, sole_atom(clause, "(group NAME) takes one name"), sizeof r->group_name);
35   } else if (strcmp(h, "member") == 0) {
36     /* (member GROUP [ALIAS]) hides the bare name from the command
37-     * line. ALIAS defaults to the rule own name */
38+ * line. ALIAS defaults to the rule own name */
39     if (clause->nkids < 2 || clause->nkids > 3 || clause->kids[1]->kind != S_ATOM)
40       eprintf("manifest: rule %s: (member GROUP [ALIAS]) needs a group name\n", name);
41     strlcpy(r->member_of, clause->kids[1]->atom, sizeof r->member_of);
42@@ -384,7 +384,7 @@ parse_set(struct SNode *form)
43   kv_set_manifest(form->kids[1]->atom, form->kids[2]->atom);
44 }
45 
46-/* (list) and (append) keep a space-joined string in the kv store.
47+/* (list) and (append) keep a space-joined string in the kv store
48  * a $(NAME) alone in a command splits back into argv entries */
49 static void
50 parse_list_or_append(struct SNode *form, int append)

 1@@ -101,8 +101,8 @@ resolve_member_path(const char *w)
 2   return r;
 3 }
 4 
 5-/* META.TARGET applies META overrides then resolves TARGET.
 6- * META alone applies META then falls back to (group default).
 7+/* META.TARGET applies META overrides then resolves TARGET
 8+ * META alone applies META then falls back to (group default)
 9  * anything else is a plain group or rule name */
10 void
11 resolve_wanted(const char *w)
12@@ -140,8 +140,8 @@ resolve_wanted(const char *w)
13   }
14 
15   /* a rule that declared (member ...) does not resolve by bare name
16-   * from the command line, only through GROUP/ALIAS or transitively
17-   * via its owning group ref list */
18+ * from the command line, only through GROUP/ALIAS or transitively
19+ * via its owning group ref list */
20   {
21     struct Rule *br = rule_find(w);
22     if (br && br->member_of[0])

   1@@ -1,4557 +1,12 @@
   2-/* See LICENSE file for copyright and license details. */
   3-
   4-#if defined(__x86_64__) && (defined(__clang__) || (defined(__GNUC__) && !defined(__TINYC__)))
   5-#define BLAKE3_X86_SIMD 1
   6-#else
   7-#define BLAKE3_X86_SIMD 0
   8-#endif
   9-
  10-#include "arg.h"
  11-#include "util.h"
  12-
  13-#include <assert.h>
  14-#if BLAKE3_X86_SIMD
  15-#include <immintrin.h>
  16-#endif
  17-#include <stdint.h>
  18-#include <stdio.h>
  19-#include <stdlib.h>
  20-#include <string.h>
  21-
  22-#define BLAKE3_VERSION_STRING "1.5.0"
  23-#define BLAKE3_KEY_LEN        32
  24-#define BLAKE3_OUT_LEN        32
  25-#define BLAKE3_BLOCK_LEN      64
  26-#define BLAKE3_CHUNK_LEN      1024
  27-#define BLAKE3_MAX_DEPTH      54
  28-
  29-#if BLAKE3_X86_SIMD
  30-#define MAX_SIMD_DEGREE 16
  31-#else
  32-#define MAX_SIMD_DEGREE 1
  33-#endif
  34-
  35-#define MAX_SIMD_DEGREE_OR_2 (MAX_SIMD_DEGREE > 2 ? MAX_SIMD_DEGREE : 2)
  36-
  37-enum Blake3Flags {
  38-  CHUNK_START         = 1 << 0,
  39-  CHUNK_END           = 1 << 1,
  40-  PARENT              = 1 << 2,
  41-  ROOT                = 1 << 3,
  42-  KEYED_HASH          = 1 << 4,
  43-  DERIVE_KEY_CONTEXT  = 1 << 5,
  44-  DERIVE_KEY_MATERIAL = 1 << 6
  45-};
  46-
  47-struct Blake3ChunkState {
  48-  uint32_t cv[8];
  49-  uint64_t chunk_counter;
  50-  uint8_t  buf[BLAKE3_BLOCK_LEN];
  51-  uint8_t  buf_len;
  52-  uint8_t  blocks_compressed;
  53-  uint8_t  flags;
  54-};
  55-
  56-struct Blake3Hasher {
  57-  uint32_t                key[8];
  58-  struct Blake3ChunkState chunk;
  59-  uint8_t                 cv_stack_len;
  60-  uint8_t                 cv_stack[(BLAKE3_MAX_DEPTH + 1) * BLAKE3_OUT_LEN];
  61-};
  62-
  63-void blake3_hasher_update(struct Blake3Hasher *self, const void *input, size_t input_len);
  64-void blake3_hasher_finalize(const struct Blake3Hasher *self, uint8_t *out, size_t out_len);
  65-
  66-struct Output {
  67-  uint32_t input_cv[8];
  68-  uint64_t counter;
  69-  uint8_t  block[BLAKE3_BLOCK_LEN];
  70-  uint8_t  block_len;
  71-  uint8_t  flags;
  72-};
  73-
  74-static const uint32_t IV[8] = {
  75-    0x6A09E667UL,
  76-    0xBB67AE85UL,
  77-    0x3C6EF372UL,
  78-    0xA54FF53AUL,
  79-    0x510E527FUL,
  80-    0x9B05688CUL,
  81-    0x1F83D9ABUL,
  82-    0x5BE0CD19UL
  83-};
  84-
  85-static const uint8_t MSG_SCHEDULE[7][16] = {
  86-    {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
  87-    {2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8},
  88-    {3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1},
  89-    {10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6},
  90-    {12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4},
  91-    {9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7},
  92-    {11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13},
  93-};
  94-
  95-static inline uint32_t
  96-load32(const void *src)
  97-{
  98-  const uint8_t *p = (const uint8_t *)src;
  99-
 100-  return ((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) | ((uint32_t)(p[2]) << 16)
 101-         | ((uint32_t)(p[3]) << 24);
 102-}
 103-
 104-static inline void
 105-store32(void *dst, uint32_t w)
 106-{
 107-  uint8_t *p = (uint8_t *)dst;
 108-
 109-  p[0] = (uint8_t)(w >> 0);
 110-  p[1] = (uint8_t)(w >> 8);
 111-  p[2] = (uint8_t)(w >> 16);
 112-  p[3] = (uint8_t)(w >> 24);
 113-}
 114-
 115-static inline void
 116-store_cv_words(uint8_t bytes_out[32], uint32_t cv_words[8])
 117-{
 118-  store32(&bytes_out[0 * 4], cv_words[0]);
 119-  store32(&bytes_out[1 * 4], cv_words[1]);
 120-  store32(&bytes_out[2 * 4], cv_words[2]);
 121-  store32(&bytes_out[3 * 4], cv_words[3]);
 122-  store32(&bytes_out[4 * 4], cv_words[4]);
 123-  store32(&bytes_out[5 * 4], cv_words[5]);
 124-  store32(&bytes_out[6 * 4], cv_words[6]);
 125-  store32(&bytes_out[7 * 4], cv_words[7]);
 126-}
 127-
 128-static inline uint32_t
 129-counter_low(uint64_t counter)
 130-{
 131-  return (uint32_t)counter;
 132-}
 133-
 134-static inline uint32_t
 135-counter_high(uint64_t counter)
 136-{
 137-  return (uint32_t)(counter >> 32);
 138-}
 139-
 140-/* forward declarations */
 141-#if BLAKE3_X86_SIMD
 142-void blake3_compress_in_place_sse2(
 143-    uint32_t      cv[8],
 144-    const uint8_t block[BLAKE3_BLOCK_LEN],
 145-    uint8_t       block_len,
 146-    uint64_t      counter,
 147-    uint8_t       flags
 148-);
 149-void blake3_compress_xof_sse2(
 150-    const uint32_t cv[8],
 151-    const uint8_t  block[BLAKE3_BLOCK_LEN],
 152-    uint8_t        block_len,
 153-    uint64_t       counter,
 154-    uint8_t        flags,
 155-    uint8_t        out[64]
 156-);
 157-void blake3_hash_many_sse2(
 158-    const uint8_t *const *inputs,
 159-    size_t                num_inputs,
 160-    size_t                blocks,
 161-    const uint32_t        key[8],
 162-    uint64_t              counter,
 163-    int                   increment_counter,
 164-    uint8_t               flags,
 165-    uint8_t               flags_start,
 166-    uint8_t               flags_end,
 167-    uint8_t              *out
 168-);
 169-
 170-void blake3_compress_in_place_sse41(
 171-    uint32_t      cv[8],
 172-    const uint8_t block[BLAKE3_BLOCK_LEN],
 173-    uint8_t       block_len,
 174-    uint64_t      counter,
 175-    uint8_t       flags
 176-);
 177-void blake3_compress_xof_sse41(
 178-    const uint32_t cv[8],
 179-    const uint8_t  block[BLAKE3_BLOCK_LEN],
 180-    uint8_t        block_len,
 181-    uint64_t       counter,
 182-    uint8_t        flags,
 183-    uint8_t        out[64]
 184-);
 185-void blake3_hash_many_sse41(
 186-    const uint8_t *const *inputs,
 187-    size_t                num_inputs,
 188-    size_t                blocks,
 189-    const uint32_t        key[8],
 190-    uint64_t              counter,
 191-    int                   increment_counter,
 192-    uint8_t               flags,
 193-    uint8_t               flags_start,
 194-    uint8_t               flags_end,
 195-    uint8_t              *out
 196-);
 197-
 198-void blake3_hash_many_avx2(
 199-    const uint8_t *const *inputs,
 200-    size_t                num_inputs,
 201-    size_t                blocks,
 202-    const uint32_t        key[8],
 203-    uint64_t              counter,
 204-    int                   increment_counter,
 205-    uint8_t               flags,
 206-    uint8_t               flags_start,
 207-    uint8_t               flags_end,
 208-    uint8_t              *out
 209-);
 210-
 211-void blake3_compress_in_place_avx512(
 212-    uint32_t      cv[8],
 213-    const uint8_t block[BLAKE3_BLOCK_LEN],
 214-    uint8_t       block_len,
 215-    uint64_t      counter,
 216-    uint8_t       flags
 217-);
 218-void blake3_compress_xof_avx512(
 219-    const uint32_t cv[8],
 220-    const uint8_t  block[BLAKE3_BLOCK_LEN],
 221-    uint8_t        block_len,
 222-    uint64_t       counter,
 223-    uint8_t        flags,
 224-    uint8_t        out[64]
 225-);
 226-void blake3_hash_many_avx512(
 227-    const uint8_t *const *inputs,
 228-    size_t                num_inputs,
 229-    size_t                blocks,
 230-    const uint32_t        key[8],
 231-    uint64_t              counter,
 232-    int                   increment_counter,
 233-    uint8_t               flags,
 234-    uint8_t               flags_start,
 235-    uint8_t               flags_end,
 236-    uint8_t              *out
 237-);
 238-#endif
 239-
 240-void blake3_compress_in_place_portable(
 241-    uint32_t      cv[8],
 242-    const uint8_t block[BLAKE3_BLOCK_LEN],
 243-    uint8_t       block_len,
 244-    uint64_t      counter,
 245-    uint8_t       flags
 246-);
 247-void blake3_compress_xof_portable(
 248-    const uint32_t cv[8],
 249-    const uint8_t  block[BLAKE3_BLOCK_LEN],
 250-    uint8_t        block_len,
 251-    uint64_t       counter,
 252-    uint8_t        flags,
 253-    uint8_t        out[64]
 254-);
 255-void blake3_hash_many_portable(
 256-    const uint8_t *const *inputs,
 257-    size_t                num_inputs,
 258-    size_t                blocks,
 259-    const uint32_t        key[8],
 260-    uint64_t              counter,
 261-    int                   increment_counter,
 262-    uint8_t               flags,
 263-    uint8_t               flags_start,
 264-    uint8_t               flags_end,
 265-    uint8_t              *out
 266-);
 267-
 268-void blake3_compress_in_place(
 269-    uint32_t      cv[8],
 270-    const uint8_t block[BLAKE3_BLOCK_LEN],
 271-    uint8_t       block_len,
 272-    uint64_t      counter,
 273-    uint8_t       flags
 274-);
 275-void blake3_compress_xof(
 276-    const uint32_t cv[8],
 277-    const uint8_t  block[BLAKE3_BLOCK_LEN],
 278-    uint8_t        block_len,
 279-    uint64_t       counter,
 280-    uint8_t        flags,
 281-    uint8_t        out[64]
 282-);
 283-void blake3_hash_many(
 284-    const uint8_t *const *inputs,
 285-    size_t                num_inputs,
 286-    size_t                blocks,
 287-    const uint32_t        key[8],
 288-    uint64_t              counter,
 289-    int                   increment_counter,
 290-    uint8_t               flags,
 291-    uint8_t               flags_start,
 292-    uint8_t               flags_end,
 293-    uint8_t              *out
 294-);
 295-
 296-/* portable implementations */
 297-static inline uint32_t
 298-rotr32(uint32_t w, uint32_t c)
 299-{
 300-  return (w >> c) | (w << (32 - c));
 301-}
 302-
 303-static inline void
 304-g_portable(uint32_t *state, size_t a, size_t b, size_t c, size_t d, uint32_t x, uint32_t y)
 305-{
 306-  state[a] = state[a] + state[b] + x;
 307-  state[d] = rotr32(state[d] ^ state[a], 16);
 308-  state[c] = state[c] + state[d];
 309-  state[b] = rotr32(state[b] ^ state[c], 12);
 310-  state[a] = state[a] + state[b] + y;
 311-  state[d] = rotr32(state[d] ^ state[a], 8);
 312-  state[c] = state[c] + state[d];
 313-  state[b] = rotr32(state[b] ^ state[c], 7);
 314-}
 315-
 316-static inline void
 317-round_fn_portable(uint32_t state[16], const uint32_t *msg, size_t round)
 318-{
 319-  const uint8_t *schedule = MSG_SCHEDULE[round];
 320-
 321-  g_portable(state, 0, 4, 8, 12, msg[schedule[0]], msg[schedule[1]]);
 322-  g_portable(state, 1, 5, 9, 13, msg[schedule[2]], msg[schedule[3]]);
 323-  g_portable(state, 2, 6, 10, 14, msg[schedule[4]], msg[schedule[5]]);
 324-  g_portable(state, 3, 7, 11, 15, msg[schedule[6]], msg[schedule[7]]);
 325-
 326-  g_portable(state, 0, 5, 10, 15, msg[schedule[8]], msg[schedule[9]]);
 327-  g_portable(state, 1, 6, 11, 12, msg[schedule[10]], msg[schedule[11]]);
 328-  g_portable(state, 2, 7, 8, 13, msg[schedule[12]], msg[schedule[13]]);
 329-  g_portable(state, 3, 4, 9, 14, msg[schedule[14]], msg[schedule[15]]);
 330-}
 331-
 332-static inline void
 333-compress_pre_portable(
 334-    uint32_t       state[16],
 335-    const uint32_t cv[8],
 336-    const uint8_t  block[BLAKE3_BLOCK_LEN],
 337-    uint8_t        block_len,
 338-    uint64_t       counter,
 339-    uint8_t        flags
 340-)
 341-{
 342-  uint32_t block_words[16];
 343-
 344-  block_words[0]  = load32(block + 4 * 0);
 345-  block_words[1]  = load32(block + 4 * 1);
 346-  block_words[2]  = load32(block + 4 * 2);
 347-  block_words[3]  = load32(block + 4 * 3);
 348-  block_words[4]  = load32(block + 4 * 4);
 349-  block_words[5]  = load32(block + 4 * 5);
 350-  block_words[6]  = load32(block + 4 * 6);
 351-  block_words[7]  = load32(block + 4 * 7);
 352-  block_words[8]  = load32(block + 4 * 8);
 353-  block_words[9]  = load32(block + 4 * 9);
 354-  block_words[10] = load32(block + 4 * 10);
 355-  block_words[11] = load32(block + 4 * 11);
 356-  block_words[12] = load32(block + 4 * 12);
 357-  block_words[13] = load32(block + 4 * 13);
 358-  block_words[14] = load32(block + 4 * 14);
 359-  block_words[15] = load32(block + 4 * 15);
 360-
 361-  state[0]  = cv[0];
 362-  state[1]  = cv[1];
 363-  state[2]  = cv[2];
 364-  state[3]  = cv[3];
 365-  state[4]  = cv[4];
 366-  state[5]  = cv[5];
 367-  state[6]  = cv[6];
 368-  state[7]  = cv[7];
 369-  state[8]  = IV[0];
 370-  state[9]  = IV[1];
 371-  state[10] = IV[2];
 372-  state[11] = IV[3];
 373-  state[12] = counter_low(counter);
 374-  state[13] = counter_high(counter);
 375-  state[14] = (uint32_t)block_len;
 376-  state[15] = (uint32_t)flags;
 377-
 378-  round_fn_portable(state, &block_words[0], 0);
 379-  round_fn_portable(state, &block_words[0], 1);
 380-  round_fn_portable(state, &block_words[0], 2);
 381-  round_fn_portable(state, &block_words[0], 3);
 382-  round_fn_portable(state, &block_words[0], 4);
 383-  round_fn_portable(state, &block_words[0], 5);
 384-  round_fn_portable(state, &block_words[0], 6);
 385-}
 386-
 387-void
 388-blake3_compress_in_place_portable(
 389-    uint32_t      cv[8],
 390-    const uint8_t block[BLAKE3_BLOCK_LEN],
 391-    uint8_t       block_len,
 392-    uint64_t      counter,
 393-    uint8_t       flags
 394-)
 395-{
 396-  uint32_t state[16];
 397-
 398-  compress_pre_portable(state, cv, block, block_len, counter, flags);
 399-  cv[0] = state[0] ^ state[8];
 400-  cv[1] = state[1] ^ state[9];
 401-  cv[2] = state[2] ^ state[10];
 402-  cv[3] = state[3] ^ state[11];
 403-  cv[4] = state[4] ^ state[12];
 404-  cv[5] = state[5] ^ state[13];
 405-  cv[6] = state[6] ^ state[14];
 406-  cv[7] = state[7] ^ state[15];
 407-}
 408-
 409-void
 410-blake3_compress_xof_portable(
 411-    const uint32_t cv[8],
 412-    const uint8_t  block[BLAKE3_BLOCK_LEN],
 413-    uint8_t        block_len,
 414-    uint64_t       counter,
 415-    uint8_t        flags,
 416-    uint8_t        out[64]
 417-)
 418-{
 419-  uint32_t state[16];
 420-
 421-  compress_pre_portable(state, cv, block, block_len, counter, flags);
 422-
 423-  store32(&out[0 * 4], state[0] ^ state[8]);
 424-  store32(&out[1 * 4], state[1] ^ state[9]);
 425-  store32(&out[2 * 4], state[2] ^ state[10]);
 426-  store32(&out[3 * 4], state[3] ^ state[11]);
 427-  store32(&out[4 * 4], state[4] ^ state[12]);
 428-  store32(&out[5 * 4], state[5] ^ state[13]);
 429-  store32(&out[6 * 4], state[6] ^ state[14]);
 430-  store32(&out[7 * 4], state[7] ^ state[15]);
 431-  store32(&out[8 * 4], state[8] ^ cv[0]);
 432-  store32(&out[9 * 4], state[9] ^ cv[1]);
 433-  store32(&out[10 * 4], state[10] ^ cv[2]);
 434-  store32(&out[11 * 4], state[11] ^ cv[3]);
 435-  store32(&out[12 * 4], state[12] ^ cv[4]);
 436-  store32(&out[13 * 4], state[13] ^ cv[5]);
 437-  store32(&out[14 * 4], state[14] ^ cv[6]);
 438-  store32(&out[15 * 4], state[15] ^ cv[7]);
 439-}
 440-
 441-static inline void
 442-hash_one_portable(
 443-    const uint8_t *input,
 444-    size_t         blocks,
 445-    const uint32_t key[8],
 446-    uint64_t       counter,
 447-    uint8_t        flags,
 448-    uint8_t        flags_start,
 449-    uint8_t        flags_end,
 450-    uint8_t        out[BLAKE3_OUT_LEN]
 451-)
 452-{
 453-  uint32_t cv[8];
 454-  uint8_t  block_flags;
 455-
 456-  memcpy(cv, key, BLAKE3_KEY_LEN);
 457-  block_flags = flags | flags_start;
 458-  while (blocks > 0) {
 459-    if (blocks == 1) {
 460-      block_flags |= flags_end;
 461-    }
 462-    blake3_compress_in_place_portable(cv, input, BLAKE3_BLOCK_LEN, counter, block_flags);
 463-    input = &input[BLAKE3_BLOCK_LEN];
 464-    blocks -= 1;
 465-    block_flags = flags;
 466-  }
 467-  store_cv_words(out, cv);
 468-}
 469-
 470-void
 471-blake3_hash_many_portable(
 472-    const uint8_t *const *inputs,
 473-    size_t                num_inputs,
 474-    size_t                blocks,
 475-    const uint32_t        key[8],
 476-    uint64_t              counter,
 477-    int                   increment_counter,
 478-    uint8_t               flags,
 479-    uint8_t               flags_start,
 480-    uint8_t               flags_end,
 481-    uint8_t              *out
 482-)
 483-{
 484-  while (num_inputs > 0) {
 485-    hash_one_portable(inputs[0], blocks, key, counter, flags, flags_start, flags_end, out);
 486-    if (increment_counter) {
 487-      counter += 1;
 488-    }
 489-    inputs += 1;
 490-    num_inputs -= 1;
 491-    out = &out[BLAKE3_OUT_LEN];
 492-  }
 493-}
 494-
 495-/* cpu features detection */
 496-enum { SSE2 = 1 << 0, SSE41 = 1 << 1, AVX2 = 1 << 2, AVX512 = 1 << 3 };
 497-
 498-static int blake3_cpu_features = 0;
 499-static int blake3_cpu_detected = 0;
 500-
 501-#if BLAKE3_X86_SIMD
 502-#include <cpuid.h>
 503-
 504-static void
 505-blake3_cpuid(uint32_t out[4], uint32_t id, uint32_t sid)
 506-{
 507-  __cpuid_count(id, sid, out[0], out[1], out[2], out[3]);
 508-}
 509-
 510-static uint64_t
 511-blake3_xgetbv(void)
 512-{
 513-  uint32_t eax, edx;
 514-
 515-  __asm__ volatile("xgetbv" : "=a"(eax), "=d"(edx) : "c"(0));
 516-  return ((uint64_t)edx << 32) | eax;
 517-}
 518-#endif
 519-
 520-static void
 521-blake3_detect_cpu_features(void)
 522-{
 523-#if BLAKE3_X86_SIMD
 524-  enum { EAX, EBX, ECX, EDX };
 525-  uint32_t regs[4];
 526-  uint64_t xcr0;
 527-  int      features = 0;
 528-
 529-  blake3_cpuid(regs, 1, 0);
 530-  if (regs[EDX] & (1UL << 26))
 531-    features |= SSE2;
 532-  if (regs[ECX] & (1UL << 19))
 533-    features |= SSE41;
 534-  /* osxsave */
 535-  if (regs[ECX] & (1UL << 27)) {
 536-    blake3_cpuid(regs, 0, 0);
 537-    if (regs[EAX] >= 7) {
 538-      blake3_cpuid(regs, 7, 0);
 539-      xcr0 = blake3_xgetbv();
 540-      /* avx2 and xcr0 sse, avx */
 541-      if ((regs[EBX] & (1UL << 5)) && (xcr0 & 0x06) == 0x06)
 542-        features |= AVX2;
 543-      /* avx512f, avx512vl and xcr0 opmask, zmm_hi256,
 544-       * hi16_zmm */
 545-      if ((regs[EBX] & (1UL << 31 | 1UL << 16)) && (xcr0 & 0xe0) == 0xe0)
 546-        features |= AVX512;
 547-    }
 548-  }
 549-  blake3_cpu_features = features;
 550-#endif
 551-  blake3_cpu_detected = 1;
 552-}
 553-
 554-#if BLAKE3_X86_SIMD
 555-__attribute__((constructor)) static void
 556-blake3_init_cpu(void)
 557-{
 558-  if (!blake3_cpu_detected)
 559-    blake3_detect_cpu_features();
 560-}
 561-#endif
 562-
 563-#if BLAKE3_X86_SIMD
 564-#if defined(__clang__)
 565-#pragma clang attribute push(__attribute__((target("sse2"))), apply_to = function)
 566-#elif defined(__GNUC__)
 567-#pragma GCC push_options
 568-#pragma GCC target("sse2")
 569-#endif
 570-#define DEGREE_SSE2 4
 571-
 572-#define _mm_shuffle_ps2(a, b, c)                                                                   \
 573-  (_mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c))))
 574-
 575-static inline __m128i
 576-loadu_sse2(const uint8_t src[16])
 577-{
 578-  return _mm_loadu_si128((const __m128i *)src);
 579-}
 580-
 581-static inline void
 582-storeu_sse2(__m128i src, uint8_t dest[16])
 583-{
 584-  _mm_storeu_si128((__m128i *)dest, src);
 585-}
 586-
 587-static inline __m128i
 588-addv_sse2(__m128i a, __m128i b)
 589-{
 590-  return _mm_add_epi32(a, b);
 591-}
 592-
 593-static inline __m128i
 594-xorv_sse2(__m128i a, __m128i b)
 595-{
 596-  return _mm_xor_si128(a, b);
 597-}
 598-
 599-static inline __m128i
 600-set1_sse2(uint32_t x)
 601-{
 602-  return _mm_set1_epi32((int32_t)x);
 603-}
 604-
 605-static inline __m128i
 606-set4_sse2(uint32_t a, uint32_t b, uint32_t c, uint32_t d)
 607-{
 608-  return _mm_setr_epi32((int32_t)a, (int32_t)b, (int32_t)c, (int32_t)d);
 609-}
 610-
 611-static inline __m128i
 612-rot16_sse2(__m128i x)
 613-{
 614-  return _mm_shufflehi_epi16(_mm_shufflelo_epi16(x, 0xB1), 0xB1);
 615-}
 616-
 617-static inline __m128i
 618-rot12_sse2(__m128i x)
 619-{
 620-  return xorv_sse2(_mm_srli_epi32(x, 12), _mm_slli_epi32(x, 32 - 12));
 621-}
 622-
 623-static inline __m128i
 624-rot8_sse2(__m128i x)
 625-{
 626-  return xorv_sse2(_mm_srli_epi32(x, 8), _mm_slli_epi32(x, 32 - 8));
 627-}
 628-
 629-static inline __m128i
 630-rot7_sse2(__m128i x)
 631-{
 632-  return xorv_sse2(_mm_srli_epi32(x, 7), _mm_slli_epi32(x, 32 - 7));
 633-}
 634-
 635-static inline void
 636-g1_sse2(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, __m128i m)
 637-{
 638-  *row0 = addv_sse2(addv_sse2(*row0, m), *row1);
 639-  *row3 = xorv_sse2(*row3, *row0);
 640-  *row3 = rot16_sse2(*row3);
 641-  *row2 = addv_sse2(*row2, *row3);
 642-  *row1 = xorv_sse2(*row1, *row2);
 643-  *row1 = rot12_sse2(*row1);
 644-}
 645-
 646-static inline void
 647-g2_sse2(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, __m128i m)
 648-{
 649-  *row0 = addv_sse2(addv_sse2(*row0, m), *row1);
 650-  *row3 = xorv_sse2(*row3, *row0);
 651-  *row3 = rot8_sse2(*row3);
 652-  *row2 = addv_sse2(*row2, *row3);
 653-  *row1 = xorv_sse2(*row1, *row2);
 654-  *row1 = rot7_sse2(*row1);
 655-}
 656-
 657-static inline void
 658-diagonalize_sse2(__m128i *row0, __m128i *row2, __m128i *row3)
 659-{
 660-  *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3));
 661-  *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2));
 662-  *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1));
 663-}
 664-
 665-static inline void
 666-undiagonalize_sse2(__m128i *row0, __m128i *row2, __m128i *row3)
 667-{
 668-  *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1));
 669-  *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2));
 670-  *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3));
 671-}
 672-
 673-static inline __m128i
 674-blend_epi16_sse2(__m128i a, __m128i b, const int16_t imm8)
 675-{
 676-  const __m128i bits = _mm_set_epi16(0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01);
 677-  __m128i       mask = _mm_set1_epi16(imm8);
 678-
 679-  mask = _mm_and_si128(mask, bits);
 680-  mask = _mm_cmpeq_epi16(mask, bits);
 681-  return _mm_or_si128(_mm_and_si128(mask, b), _mm_andnot_si128(mask, a));
 682-}
 683-
 684-static inline void
 685-compress_pre_sse2(
 686-    __m128i        rows[4],
 687-    const uint32_t cv[8],
 688-    const uint8_t  block[BLAKE3_BLOCK_LEN],
 689-    uint8_t        block_len,
 690-    uint64_t       counter,
 691-    uint8_t        flags
 692-)
 693-{
 694-  __m128i m0, m1, m2, m3;
 695-  __m128i t0, t1, t2, t3, tt;
 696-
 697-  rows[0] = loadu_sse2((uint8_t *)&cv[0]);
 698-  rows[1] = loadu_sse2((uint8_t *)&cv[4]);
 699-  rows[2] = set4_sse2(IV[0], IV[1], IV[2], IV[3]);
 700-  rows[3] =
 701-      set4_sse2(counter_low(counter), counter_high(counter), (uint32_t)block_len, (uint32_t)flags);
 702-
 703-  m0 = loadu_sse2(&block[sizeof(__m128i) * 0]);
 704-  m1 = loadu_sse2(&block[sizeof(__m128i) * 1]);
 705-  m2 = loadu_sse2(&block[sizeof(__m128i) * 2]);
 706-  m3 = loadu_sse2(&block[sizeof(__m128i) * 3]);
 707-
 708-  /* round 1 */
 709-  t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(2, 0, 2, 0));
 710-  g1_sse2(&rows[0], &rows[1], &rows[2], &rows[3], t0);
 711-  t1 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 3, 1));
 712-  g2_sse2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
 713-  diagonalize_sse2(&rows[0], &rows[2], &rows[3]);
 714-  t2 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(2, 0, 2, 0));
 715-  t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3));
 716-  g1_sse2(&rows[0], &rows[1], &rows[2], &rows[3], t2);
 717-  t3 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 1, 3, 1));
 718-  t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3));
 719-  g2_sse2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
 720-  undiagonalize_sse2(&rows[0], &rows[2], &rows[3]);
 721-  m0 = t0;
 722-  m1 = t1;
 723-  m2 = t2;
 724-  m3 = t3;
 725-
 726-  /* round 2 */
 727-  t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
 728-  t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
 729-  g1_sse2(&rows[0], &rows[1], &rows[2], &rows[3], t0);
 730-  t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
 731-  tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
 732-  t1 = blend_epi16_sse2(tt, t1, 0xCC);
 733-  g2_sse2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
 734-  diagonalize_sse2(&rows[0], &rows[2], &rows[3]);
 735-  t2 = _mm_unpacklo_epi64(m3, m1);
 736-  tt = blend_epi16_sse2(t2, m2, 0xC0);
 737-  t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
 738-  g1_sse2(&rows[0], &rows[1], &rows[2], &rows[3], t2);
 739-  t3 = _mm_unpackhi_epi32(m1, m3);
 740-  tt = _mm_unpacklo_epi32(m2, t3);
 741-  t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
 742-  g2_sse2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
 743-  undiagonalize_sse2(&rows[0], &rows[2], &rows[3]);
 744-  m0 = t0;
 745-  m1 = t1;
 746-  m2 = t2;
 747-  m3 = t3;
 748-
 749-  /* round 3 */
 750-  t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
 751-  t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
 752-  g1_sse2(&rows[0], &rows[1], &rows[2], &rows[3], t0);
 753-  t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
 754-  tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
 755-  t1 = blend_epi16_sse2(tt, t1, 0xCC);
 756-  g2_sse2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
 757-  diagonalize_sse2(&rows[0], &rows[2], &rows[3]);
 758-  t2 = _mm_unpacklo_epi64(m3, m1);
 759-  tt = blend_epi16_sse2(t2, m2, 0xC0);
 760-  t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
 761-  g1_sse2(&rows[0], &rows[1], &rows[2], &rows[3], t2);
 762-  t3 = _mm_unpackhi_epi32(m1, m3);
 763-  tt = _mm_unpacklo_epi32(m2, t3);
 764-  t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
 765-  g2_sse2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
 766-  undiagonalize_sse2(&rows[0], &rows[2], &rows[3]);
 767-  m0 = t0;
 768-  m1 = t1;
 769-  m2 = t2;
 770-  m3 = t3;
 771-
 772-  /* round 4 */
 773-  t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
 774-  t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
 775-  g1_sse2(&rows[0], &rows[1], &rows[2], &rows[3], t0);
 776-  t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
 777-  tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
 778-  t1 = blend_epi16_sse2(tt, t1, 0xCC);
 779-  g2_sse2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
 780-  diagonalize_sse2(&rows[0], &rows[2], &rows[3]);
 781-  t2 = _mm_unpacklo_epi64(m3, m1);
 782-  tt = blend_epi16_sse2(t2, m2, 0xC0);
 783-  t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
 784-  g1_sse2(&rows[0], &rows[1], &rows[2], &rows[3], t2);
 785-  t3 = _mm_unpackhi_epi32(m1, m3);
 786-  tt = _mm_unpacklo_epi32(m2, t3);
 787-  t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
 788-  g2_sse2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
 789-  undiagonalize_sse2(&rows[0], &rows[2], &rows[3]);
 790-  m0 = t0;
 791-  m1 = t1;
 792-  m2 = t2;
 793-  m3 = t3;
 794-
 795-  /* round 5 */
 796-  t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
 797-  t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
 798-  g1_sse2(&rows[0], &rows[1], &rows[2], &rows[3], t0);
 799-  t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
 800-  tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
 801-  t1 = blend_epi16_sse2(tt, t1, 0xCC);
 802-  g2_sse2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
 803-  diagonalize_sse2(&rows[0], &rows[2], &rows[3]);
 804-  t2 = _mm_unpacklo_epi64(m3, m1);
 805-  tt = blend_epi16_sse2(t2, m2, 0xC0);
 806-  t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
 807-  g1_sse2(&rows[0], &rows[1], &rows[2], &rows[3], t2);
 808-  t3 = _mm_unpackhi_epi32(m1, m3);
 809-  tt = _mm_unpacklo_epi32(m2, t3);
 810-  t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
 811-  g2_sse2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
 812-  undiagonalize_sse2(&rows[0], &rows[2], &rows[3]);
 813-  m0 = t0;
 814-  m1 = t1;
 815-  m2 = t2;
 816-  m3 = t3;
 817-
 818-  /* round 6 */
 819-  t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
 820-  t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
 821-  g1_sse2(&rows[0], &rows[1], &rows[2], &rows[3], t0);
 822-  t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
 823-  tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
 824-  t1 = blend_epi16_sse2(tt, t1, 0xCC);
 825-  g2_sse2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
 826-  diagonalize_sse2(&rows[0], &rows[2], &rows[3]);
 827-  t2 = _mm_unpacklo_epi64(m3, m1);
 828-  tt = blend_epi16_sse2(t2, m2, 0xC0);
 829-  t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
 830-  g1_sse2(&rows[0], &rows[1], &rows[2], &rows[3], t2);
 831-  t3 = _mm_unpackhi_epi32(m1, m3);
 832-  tt = _mm_unpacklo_epi32(m2, t3);
 833-  t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
 834-  g2_sse2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
 835-  undiagonalize_sse2(&rows[0], &rows[2], &rows[3]);
 836-  m0 = t0;
 837-  m1 = t1;
 838-  m2 = t2;
 839-  m3 = t3;
 840-
 841-  /* round 7 */
 842-  t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
 843-  t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
 844-  g1_sse2(&rows[0], &rows[1], &rows[2], &rows[3], t0);
 845-  t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
 846-  tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
 847-  t1 = blend_epi16_sse2(tt, t1, 0xCC);
 848-  g2_sse2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
 849-  diagonalize_sse2(&rows[0], &rows[2], &rows[3]);
 850-  t2 = _mm_unpacklo_epi64(m3, m1);
 851-  tt = blend_epi16_sse2(t2, m2, 0xC0);
 852-  t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
 853-  g1_sse2(&rows[0], &rows[1], &rows[2], &rows[3], t2);
 854-  t3 = _mm_unpackhi_epi32(m1, m3);
 855-  tt = _mm_unpacklo_epi32(m2, t3);
 856-  t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
 857-  g2_sse2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
 858-  undiagonalize_sse2(&rows[0], &rows[2], &rows[3]);
 859-}
 860-
 861-void
 862-blake3_compress_in_place_sse2(
 863-    uint32_t      cv[8],
 864-    const uint8_t block[BLAKE3_BLOCK_LEN],
 865-    uint8_t       block_len,
 866-    uint64_t      counter,
 867-    uint8_t       flags
 868-)
 869-{
 870-  __m128i rows[4];
 871-
 872-  compress_pre_sse2(rows, cv, block, block_len, counter, flags);
 873-  storeu_sse2(xorv_sse2(rows[0], rows[2]), (uint8_t *)&cv[0]);
 874-  storeu_sse2(xorv_sse2(rows[1], rows[3]), (uint8_t *)&cv[4]);
 875-}
 876-
 877-void
 878-blake3_compress_xof_sse2(
 879-    const uint32_t cv[8],
 880-    const uint8_t  block[BLAKE3_BLOCK_LEN],
 881-    uint8_t        block_len,
 882-    uint64_t       counter,
 883-    uint8_t        flags,
 884-    uint8_t        out_buf[64]
 885-)
 886-{
 887-  __m128i rows[4];
 888-
 889-  compress_pre_sse2(rows, cv, block, block_len, counter, flags);
 890-  storeu_sse2(xorv_sse2(rows[0], rows[2]), &out_buf[0]);
 891-  storeu_sse2(xorv_sse2(rows[1], rows[3]), &out_buf[16]);
 892-  storeu_sse2(xorv_sse2(rows[2], loadu_sse2((uint8_t *)&cv[0])), &out_buf[32]);
 893-  storeu_sse2(xorv_sse2(rows[3], loadu_sse2((uint8_t *)&cv[4])), &out_buf[48]);
 894-}
 895-
 896-static inline void
 897-round_fn_sse2(__m128i v[16], __m128i m[16], size_t r)
 898-{
 899-  v[0]  = addv_sse2(v[0], m[(size_t)MSG_SCHEDULE[r][0]]);
 900-  v[1]  = addv_sse2(v[1], m[(size_t)MSG_SCHEDULE[r][2]]);
 901-  v[2]  = addv_sse2(v[2], m[(size_t)MSG_SCHEDULE[r][4]]);
 902-  v[3]  = addv_sse2(v[3], m[(size_t)MSG_SCHEDULE[r][6]]);
 903-  v[0]  = addv_sse2(v[0], v[4]);
 904-  v[1]  = addv_sse2(v[1], v[5]);
 905-  v[2]  = addv_sse2(v[2], v[6]);
 906-  v[3]  = addv_sse2(v[3], v[7]);
 907-  v[12] = xorv_sse2(v[12], v[0]);
 908-  v[13] = xorv_sse2(v[13], v[1]);
 909-  v[14] = xorv_sse2(v[14], v[2]);
 910-  v[15] = xorv_sse2(v[15], v[3]);
 911-  v[12] = rot16_sse2(v[12]);
 912-  v[13] = rot16_sse2(v[13]);
 913-  v[14] = rot16_sse2(v[14]);
 914-  v[15] = rot16_sse2(v[15]);
 915-  v[8]  = addv_sse2(v[8], v[12]);
 916-  v[9]  = addv_sse2(v[9], v[13]);
 917-  v[10] = addv_sse2(v[10], v[14]);
 918-  v[11] = addv_sse2(v[11], v[15]);
 919-  v[4]  = xorv_sse2(v[4], v[8]);
 920-  v[5]  = xorv_sse2(v[5], v[9]);
 921-  v[6]  = xorv_sse2(v[6], v[10]);
 922-  v[7]  = xorv_sse2(v[7], v[11]);
 923-  v[4]  = rot12_sse2(v[4]);
 924-  v[5]  = rot12_sse2(v[5]);
 925-  v[6]  = rot12_sse2(v[6]);
 926-  v[7]  = rot12_sse2(v[7]);
 927-  v[0]  = addv_sse2(v[0], m[(size_t)MSG_SCHEDULE[r][1]]);
 928-  v[1]  = addv_sse2(v[1], m[(size_t)MSG_SCHEDULE[r][3]]);
 929-  v[2]  = addv_sse2(v[2], m[(size_t)MSG_SCHEDULE[r][5]]);
 930-  v[3]  = addv_sse2(v[3], m[(size_t)MSG_SCHEDULE[r][7]]);
 931-  v[0]  = addv_sse2(v[0], v[4]);
 932-  v[1]  = addv_sse2(v[1], v[5]);
 933-  v[2]  = addv_sse2(v[2], v[6]);
 934-  v[3]  = addv_sse2(v[3], v[7]);
 935-  v[12] = xorv_sse2(v[12], v[0]);
 936-  v[13] = xorv_sse2(v[13], v[1]);
 937-  v[14] = xorv_sse2(v[14], v[2]);
 938-  v[15] = xorv_sse2(v[15], v[3]);
 939-  v[12] = rot8_sse2(v[12]);
 940-  v[13] = rot8_sse2(v[13]);
 941-  v[14] = rot8_sse2(v[14]);
 942-  v[15] = rot8_sse2(v[15]);
 943-  v[8]  = addv_sse2(v[8], v[12]);
 944-  v[9]  = addv_sse2(v[9], v[13]);
 945-  v[10] = addv_sse2(v[10], v[14]);
 946-  v[11] = addv_sse2(v[11], v[15]);
 947-  v[4]  = xorv_sse2(v[4], v[8]);
 948-  v[5]  = xorv_sse2(v[5], v[9]);
 949-  v[6]  = xorv_sse2(v[6], v[10]);
 950-  v[7]  = xorv_sse2(v[7], v[11]);
 951-  v[4]  = rot7_sse2(v[4]);
 952-  v[5]  = rot7_sse2(v[5]);
 953-  v[6]  = rot7_sse2(v[6]);
 954-  v[7]  = rot7_sse2(v[7]);
 955-
 956-  v[0]  = addv_sse2(v[0], m[(size_t)MSG_SCHEDULE[r][8]]);
 957-  v[1]  = addv_sse2(v[1], m[(size_t)MSG_SCHEDULE[r][10]]);
 958-  v[2]  = addv_sse2(v[2], m[(size_t)MSG_SCHEDULE[r][12]]);
 959-  v[3]  = addv_sse2(v[3], m[(size_t)MSG_SCHEDULE[r][14]]);
 960-  v[0]  = addv_sse2(v[0], v[5]);
 961-  v[1]  = addv_sse2(v[1], v[6]);
 962-  v[2]  = addv_sse2(v[2], v[7]);
 963-  v[3]  = addv_sse2(v[3], v[4]);
 964-  v[15] = xorv_sse2(v[15], v[0]);
 965-  v[12] = xorv_sse2(v[12], v[1]);
 966-  v[13] = xorv_sse2(v[13], v[2]);
 967-  v[14] = xorv_sse2(v[14], v[3]);
 968-  v[15] = rot16_sse2(v[15]);
 969-  v[12] = rot16_sse2(v[12]);
 970-  v[13] = rot16_sse2(v[13]);
 971-  v[14] = rot16_sse2(v[14]);
 972-  v[10] = addv_sse2(v[10], v[15]);
 973-  v[11] = addv_sse2(v[11], v[12]);
 974-  v[8]  = addv_sse2(v[8], v[13]);
 975-  v[9]  = addv_sse2(v[9], v[14]);
 976-  v[5]  = xorv_sse2(v[5], v[10]);
 977-  v[6]  = xorv_sse2(v[6], v[11]);
 978-  v[7]  = xorv_sse2(v[7], v[8]);
 979-  v[4]  = xorv_sse2(v[4], v[9]);
 980-  v[5]  = rot12_sse2(v[5]);
 981-  v[6]  = rot12_sse2(v[6]);
 982-  v[7]  = rot12_sse2(v[7]);
 983-  v[4]  = rot12_sse2(v[4]);
 984-  v[0]  = addv_sse2(v[0], m[(size_t)MSG_SCHEDULE[r][9]]);
 985-  v[1]  = addv_sse2(v[1], m[(size_t)MSG_SCHEDULE[r][11]]);
 986-  v[2]  = addv_sse2(v[2], m[(size_t)MSG_SCHEDULE[r][13]]);
 987-  v[3]  = addv_sse2(v[3], m[(size_t)MSG_SCHEDULE[r][15]]);
 988-  v[0]  = addv_sse2(v[0], v[5]);
 989-  v[1]  = addv_sse2(v[1], v[6]);
 990-  v[2]  = addv_sse2(v[2], v[7]);
 991-  v[3]  = addv_sse2(v[3], v[4]);
 992-  v[15] = xorv_sse2(v[15], v[0]);
 993-  v[12] = xorv_sse2(v[12], v[1]);
 994-  v[13] = xorv_sse2(v[13], v[2]);
 995-  v[14] = xorv_sse2(v[14], v[3]);
 996-  v[15] = rot8_sse2(v[15]);
 997-  v[12] = rot8_sse2(v[12]);
 998-  v[13] = rot8_sse2(v[13]);
 999-  v[14] = rot8_sse2(v[14]);
1000-  v[10] = addv_sse2(v[10], v[15]);
1001-  v[11] = addv_sse2(v[11], v[12]);
1002-  v[8]  = addv_sse2(v[8], v[13]);
1003-  v[9]  = addv_sse2(v[9], v[14]);
1004-  v[5]  = xorv_sse2(v[5], v[10]);
1005-  v[6]  = xorv_sse2(v[6], v[11]);
1006-  v[7]  = xorv_sse2(v[7], v[8]);
1007-  v[4]  = xorv_sse2(v[4], v[9]);
1008-  v[5]  = rot7_sse2(v[5]);
1009-  v[6]  = rot7_sse2(v[6]);
1010-  v[7]  = rot7_sse2(v[7]);
1011-  v[4]  = rot7_sse2(v[4]);
1012-}
1013-
1014-static inline void
1015-transpose_vecs_sse2(__m128i vecs[DEGREE_SSE2])
1016-{
1017-  __m128i ab_01 = _mm_unpacklo_epi32(vecs[0], vecs[1]);
1018-  __m128i ab_23 = _mm_unpackhi_epi32(vecs[0], vecs[1]);
1019-  __m128i cd_01 = _mm_unpacklo_epi32(vecs[2], vecs[3]);
1020-  __m128i cd_23 = _mm_unpackhi_epi32(vecs[2], vecs[3]);
1021-
1022-  __m128i abcd_0 = _mm_unpacklo_epi64(ab_01, cd_01);
1023-  __m128i abcd_1 = _mm_unpackhi_epi64(ab_01, cd_01);
1024-  __m128i abcd_2 = _mm_unpacklo_epi64(ab_23, cd_23);
1025-  __m128i abcd_3 = _mm_unpackhi_epi64(ab_23, cd_23);
1026-
1027-  vecs[0] = abcd_0;
1028-  vecs[1] = abcd_1;
1029-  vecs[2] = abcd_2;
1030-  vecs[3] = abcd_3;
1031-}
1032-
1033-static inline void
1034-transpose_msg_vecs_sse2(const uint8_t *const *inputs, size_t block_offset, __m128i out_msg[16])
1035-{
1036-  size_t i;
1037-
1038-  out_msg[0]  = loadu_sse2(&inputs[0][block_offset + 0 * sizeof(__m128i)]);
1039-  out_msg[1]  = loadu_sse2(&inputs[1][block_offset + 0 * sizeof(__m128i)]);
1040-  out_msg[2]  = loadu_sse2(&inputs[2][block_offset + 0 * sizeof(__m128i)]);
1041-  out_msg[3]  = loadu_sse2(&inputs[3][block_offset + 0 * sizeof(__m128i)]);
1042-  out_msg[4]  = loadu_sse2(&inputs[0][block_offset + 1 * sizeof(__m128i)]);
1043-  out_msg[5]  = loadu_sse2(&inputs[1][block_offset + 1 * sizeof(__m128i)]);
1044-  out_msg[6]  = loadu_sse2(&inputs[2][block_offset + 1 * sizeof(__m128i)]);
1045-  out_msg[7]  = loadu_sse2(&inputs[3][block_offset + 1 * sizeof(__m128i)]);
1046-  out_msg[8]  = loadu_sse2(&inputs[0][block_offset + 2 * sizeof(__m128i)]);
1047-  out_msg[9]  = loadu_sse2(&inputs[1][block_offset + 2 * sizeof(__m128i)]);
1048-  out_msg[10] = loadu_sse2(&inputs[2][block_offset + 2 * sizeof(__m128i)]);
1049-  out_msg[11] = loadu_sse2(&inputs[3][block_offset + 2 * sizeof(__m128i)]);
1050-  out_msg[12] = loadu_sse2(&inputs[0][block_offset + 3 * sizeof(__m128i)]);
1051-  out_msg[13] = loadu_sse2(&inputs[1][block_offset + 3 * sizeof(__m128i)]);
1052-  out_msg[14] = loadu_sse2(&inputs[2][block_offset + 3 * sizeof(__m128i)]);
1053-  out_msg[15] = loadu_sse2(&inputs[3][block_offset + 3 * sizeof(__m128i)]);
1054-
1055-  for (i = 0; i < 4; i++) {
1056-    _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0);
1057-  }
1058-  transpose_vecs_sse2(&out_msg[0]);
1059-  transpose_vecs_sse2(&out_msg[4]);
1060-  transpose_vecs_sse2(&out_msg[8]);
1061-  transpose_vecs_sse2(&out_msg[12]);
1062-}
1063-
1064-static inline void
1065-load_counters_sse2(uint64_t counter, int increment_counter, __m128i *out_lo, __m128i *out_hi)
1066-{
1067-  const __m128i mask  = _mm_set1_epi32(-increment_counter);
1068-  const __m128i add0  = _mm_set_epi32(3, 2, 1, 0);
1069-  const __m128i add1  = _mm_and_si128(mask, add0);
1070-  __m128i       l     = _mm_add_epi32(_mm_set1_epi32((int32_t)counter), add1);
1071-  __m128i       carry = _mm_cmpgt_epi32(
1072-      _mm_xor_si128(add1, _mm_set1_epi32(0x80000000)), _mm_xor_si128(l, _mm_set1_epi32(0x80000000))
1073-  );
1074-  __m128i h = _mm_sub_epi32(_mm_set1_epi32((int32_t)(counter >> 32)), carry);
1075-
1076-  *out_lo = l;
1077-  *out_hi = h;
1078-}
1079-
1080-void
1081-blake3_hash4_sse2(
1082-    const uint8_t *const *inputs,
1083-    size_t                blocks,
1084-    const uint32_t        key[8],
1085-    uint64_t              counter,
1086-    int                   increment_counter,
1087-    uint8_t               flags,
1088-    uint8_t               flags_start,
1089-    uint8_t               flags_end,
1090-    uint8_t              *out_bytes
1091-)
1092-{
1093-  __m128i h_vecs[8];
1094-  __m128i counter_low_vec, counter_high_vec;
1095-  uint8_t block_flags;
1096-  size_t  block;
1097-
1098-  h_vecs[0] = set1_sse2(key[0]);
1099-  h_vecs[1] = set1_sse2(key[1]);
1100-  h_vecs[2] = set1_sse2(key[2]);
1101-  h_vecs[3] = set1_sse2(key[3]);
1102-  h_vecs[4] = set1_sse2(key[4]);
1103-  h_vecs[5] = set1_sse2(key[5]);
1104-  h_vecs[6] = set1_sse2(key[6]);
1105-  h_vecs[7] = set1_sse2(key[7]);
1106-
1107-  load_counters_sse2(counter, increment_counter, &counter_low_vec, &counter_high_vec);
1108-  block_flags = flags | flags_start;
1109-
1110-  for (block = 0; block < blocks; block++) {
1111-    __m128i block_len_vec;
1112-    __m128i block_flags_vec;
1113-    __m128i msg_vecs[16];
1114-    __m128i v[16];
1115-
1116-    if (block + 1 == blocks) {
1117-      block_flags |= flags_end;
1118-    }
1119-    block_len_vec   = set1_sse2(BLAKE3_BLOCK_LEN);
1120-    block_flags_vec = set1_sse2(block_flags);
1121-    transpose_msg_vecs_sse2(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs);
1122-
1123-    v[0]  = h_vecs[0];
1124-    v[1]  = h_vecs[1];
1125-    v[2]  = h_vecs[2];
1126-    v[3]  = h_vecs[3];
1127-    v[4]  = h_vecs[4];
1128-    v[5]  = h_vecs[5];
1129-    v[6]  = h_vecs[6];
1130-    v[7]  = h_vecs[7];
1131-    v[8]  = set1_sse2(IV[0]);
1132-    v[9]  = set1_sse2(IV[1]);
1133-    v[10] = set1_sse2(IV[2]);
1134-    v[11] = set1_sse2(IV[3]);
1135-    v[12] = counter_low_vec;
1136-    v[13] = counter_high_vec;
1137-    v[14] = block_len_vec;
1138-    v[15] = block_flags_vec;
1139-
1140-    round_fn_sse2(v, msg_vecs, 0);
1141-    round_fn_sse2(v, msg_vecs, 1);
1142-    round_fn_sse2(v, msg_vecs, 2);
1143-    round_fn_sse2(v, msg_vecs, 3);
1144-    round_fn_sse2(v, msg_vecs, 4);
1145-    round_fn_sse2(v, msg_vecs, 5);
1146-    round_fn_sse2(v, msg_vecs, 6);
1147-
1148-    h_vecs[0] = xorv_sse2(v[0], v[8]);
1149-    h_vecs[1] = xorv_sse2(v[1], v[9]);
1150-    h_vecs[2] = xorv_sse2(v[2], v[10]);
1151-    h_vecs[3] = xorv_sse2(v[3], v[11]);
1152-    h_vecs[4] = xorv_sse2(v[4], v[12]);
1153-    h_vecs[5] = xorv_sse2(v[5], v[13]);
1154-    h_vecs[6] = xorv_sse2(v[6], v[14]);
1155-    h_vecs[7] = xorv_sse2(v[7], v[15]);
1156-
1157-    block_flags = flags;
1158-  }
1159-
1160-  transpose_vecs_sse2(&h_vecs[0]);
1161-  transpose_vecs_sse2(&h_vecs[4]);
1162-  storeu_sse2(h_vecs[0], &out_bytes[0 * sizeof(__m128i)]);
1163-  storeu_sse2(h_vecs[4], &out_bytes[1 * sizeof(__m128i)]);
1164-  storeu_sse2(h_vecs[1], &out_bytes[2 * sizeof(__m128i)]);
1165-  storeu_sse2(h_vecs[5], &out_bytes[3 * sizeof(__m128i)]);
1166-  storeu_sse2(h_vecs[2], &out_bytes[4 * sizeof(__m128i)]);
1167-  storeu_sse2(h_vecs[6], &out_bytes[5 * sizeof(__m128i)]);
1168-  storeu_sse2(h_vecs[3], &out_bytes[6 * sizeof(__m128i)]);
1169-  storeu_sse2(h_vecs[7], &out_bytes[7 * sizeof(__m128i)]);
1170-}
1171-
1172-static inline void
1173-hash_one_sse2(
1174-    const uint8_t *input,
1175-    size_t         blocks,
1176-    const uint32_t key[8],
1177-    uint64_t       counter,
1178-    uint8_t        flags,
1179-    uint8_t        flags_start,
1180-    uint8_t        flags_end,
1181-    uint8_t        out_bytes[BLAKE3_OUT_LEN]
1182-)
1183-{
1184-  uint32_t cv[8];
1185-  uint8_t  block_flags;
1186-
1187-  memcpy(cv, key, BLAKE3_KEY_LEN);
1188-  block_flags = flags | flags_start;
1189-  while (blocks > 0) {
1190-    if (blocks == 1) {
1191-      block_flags |= flags_end;
1192-    }
1193-    blake3_compress_in_place_sse2(cv, input, BLAKE3_BLOCK_LEN, counter, block_flags);
1194-    input = &input[BLAKE3_BLOCK_LEN];
1195-    blocks -= 1;
1196-    block_flags = flags;
1197-  }
1198-  memcpy(out_bytes, cv, BLAKE3_OUT_LEN);
1199-}
1200-
1201-void
1202-blake3_hash_many_sse2(
1203-    const uint8_t *const *inputs,
1204-    size_t                num_inputs,
1205-    size_t                blocks,
1206-    const uint32_t        key[8],
1207-    uint64_t              counter,
1208-    int                   increment_counter,
1209-    uint8_t               flags,
1210-    uint8_t               flags_start,
1211-    uint8_t               flags_end,
1212-    uint8_t              *out_bytes
1213-)
1214-{
1215-  while (num_inputs >= DEGREE_SSE2) {
1216-    blake3_hash4_sse2(
1217-        inputs, blocks, key, counter, increment_counter, flags, flags_start, flags_end, out_bytes
1218-    );
1219-    if (increment_counter) {
1220-      counter += DEGREE_SSE2;
1221-    }
1222-    inputs += DEGREE_SSE2;
1223-    num_inputs -= DEGREE_SSE2;
1224-    out_bytes = &out_bytes[DEGREE_SSE2 * BLAKE3_OUT_LEN];
1225-  }
1226-  while (num_inputs > 0) {
1227-    hash_one_sse2(inputs[0], blocks, key, counter, flags, flags_start, flags_end, out_bytes);
1228-    if (increment_counter) {
1229-      counter += 1;
1230-    }
1231-    inputs += 1;
1232-    num_inputs -= 1;
1233-    out_bytes = &out_bytes[BLAKE3_OUT_LEN];
1234-  }
1235-}
1236-#if defined(__clang__)
1237-#pragma clang attribute pop
1238-#elif defined(__GNUC__)
1239-#pragma GCC pop_options
1240-#endif
1241-
1242-#if defined(__clang__)
1243-#pragma clang attribute push(__attribute__((target("sse4.1"))), apply_to = function)
1244-#elif defined(__GNUC__)
1245-#pragma GCC push_options
1246-#pragma GCC target("sse4.1")
1247-#endif
1248-#define DEGREE_SSE41 4
1249-
1250-#define _mm_shuffle_ps2_sse41(a, b, c)                                                             \
1251-  (_mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c))))
1252-
1253-static inline __m128i
1254-loadu_sse41(const uint8_t src[16])
1255-{
1256-  return _mm_loadu_si128((const __m128i *)src);
1257-}
1258-
1259-static inline void
1260-storeu_sse41(__m128i src, uint8_t dest[16])
1261-{
1262-  _mm_storeu_si128((__m128i *)dest, src);
1263-}
1264-
1265-static inline __m128i
1266-addv_sse41(__m128i a, __m128i b)
1267-{
1268-  return _mm_add_epi32(a, b);
1269-}
1270-
1271-static inline __m128i
1272-xorv_sse41(__m128i a, __m128i b)
1273-{
1274-  return _mm_xor_si128(a, b);
1275-}
1276-
1277-static inline __m128i
1278-set1_sse41(uint32_t x)
1279-{
1280-  return _mm_set1_epi32((int32_t)x);
1281-}
1282-
1283-static inline __m128i
1284-set4_sse41(uint32_t a, uint32_t b, uint32_t c, uint32_t d)
1285-{
1286-  return _mm_setr_epi32((int32_t)a, (int32_t)b, (int32_t)c, (int32_t)d);
1287-}
1288-
1289-static inline __m128i
1290-rot16_sse41(__m128i x)
1291-{
1292-  return _mm_shuffle_epi8(x, _mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2));
1293-}
1294-
1295-static inline __m128i
1296-rot12_sse41(__m128i x)
1297-{
1298-  return xorv_sse41(_mm_srli_epi32(x, 12), _mm_slli_epi32(x, 32 - 12));
1299-}
1300-
1301-static inline __m128i
1302-rot8_sse41(__m128i x)
1303-{
1304-  return _mm_shuffle_epi8(x, _mm_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1));
1305-}
1306-
1307-static inline __m128i
1308-rot7_sse41(__m128i x)
1309-{
1310-  return xorv_sse41(_mm_srli_epi32(x, 7), _mm_slli_epi32(x, 32 - 7));
1311-}
1312-
1313-static inline void
1314-g1_sse41(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, __m128i m)
1315-{
1316-  *row0 = addv_sse41(addv_sse41(*row0, m), *row1);
1317-  *row3 = xorv_sse41(*row3, *row0);
1318-  *row3 = rot16_sse41(*row3);
1319-  *row2 = addv_sse41(*row2, *row3);
1320-  *row1 = xorv_sse41(*row1, *row2);
1321-  *row1 = rot12_sse41(*row1);
1322-}
1323-
1324-static inline void
1325-g2_sse41(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, __m128i m)
1326-{
1327-  *row0 = addv_sse41(addv_sse41(*row0, m), *row1);
1328-  *row3 = xorv_sse41(*row3, *row0);
1329-  *row3 = rot8_sse41(*row3);
1330-  *row2 = addv_sse41(*row2, *row3);
1331-  *row1 = xorv_sse41(*row1, *row2);
1332-  *row1 = rot7_sse41(*row1);
1333-}
1334-
1335-static inline void
1336-diagonalize_sse41(__m128i *row0, __m128i *row2, __m128i *row3)
1337-{
1338-  *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3));
1339-  *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2));
1340-  *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1));
1341-}
1342-
1343-static inline void
1344-undiagonalize_sse41(__m128i *row0, __m128i *row2, __m128i *row3)
1345-{
1346-  *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1));
1347-  *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2));
1348-  *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3));
1349-}
1350-
1351-static inline void
1352-compress_pre_sse41(
1353-    __m128i        rows[4],
1354-    const uint32_t cv[8],
1355-    const uint8_t  block[BLAKE3_BLOCK_LEN],
1356-    uint8_t        block_len,
1357-    uint64_t       counter,
1358-    uint8_t        flags
1359-)
1360-{
1361-  __m128i m0, m1, m2, m3;
1362-  __m128i t0, t1, t2, t3, tt;
1363-
1364-  rows[0] = loadu_sse41((uint8_t *)&cv[0]);
1365-  rows[1] = loadu_sse41((uint8_t *)&cv[4]);
1366-  rows[2] = set4_sse41(IV[0], IV[1], IV[2], IV[3]);
1367-  rows[3] =
1368-      set4_sse41(counter_low(counter), counter_high(counter), (uint32_t)block_len, (uint32_t)flags);
1369-
1370-  m0 = loadu_sse41(&block[sizeof(__m128i) * 0]);
1371-  m1 = loadu_sse41(&block[sizeof(__m128i) * 1]);
1372-  m2 = loadu_sse41(&block[sizeof(__m128i) * 2]);
1373-  m3 = loadu_sse41(&block[sizeof(__m128i) * 3]);
1374-
1375-  /* round 1 */
1376-  t0 = _mm_shuffle_ps2_sse41(m0, m1, _MM_SHUFFLE(2, 0, 2, 0));
1377-  g1_sse41(&rows[0], &rows[1], &rows[2], &rows[3], t0);
1378-  t1 = _mm_shuffle_ps2_sse41(m0, m1, _MM_SHUFFLE(3, 1, 3, 1));
1379-  g2_sse41(&rows[0], &rows[1], &rows[2], &rows[3], t1);
1380-  diagonalize_sse41(&rows[0], &rows[2], &rows[3]);
1381-  t2 = _mm_shuffle_ps2_sse41(m2, m3, _MM_SHUFFLE(2, 0, 2, 0));
1382-  t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3));
1383-  g1_sse41(&rows[0], &rows[1], &rows[2], &rows[3], t2);
1384-  t3 = _mm_shuffle_ps2_sse41(m2, m3, _MM_SHUFFLE(3, 1, 3, 1));
1385-  t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3));
1386-  g2_sse41(&rows[0], &rows[1], &rows[2], &rows[3], t3);
1387-  undiagonalize_sse41(&rows[0], &rows[2], &rows[3]);
1388-  m0 = t0;
1389-  m1 = t1;
1390-  m2 = t2;
1391-  m3 = t3;
1392-
1393-  /* round 2 */
1394-  t0 = _mm_shuffle_ps2_sse41(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
1395-  t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
1396-  g1_sse41(&rows[0], &rows[1], &rows[2], &rows[3], t0);
1397-  t1 = _mm_shuffle_ps2_sse41(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
1398-  tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
1399-  t1 = _mm_blend_epi16(tt, t1, 0xCC);
1400-  g2_sse41(&rows[0], &rows[1], &rows[2], &rows[3], t1);
1401-  diagonalize_sse41(&rows[0], &rows[2], &rows[3]);
1402-  t2 = _mm_unpacklo_epi64(m3, m1);
1403-  tt = _mm_blend_epi16(t2, m2, 0xC0);
1404-  t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
1405-  g1_sse41(&rows[0], &rows[1], &rows[2], &rows[3], t2);
1406-  t3 = _mm_unpackhi_epi32(m1, m3);
1407-  tt = _mm_unpacklo_epi32(m2, t3);
1408-  t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
1409-  g2_sse41(&rows[0], &rows[1], &rows[2], &rows[3], t3);
1410-  undiagonalize_sse41(&rows[0], &rows[2], &rows[3]);
1411-  m0 = t0;
1412-  m1 = t1;
1413-  m2 = t2;
1414-  m3 = t3;
1415-
1416-  /* round 3 */
1417-  t0 = _mm_shuffle_ps2_sse41(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
1418-  t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
1419-  g1_sse41(&rows[0], &rows[1], &rows[2], &rows[3], t0);
1420-  t1 = _mm_shuffle_ps2_sse41(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
1421-  tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
1422-  t1 = _mm_blend_epi16(tt, t1, 0xCC);
1423-  g2_sse41(&rows[0], &rows[1], &rows[2], &rows[3], t1);
1424-  diagonalize_sse41(&rows[0], &rows[2], &rows[3]);
1425-  t2 = _mm_unpacklo_epi64(m3, m1);
1426-  tt = _mm_blend_epi16(t2, m2, 0xC0);
1427-  t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
1428-  g1_sse41(&rows[0], &rows[1], &rows[2], &rows[3], t2);
1429-  t3 = _mm_unpackhi_epi32(m1, m3);
1430-  tt = _mm_unpacklo_epi32(m2, t3);
1431-  t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
1432-  g2_sse41(&rows[0], &rows[1], &rows[2], &rows[3], t3);
1433-  undiagonalize_sse41(&rows[0], &rows[2], &rows[3]);
1434-  m0 = t0;
1435-  m1 = t1;
1436-  m2 = t2;
1437-  m3 = t3;
1438-
1439-  /* round 4 */
1440-  t0 = _mm_shuffle_ps2_sse41(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
1441-  t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
1442-  g1_sse41(&rows[0], &rows[1], &rows[2], &rows[3], t0);
1443-  t1 = _mm_shuffle_ps2_sse41(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
1444-  tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
1445-  t1 = _mm_blend_epi16(tt, t1, 0xCC);
1446-  g2_sse41(&rows[0], &rows[1], &rows[2], &rows[3], t1);
1447-  diagonalize_sse41(&rows[0], &rows[2], &rows[3]);
1448-  t2 = _mm_unpacklo_epi64(m3, m1);
1449-  tt = _mm_blend_epi16(t2, m2, 0xC0);
1450-  t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
1451-  g1_sse41(&rows[0], &rows[1], &rows[2], &rows[3], t2);
1452-  t3 = _mm_unpackhi_epi32(m1, m3);
1453-  tt = _mm_unpacklo_epi32(m2, t3);
1454-  t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
1455-  g2_sse41(&rows[0], &rows[1], &rows[2], &rows[3], t3);
1456-  undiagonalize_sse41(&rows[0], &rows[2], &rows[3]);
1457-  m0 = t0;
1458-  m1 = t1;
1459-  m2 = t2;
1460-  m3 = t3;
1461-
1462-  /* round 5 */
1463-  t0 = _mm_shuffle_ps2_sse41(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
1464-  t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
1465-  g1_sse41(&rows[0], &rows[1], &rows[2], &rows[3], t0);
1466-  t1 = _mm_shuffle_ps2_sse41(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
1467-  tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
1468-  t1 = _mm_blend_epi16(tt, t1, 0xCC);
1469-  g2_sse41(&rows[0], &rows[1], &rows[2], &rows[3], t1);
1470-  diagonalize_sse41(&rows[0], &rows[2], &rows[3]);
1471-  t2 = _mm_unpacklo_epi64(m3, m1);
1472-  tt = _mm_blend_epi16(t2, m2, 0xC0);
1473-  t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
1474-  g1_sse41(&rows[0], &rows[1], &rows[2], &rows[3], t2);
1475-  t3 = _mm_unpackhi_epi32(m1, m3);
1476-  tt = _mm_unpacklo_epi32(m2, t3);
1477-  t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
1478-  g2_sse41(&rows[0], &rows[1], &rows[2], &rows[3], t3);
1479-  undiagonalize_sse41(&rows[0], &rows[2], &rows[3]);
1480-  m0 = t0;
1481-  m1 = t1;
1482-  m2 = t2;
1483-  m3 = t3;
1484-
1485-  /* round 6 */
1486-  t0 = _mm_shuffle_ps2_sse41(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
1487-  t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
1488-  g1_sse41(&rows[0], &rows[1], &rows[2], &rows[3], t0);
1489-  t1 = _mm_shuffle_ps2_sse41(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
1490-  tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
1491-  t1 = _mm_blend_epi16(tt, t1, 0xCC);
1492-  g2_sse41(&rows[0], &rows[1], &rows[2], &rows[3], t1);
1493-  diagonalize_sse41(&rows[0], &rows[2], &rows[3]);
1494-  t2 = _mm_unpacklo_epi64(m3, m1);
1495-  tt = _mm_blend_epi16(t2, m2, 0xC0);
1496-  t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
1497-  g1_sse41(&rows[0], &rows[1], &rows[2], &rows[3], t2);
1498-  t3 = _mm_unpackhi_epi32(m1, m3);
1499-  tt = _mm_unpacklo_epi32(m2, t3);
1500-  t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
1501-  g2_sse41(&rows[0], &rows[1], &rows[2], &rows[3], t3);
1502-  undiagonalize_sse41(&rows[0], &rows[2], &rows[3]);
1503-  m0 = t0;
1504-  m1 = t1;
1505-  m2 = t2;
1506-  m3 = t3;
1507-
1508-  /* round 7 */
1509-  t0 = _mm_shuffle_ps2_sse41(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
1510-  t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
1511-  g1_sse41(&rows[0], &rows[1], &rows[2], &rows[3], t0);
1512-  t1 = _mm_shuffle_ps2_sse41(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
1513-  tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
1514-  t1 = _mm_blend_epi16(tt, t1, 0xCC);
1515-  g2_sse41(&rows[0], &rows[1], &rows[2], &rows[3], t1);
1516-  diagonalize_sse41(&rows[0], &rows[2], &rows[3]);
1517-  t2 = _mm_unpacklo_epi64(m3, m1);
1518-  tt = _mm_blend_epi16(t2, m2, 0xC0);
1519-  t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
1520-  g1_sse41(&rows[0], &rows[1], &rows[2], &rows[3], t2);
1521-  t3 = _mm_unpackhi_epi32(m1, m3);
1522-  tt = _mm_unpacklo_epi32(m2, t3);
1523-  t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
1524-  g2_sse41(&rows[0], &rows[1], &rows[2], &rows[3], t3);
1525-  undiagonalize_sse41(&rows[0], &rows[2], &rows[3]);
1526-}
1527-
1528-void
1529-blake3_compress_in_place_sse41(
1530-    uint32_t      cv[8],
1531-    const uint8_t block[BLAKE3_BLOCK_LEN],
1532-    uint8_t       block_len,
1533-    uint64_t      counter,
1534-    uint8_t       flags
1535-)
1536-{
1537-  __m128i rows[4];
1538-
1539-  compress_pre_sse41(rows, cv, block, block_len, counter, flags);
1540-  storeu_sse41(xorv_sse41(rows[0], rows[2]), (uint8_t *)&cv[0]);
1541-  storeu_sse41(xorv_sse41(rows[1], rows[3]), (uint8_t *)&cv[4]);
1542-}
1543-
1544-void
1545-blake3_compress_xof_sse41(
1546-    const uint32_t cv[8],
1547-    const uint8_t  block[BLAKE3_BLOCK_LEN],
1548-    uint8_t        block_len,
1549-    uint64_t       counter,
1550-    uint8_t        flags,
1551-    uint8_t        out_buf[64]
1552-)
1553-{
1554-  __m128i rows[4];
1555-
1556-  compress_pre_sse41(rows, cv, block, block_len, counter, flags);
1557-  storeu_sse41(xorv_sse41(rows[0], rows[2]), &out_buf[0]);
1558-  storeu_sse41(xorv_sse41(rows[1], rows[3]), &out_buf[16]);
1559-  storeu_sse41(xorv_sse41(rows[2], loadu_sse41((uint8_t *)&cv[0])), &out_buf[32]);
1560-  storeu_sse41(xorv_sse41(rows[3], loadu_sse41((uint8_t *)&cv[4])), &out_buf[48]);
1561-}
1562-
1563-static inline void
1564-round_fn_sse41(__m128i v[16], __m128i m[16], size_t r)
1565-{
1566-  v[0]  = addv_sse41(v[0], m[(size_t)MSG_SCHEDULE[r][0]]);
1567-  v[1]  = addv_sse41(v[1], m[(size_t)MSG_SCHEDULE[r][2]]);
1568-  v[2]  = addv_sse41(v[2], m[(size_t)MSG_SCHEDULE[r][4]]);
1569-  v[3]  = addv_sse41(v[3], m[(size_t)MSG_SCHEDULE[r][6]]);
1570-  v[0]  = addv_sse41(v[0], v[4]);
1571-  v[1]  = addv_sse41(v[1], v[5]);
1572-  v[2]  = addv_sse41(v[2], v[6]);
1573-  v[3]  = addv_sse41(v[3], v[7]);
1574-  v[12] = xorv_sse41(v[12], v[0]);
1575-  v[13] = xorv_sse41(v[13], v[1]);
1576-  v[14] = xorv_sse41(v[14], v[2]);
1577-  v[15] = xorv_sse41(v[15], v[3]);
1578-  v[12] = rot16_sse41(v[12]);
1579-  v[13] = rot16_sse41(v[13]);
1580-  v[14] = rot16_sse41(v[14]);
1581-  v[15] = rot16_sse41(v[15]);
1582-  v[8]  = addv_sse41(v[8], v[12]);
1583-  v[9]  = addv_sse41(v[9], v[13]);
1584-  v[10] = addv_sse41(v[10], v[14]);
1585-  v[11] = addv_sse41(v[11], v[15]);
1586-  v[4]  = xorv_sse41(v[4], v[8]);
1587-  v[5]  = xorv_sse41(v[5], v[9]);
1588-  v[6]  = xorv_sse41(v[6], v[10]);
1589-  v[7]  = xorv_sse41(v[7], v[11]);
1590-  v[4]  = rot12_sse41(v[4]);
1591-  v[5]  = rot12_sse41(v[5]);
1592-  v[6]  = rot12_sse41(v[6]);
1593-  v[7]  = rot12_sse41(v[7]);
1594-  v[0]  = addv_sse41(v[0], m[(size_t)MSG_SCHEDULE[r][1]]);
1595-  v[1]  = addv_sse41(v[1], m[(size_t)MSG_SCHEDULE[r][3]]);
1596-  v[2]  = addv_sse41(v[2], m[(size_t)MSG_SCHEDULE[r][5]]);
1597-  v[3]  = addv_sse41(v[3], m[(size_t)MSG_SCHEDULE[r][7]]);
1598-  v[0]  = addv_sse41(v[0], v[4]);
1599-  v[1]  = addv_sse41(v[1], v[5]);
1600-  v[2]  = addv_sse41(v[2], v[6]);
1601-  v[3]  = addv_sse41(v[3], v[7]);
1602-  v[12] = xorv_sse41(v[12], v[0]);
1603-  v[13] = xorv_sse41(v[13], v[1]);
1604-  v[14] = xorv_sse41(v[14], v[2]);
1605-  v[15] = xorv_sse41(v[15], v[3]);
1606-  v[12] = rot8_sse41(v[12]);
1607-  v[13] = rot8_sse41(v[13]);
1608-  v[14] = rot8_sse41(v[14]);
1609-  v[15] = rot8_sse41(v[15]);
1610-  v[8]  = addv_sse41(v[8], v[12]);
1611-  v[9]  = addv_sse41(v[9], v[13]);
1612-  v[10] = addv_sse41(v[10], v[14]);
1613-  v[11] = addv_sse41(v[11], v[15]);
1614-  v[4]  = xorv_sse41(v[4], v[8]);
1615-  v[5]  = xorv_sse41(v[5], v[9]);
1616-  v[6]  = xorv_sse41(v[6], v[10]);
1617-  v[7]  = xorv_sse41(v[7], v[11]);
1618-  v[4]  = rot7_sse41(v[4]);
1619-  v[5]  = rot7_sse41(v[5]);
1620-  v[6]  = rot7_sse41(v[6]);
1621-  v[7]  = rot7_sse41(v[7]);
1622-
1623-  v[0]  = addv_sse41(v[0], m[(size_t)MSG_SCHEDULE[r][8]]);
1624-  v[1]  = addv_sse41(v[1], m[(size_t)MSG_SCHEDULE[r][10]]);
1625-  v[2]  = addv_sse41(v[2], m[(size_t)MSG_SCHEDULE[r][12]]);
1626-  v[3]  = addv_sse41(v[3], m[(size_t)MSG_SCHEDULE[r][14]]);
1627-  v[0]  = addv_sse41(v[0], v[5]);
1628-  v[1]  = addv_sse41(v[1], v[6]);
1629-  v[2]  = addv_sse41(v[2], v[7]);
1630-  v[3]  = addv_sse41(v[3], v[4]);
1631-  v[15] = xorv_sse41(v[15], v[0]);
1632-  v[12] = xorv_sse41(v[12], v[1]);
1633-  v[13] = xorv_sse41(v[13], v[2]);
1634-  v[14] = xorv_sse41(v[14], v[3]);
1635-  v[15] = rot16_sse41(v[15]);
1636-  v[12] = rot16_sse41(v[12]);
1637-  v[13] = rot16_sse41(v[13]);
1638-  v[14] = rot16_sse41(v[14]);
1639-  v[10] = addv_sse41(v[10], v[15]);
1640-  v[11] = addv_sse41(v[11], v[12]);
1641-  v[8]  = addv_sse41(v[8], v[13]);
1642-  v[9]  = addv_sse41(v[9], v[14]);
1643-  v[5]  = xorv_sse41(v[5], v[10]);
1644-  v[6]  = xorv_sse41(v[6], v[11]);
1645-  v[7]  = xorv_sse41(v[7], v[8]);
1646-  v[4]  = xorv_sse41(v[4], v[9]);
1647-  v[5]  = rot12_sse41(v[5]);
1648-  v[6]  = rot12_sse41(v[6]);
1649-  v[7]  = rot12_sse41(v[7]);
1650-  v[4]  = rot12_sse41(v[4]);
1651-  v[0]  = addv_sse41(v[0], m[(size_t)MSG_SCHEDULE[r][9]]);
1652-  v[1]  = addv_sse41(v[1], m[(size_t)MSG_SCHEDULE[r][11]]);
1653-  v[2]  = addv_sse41(v[2], m[(size_t)MSG_SCHEDULE[r][13]]);
1654-  v[3]  = addv_sse41(v[3], m[(size_t)MSG_SCHEDULE[r][15]]);
1655-  v[0]  = addv_sse41(v[0], v[5]);
1656-  v[1]  = addv_sse41(v[1], v[6]);
1657-  v[2]  = addv_sse41(v[2], v[7]);
1658-  v[3]  = addv_sse41(v[3], v[4]);
1659-  v[15] = xorv_sse41(v[15], v[0]);
1660-  v[12] = xorv_sse41(v[12], v[1]);
1661-  v[13] = xorv_sse41(v[13], v[2]);
1662-  v[14] = xorv_sse41(v[14], v[3]);
1663-  v[15] = rot8_sse41(v[15]);
1664-  v[12] = rot8_sse41(v[12]);
1665-  v[13] = rot8_sse41(v[13]);
1666-  v[14] = rot8_sse41(v[14]);
1667-  v[10] = addv_sse41(v[10], v[15]);
1668-  v[11] = addv_sse41(v[11], v[12]);
1669-  v[8]  = addv_sse41(v[8], v[13]);
1670-  v[9]  = addv_sse41(v[9], v[14]);
1671-  v[5]  = xorv_sse41(v[5], v[10]);
1672-  v[6]  = xorv_sse41(v[6], v[11]);
1673-  v[7]  = xorv_sse41(v[7], v[8]);
1674-  v[4]  = xorv_sse41(v[4], v[9]);
1675-  v[5]  = rot7_sse41(v[5]);
1676-  v[6]  = rot7_sse41(v[6]);
1677-  v[7]  = rot7_sse41(v[7]);
1678-  v[4]  = rot7_sse41(v[4]);
1679-}
1680-
1681-static inline void
1682-transpose_vecs_sse41(__m128i vecs[DEGREE_SSE41])
1683-{
1684-  __m128i ab_01 = _mm_unpacklo_epi32(vecs[0], vecs[1]);
1685-  __m128i ab_23 = _mm_unpackhi_epi32(vecs[0], vecs[1]);
1686-  __m128i cd_01 = _mm_unpacklo_epi32(vecs[2], vecs[3]);
1687-  __m128i cd_23 = _mm_unpackhi_epi32(vecs[2], vecs[3]);
1688-
1689-  __m128i abcd_0 = _mm_unpacklo_epi64(ab_01, cd_01);
1690-  __m128i abcd_1 = _mm_unpackhi_epi64(ab_01, cd_01);
1691-  __m128i abcd_2 = _mm_unpacklo_epi64(ab_23, cd_23);
1692-  __m128i abcd_3 = _mm_unpackhi_epi64(ab_23, cd_23);
1693-
1694-  vecs[0] = abcd_0;
1695-  vecs[1] = abcd_1;
1696-  vecs[2] = abcd_2;
1697-  vecs[3] = abcd_3;
1698-}
1699-
1700-static inline void
1701-transpose_msg_vecs_sse41(const uint8_t *const *inputs, size_t block_offset, __m128i out_msg[16])
1702-{
1703-  size_t i;
1704-
1705-  out_msg[0]  = loadu_sse41(&inputs[0][block_offset + 0 * sizeof(__m128i)]);
1706-  out_msg[1]  = loadu_sse41(&inputs[1][block_offset + 0 * sizeof(__m128i)]);
1707-  out_msg[2]  = loadu_sse41(&inputs[2][block_offset + 0 * sizeof(__m128i)]);
1708-  out_msg[3]  = loadu_sse41(&inputs[3][block_offset + 0 * sizeof(__m128i)]);
1709-  out_msg[4]  = loadu_sse41(&inputs[0][block_offset + 1 * sizeof(__m128i)]);
1710-  out_msg[5]  = loadu_sse41(&inputs[1][block_offset + 1 * sizeof(__m128i)]);
1711-  out_msg[6]  = loadu_sse41(&inputs[2][block_offset + 1 * sizeof(__m128i)]);
1712-  out_msg[7]  = loadu_sse41(&inputs[3][block_offset + 1 * sizeof(__m128i)]);
1713-  out_msg[8]  = loadu_sse41(&inputs[0][block_offset + 2 * sizeof(__m128i)]);
1714-  out_msg[9]  = loadu_sse41(&inputs[1][block_offset + 2 * sizeof(__m128i)]);
1715-  out_msg[10] = loadu_sse41(&inputs[2][block_offset + 2 * sizeof(__m128i)]);
1716-  out_msg[11] = loadu_sse41(&inputs[3][block_offset + 2 * sizeof(__m128i)]);
1717-  out_msg[12] = loadu_sse41(&inputs[0][block_offset + 3 * sizeof(__m128i)]);
1718-  out_msg[13] = loadu_sse41(&inputs[1][block_offset + 3 * sizeof(__m128i)]);
1719-  out_msg[14] = loadu_sse41(&inputs[2][block_offset + 3 * sizeof(__m128i)]);
1720-  out_msg[15] = loadu_sse41(&inputs[3][block_offset + 3 * sizeof(__m128i)]);
1721-
1722-  for (i = 0; i < 4; i++) {
1723-    _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0);
1724-  }
1725-  transpose_vecs_sse41(&out_msg[0]);
1726-  transpose_vecs_sse41(&out_msg[4]);
1727-  transpose_vecs_sse41(&out_msg[8]);
1728-  transpose_vecs_sse41(&out_msg[12]);
1729-}
1730-
1731-static inline void
1732-load_counters_sse41(uint64_t counter, int increment_counter, __m128i *out_lo, __m128i *out_hi)
1733-{
1734-  const __m128i mask  = _mm_set1_epi32(-increment_counter);
1735-  const __m128i add0  = _mm_set_epi32(3, 2, 1, 0);
1736-  const __m128i add1  = _mm_and_si128(mask, add0);
1737-  __m128i       l     = _mm_add_epi32(_mm_set1_epi32((int32_t)counter), add1);
1738-  __m128i       carry = _mm_cmpgt_epi32(
1739-      _mm_xor_si128(add1, _mm_set1_epi32(0x80000000)), _mm_xor_si128(l, _mm_set1_epi32(0x80000000))
1740-  );
1741-  __m128i h = _mm_sub_epi32(_mm_set1_epi32((int32_t)(counter >> 32)), carry);
1742-
1743-  *out_lo = l;
1744-  *out_hi = h;
1745-}
1746-
1747-void
1748-blake3_hash4_sse41(
1749-    const uint8_t *const *inputs,
1750-    size_t                blocks,
1751-    const uint32_t        key[8],
1752-    uint64_t              counter,
1753-    int                   increment_counter,
1754-    uint8_t               flags,
1755-    uint8_t               flags_start,
1756-    uint8_t               flags_end,
1757-    uint8_t              *out_bytes
1758-)
1759-{
1760-  __m128i h_vecs[8];
1761-  __m128i counter_low_vec, counter_high_vec;
1762-  uint8_t block_flags;
1763-  size_t  block;
1764-
1765-  h_vecs[0] = set1_sse41(key[0]);
1766-  h_vecs[1] = set1_sse41(key[1]);
1767-  h_vecs[2] = set1_sse41(key[2]);
1768-  h_vecs[3] = set1_sse41(key[3]);
1769-  h_vecs[4] = set1_sse41(key[4]);
1770-  h_vecs[5] = set1_sse41(key[5]);
1771-  h_vecs[6] = set1_sse41(key[6]);
1772-  h_vecs[7] = set1_sse41(key[7]);
1773-
1774-  load_counters_sse41(counter, increment_counter, &counter_low_vec, &counter_high_vec);
1775-  block_flags = flags | flags_start;
1776-
1777-  for (block = 0; block < blocks; block++) {
1778-    __m128i block_len_vec;
1779-    __m128i block_flags_vec;
1780-    __m128i msg_vecs[16];
1781-    __m128i v[16];
1782-
1783-    if (block + 1 == blocks) {
1784-      block_flags |= flags_end;
1785-    }
1786-    block_len_vec   = set1_sse41(BLAKE3_BLOCK_LEN);
1787-    block_flags_vec = set1_sse41(block_flags);
1788-    transpose_msg_vecs_sse41(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs);
1789-
1790-    v[0]  = h_vecs[0];
1791-    v[1]  = h_vecs[1];
1792-    v[2]  = h_vecs[2];
1793-    v[3]  = h_vecs[3];
1794-    v[4]  = h_vecs[4];
1795-    v[5]  = h_vecs[5];
1796-    v[6]  = h_vecs[6];
1797-    v[7]  = h_vecs[7];
1798-    v[8]  = set1_sse41(IV[0]);
1799-    v[9]  = set1_sse41(IV[1]);
1800-    v[10] = set1_sse41(IV[2]);
1801-    v[11] = set1_sse41(IV[3]);
1802-    v[12] = counter_low_vec;
1803-    v[13] = counter_high_vec;
1804-    v[14] = block_len_vec;
1805-    v[15] = block_flags_vec;
1806-
1807-    round_fn_sse41(v, msg_vecs, 0);
1808-    round_fn_sse41(v, msg_vecs, 1);
1809-    round_fn_sse41(v, msg_vecs, 2);
1810-    round_fn_sse41(v, msg_vecs, 3);
1811-    round_fn_sse41(v, msg_vecs, 4);
1812-    round_fn_sse41(v, msg_vecs, 5);
1813-    round_fn_sse41(v, msg_vecs, 6);
1814-
1815-    h_vecs[0] = xorv_sse41(v[0], v[8]);
1816-    h_vecs[1] = xorv_sse41(v[1], v[9]);
1817-    h_vecs[2] = xorv_sse41(v[2], v[10]);
1818-    h_vecs[3] = xorv_sse41(v[3], v[11]);
1819-    h_vecs[4] = xorv_sse41(v[4], v[12]);
1820-    h_vecs[5] = xorv_sse41(v[5], v[13]);
1821-    h_vecs[6] = xorv_sse41(v[6], v[14]);
1822-    h_vecs[7] = xorv_sse41(v[7], v[15]);
1823-
1824-    block_flags = flags;
1825-  }
1826-
1827-  transpose_vecs_sse41(&h_vecs[0]);
1828-  transpose_vecs_sse41(&h_vecs[4]);
1829-  storeu_sse41(h_vecs[0], &out_bytes[0 * sizeof(__m128i)]);
1830-  storeu_sse41(h_vecs[4], &out_bytes[1 * sizeof(__m128i)]);
1831-  storeu_sse41(h_vecs[1], &out_bytes[2 * sizeof(__m128i)]);
1832-  storeu_sse41(h_vecs[5], &out_bytes[3 * sizeof(__m128i)]);
1833-  storeu_sse41(h_vecs[2], &out_bytes[4 * sizeof(__m128i)]);
1834-  storeu_sse41(h_vecs[6], &out_bytes[5 * sizeof(__m128i)]);
1835-  storeu_sse41(h_vecs[3], &out_bytes[6 * sizeof(__m128i)]);
1836-  storeu_sse41(h_vecs[7], &out_bytes[7 * sizeof(__m128i)]);
1837-}
1838-
1839-static inline void
1840-hash_one_sse41(
1841-    const uint8_t *input,
1842-    size_t         blocks,
1843-    const uint32_t key[8],
1844-    uint64_t       counter,
1845-    uint8_t        flags,
1846-    uint8_t        flags_start,
1847-    uint8_t        flags_end,
1848-    uint8_t        out_bytes[BLAKE3_OUT_LEN]
1849-)
1850-{
1851-  uint32_t cv[8];
1852-  uint8_t  block_flags;
1853-
1854-  memcpy(cv, key, BLAKE3_KEY_LEN);
1855-  block_flags = flags | flags_start;
1856-  while (blocks > 0) {
1857-    if (blocks == 1) {
1858-      block_flags |= flags_end;
1859-    }
1860-    blake3_compress_in_place_sse41(cv, input, BLAKE3_BLOCK_LEN, counter, block_flags);
1861-    input = &input[BLAKE3_BLOCK_LEN];
1862-    blocks -= 1;
1863-    block_flags = flags;
1864-  }
1865-  memcpy(out_bytes, cv, BLAKE3_OUT_LEN);
1866-}
1867-
1868-void
1869-blake3_hash_many_sse41(
1870-    const uint8_t *const *inputs,
1871-    size_t                num_inputs,
1872-    size_t                blocks,
1873-    const uint32_t        key[8],
1874-    uint64_t              counter,
1875-    int                   increment_counter,
1876-    uint8_t               flags,
1877-    uint8_t               flags_start,
1878-    uint8_t               flags_end,
1879-    uint8_t              *out_bytes
1880-)
1881-{
1882-  while (num_inputs >= DEGREE_SSE41) {
1883-    blake3_hash4_sse41(
1884-        inputs, blocks, key, counter, increment_counter, flags, flags_start, flags_end, out_bytes
1885-    );
1886-    if (increment_counter) {
1887-      counter += DEGREE_SSE41;
1888-    }
1889-    inputs += DEGREE_SSE41;
1890-    num_inputs -= DEGREE_SSE41;
1891-    out_bytes = &out_bytes[DEGREE_SSE41 * BLAKE3_OUT_LEN];
1892-  }
1893-  while (num_inputs > 0) {
1894-    hash_one_sse41(inputs[0], blocks, key, counter, flags, flags_start, flags_end, out_bytes);
1895-    if (increment_counter) {
1896-      counter += 1;
1897-    }
1898-    inputs += 1;
1899-    num_inputs -= 1;
1900-    out_bytes = &out_bytes[BLAKE3_OUT_LEN];
1901-  }
1902-}
1903-#if defined(__clang__)
1904-#pragma clang attribute pop
1905-#elif defined(__GNUC__)
1906-#pragma GCC pop_options
1907-#endif
1908-
1909-#if defined(__clang__)
1910-#pragma clang attribute push(__attribute__((target("avx2"))), apply_to = function)
1911-#elif defined(__GNUC__)
1912-#pragma GCC push_options
1913-#pragma GCC target("avx2")
1914-#endif
1915-#define DEGREE_AVX2 8
1916-
1917-static inline __m256i
1918-loadu_avx2(const uint8_t src[32])
1919-{
1920-  return _mm256_loadu_si256((const __m256i *)src);
1921-}
1922-
1923-static inline void
1924-storeu_avx2(__m256i src, uint8_t dest[32])
1925-{
1926-  _mm256_storeu_si256((__m256i *)dest, src);
1927-}
1928-
1929-static inline __m256i
1930-addv_avx2(__m256i a, __m256i b)
1931-{
1932-  return _mm256_add_epi32(a, b);
1933-}
1934-
1935-static inline __m256i
1936-xorv_avx2(__m256i a, __m256i b)
1937-{
1938-  return _mm256_xor_si256(a, b);
1939-}
1940-
1941-static inline __m256i
1942-set1_avx2(uint32_t x)
1943-{
1944-  return _mm256_set1_epi32((int32_t)x);
1945-}
1946-
1947-static inline __m256i
1948-rot16_avx2(__m256i x)
1949-{
1950-  return _mm256_shuffle_epi8(
1951-      x,
1952-      _mm256_set_epi8(
1953-          13,
1954-          12,
1955-          15,
1956-          14,
1957-          9,
1958-          8,
1959-          11,
1960-          10,
1961-          5,
1962-          4,
1963-          7,
1964-          6,
1965-          1,
1966-          0,
1967-          3,
1968-          2,
1969-          13,
1970-          12,
1971-          15,
1972-          14,
1973-          9,
1974-          8,
1975-          11,
1976-          10,
1977-          5,
1978-          4,
1979-          7,
1980-          6,
1981-          1,
1982-          0,
1983-          3,
1984-          2
1985-      )
1986-  );
1987-}
1988-
1989-static inline __m256i
1990-rot12_avx2(__m256i x)
1991-{
1992-  return _mm256_or_si256(_mm256_srli_epi32(x, 12), _mm256_slli_epi32(x, 32 - 12));
1993-}
1994-
1995-static inline __m256i
1996-rot8_avx2(__m256i x)
1997-{
1998-  return _mm256_shuffle_epi8(
1999-      x,
2000-      _mm256_set_epi8(
2001-          12,
2002-          15,
2003-          14,
2004-          13,
2005-          8,
2006-          11,
2007-          10,
2008-          9,
2009-          4,
2010-          7,
2011-          6,
2012-          5,
2013-          0,
2014-          3,
2015-          2,
2016-          1,
2017-          12,
2018-          15,
2019-          14,
2020-          13,
2021-          8,
2022-          11,
2023-          10,
2024-          9,
2025-          4,
2026-          7,
2027-          6,
2028-          5,
2029-          0,
2030-          3,
2031-          2,
2032-          1
2033-      )
2034-  );
2035-}
2036-
2037-static inline __m256i
2038-rot7_avx2(__m256i x)
2039-{
2040-  return _mm256_or_si256(_mm256_srli_epi32(x, 7), _mm256_slli_epi32(x, 32 - 7));
2041-}
2042-
2043-static inline void
2044-round_fn_avx2(__m256i v[16], __m256i m[16], size_t r)
2045-{
2046-  v[0]  = addv_avx2(v[0], m[(size_t)MSG_SCHEDULE[r][0]]);
2047-  v[1]  = addv_avx2(v[1], m[(size_t)MSG_SCHEDULE[r][2]]);
2048-  v[2]  = addv_avx2(v[2], m[(size_t)MSG_SCHEDULE[r][4]]);
2049-  v[3]  = addv_avx2(v[3], m[(size_t)MSG_SCHEDULE[r][6]]);
2050-  v[0]  = addv_avx2(v[0], v[4]);
2051-  v[1]  = addv_avx2(v[1], v[5]);
2052-  v[2]  = addv_avx2(v[2], v[6]);
2053-  v[3]  = addv_avx2(v[3], v[7]);
2054-  v[12] = xorv_avx2(v[12], v[0]);
2055-  v[13] = xorv_avx2(v[13], v[1]);
2056-  v[14] = xorv_avx2(v[14], v[2]);
2057-  v[15] = xorv_avx2(v[15], v[3]);
2058-  v[12] = rot16_avx2(v[12]);
2059-  v[13] = rot16_avx2(v[13]);
2060-  v[14] = rot16_avx2(v[14]);
2061-  v[15] = rot16_avx2(v[15]);
2062-  v[8]  = addv_avx2(v[8], v[12]);
2063-  v[9]  = addv_avx2(v[9], v[13]);
2064-  v[10] = addv_avx2(v[10], v[14]);
2065-  v[11] = addv_avx2(v[11], v[15]);
2066-  v[4]  = xorv_avx2(v[4], v[8]);
2067-  v[5]  = xorv_avx2(v[5], v[9]);
2068-  v[6]  = xorv_avx2(v[6], v[10]);
2069-  v[7]  = xorv_avx2(v[7], v[11]);
2070-  v[4]  = rot12_avx2(v[4]);
2071-  v[5]  = rot12_avx2(v[5]);
2072-  v[6]  = rot12_avx2(v[6]);
2073-  v[7]  = rot12_avx2(v[7]);
2074-  v[0]  = addv_avx2(v[0], m[(size_t)MSG_SCHEDULE[r][1]]);
2075-  v[1]  = addv_avx2(v[1], m[(size_t)MSG_SCHEDULE[r][3]]);
2076-  v[2]  = addv_avx2(v[2], m[(size_t)MSG_SCHEDULE[r][5]]);
2077-  v[3]  = addv_avx2(v[3], m[(size_t)MSG_SCHEDULE[r][7]]);
2078-  v[0]  = addv_avx2(v[0], v[4]);
2079-  v[1]  = addv_avx2(v[1], v[5]);
2080-  v[2]  = addv_avx2(v[2], v[6]);
2081-  v[3]  = addv_avx2(v[3], v[7]);
2082-  v[12] = xorv_avx2(v[12], v[0]);
2083-  v[13] = xorv_avx2(v[13], v[1]);
2084-  v[14] = xorv_avx2(v[14], v[2]);
2085-  v[15] = xorv_avx2(v[15], v[3]);
2086-  v[12] = rot8_avx2(v[12]);
2087-  v[13] = rot8_avx2(v[13]);
2088-  v[14] = rot8_avx2(v[14]);
2089-  v[15] = rot8_avx2(v[15]);
2090-  v[8]  = addv_avx2(v[8], v[12]);
2091-  v[9]  = addv_avx2(v[9], v[13]);
2092-  v[10] = addv_avx2(v[10], v[14]);
2093-  v[11] = addv_avx2(v[11], v[15]);
2094-  v[4]  = xorv_avx2(v[4], v[8]);
2095-  v[5]  = xorv_avx2(v[5], v[9]);
2096-  v[6]  = xorv_avx2(v[6], v[10]);
2097-  v[7]  = xorv_avx2(v[7], v[11]);
2098-  v[4]  = rot7_avx2(v[4]);
2099-  v[5]  = rot7_avx2(v[5]);
2100-  v[6]  = rot7_avx2(v[6]);
2101-  v[7]  = rot7_avx2(v[7]);
2102-
2103-  v[0]  = addv_avx2(v[0], m[(size_t)MSG_SCHEDULE[r][8]]);
2104-  v[1]  = addv_avx2(v[1], m[(size_t)MSG_SCHEDULE[r][10]]);
2105-  v[2]  = addv_avx2(v[2], m[(size_t)MSG_SCHEDULE[r][12]]);
2106-  v[3]  = addv_avx2(v[3], m[(size_t)MSG_SCHEDULE[r][14]]);
2107-  v[0]  = addv_avx2(v[0], v[5]);
2108-  v[1]  = addv_avx2(v[1], v[6]);
2109-  v[2]  = addv_avx2(v[2], v[7]);
2110-  v[3]  = addv_avx2(v[3], v[4]);
2111-  v[15] = xorv_avx2(v[15], v[0]);
2112-  v[12] = xorv_avx2(v[12], v[1]);
2113-  v[13] = xorv_avx2(v[13], v[2]);
2114-  v[14] = xorv_avx2(v[14], v[3]);
2115-  v[15] = rot16_avx2(v[15]);
2116-  v[12] = rot16_avx2(v[12]);
2117-  v[13] = rot16_avx2(v[13]);
2118-  v[14] = rot16_avx2(v[14]);
2119-  v[10] = addv_avx2(v[10], v[15]);
2120-  v[11] = addv_avx2(v[11], v[12]);
2121-  v[8]  = addv_avx2(v[8], v[13]);
2122-  v[9]  = addv_avx2(v[9], v[14]);
2123-  v[5]  = xorv_avx2(v[5], v[10]);
2124-  v[6]  = xorv_avx2(v[6], v[11]);
2125-  v[7]  = xorv_avx2(v[7], v[8]);
2126-  v[4]  = xorv_avx2(v[4], v[9]);
2127-  v[5]  = rot12_avx2(v[5]);
2128-  v[6]  = rot12_avx2(v[6]);
2129-  v[7]  = rot12_avx2(v[7]);
2130-  v[4]  = rot12_avx2(v[4]);
2131-  v[0]  = addv_avx2(v[0], m[(size_t)MSG_SCHEDULE[r][9]]);
2132-  v[1]  = addv_avx2(v[1], m[(size_t)MSG_SCHEDULE[r][11]]);
2133-  v[2]  = addv_avx2(v[2], m[(size_t)MSG_SCHEDULE[r][13]]);
2134-  v[3]  = addv_avx2(v[3], m[(size_t)MSG_SCHEDULE[r][15]]);
2135-  v[0]  = addv_avx2(v[0], v[5]);
2136-  v[1]  = addv_avx2(v[1], v[6]);
2137-  v[2]  = addv_avx2(v[2], v[7]);
2138-  v[3]  = addv_avx2(v[3], v[4]);
2139-  v[15] = xorv_avx2(v[15], v[0]);
2140-  v[12] = xorv_avx2(v[12], v[1]);
2141-  v[13] = xorv_avx2(v[13], v[2]);
2142-  v[14] = xorv_avx2(v[14], v[3]);
2143-  v[15] = rot8_avx2(v[15]);
2144-  v[12] = rot8_avx2(v[12]);
2145-  v[13] = rot8_avx2(v[13]);
2146-  v[14] = rot8_avx2(v[14]);
2147-  v[10] = addv_avx2(v[10], v[15]);
2148-  v[11] = addv_avx2(v[11], v[12]);
2149-  v[8]  = addv_avx2(v[8], v[13]);
2150-  v[9]  = addv_avx2(v[9], v[14]);
2151-  v[5]  = xorv_avx2(v[5], v[10]);
2152-  v[6]  = xorv_avx2(v[6], v[11]);
2153-  v[7]  = xorv_avx2(v[7], v[8]);
2154-  v[4]  = xorv_avx2(v[4], v[9]);
2155-  v[5]  = rot7_avx2(v[5]);
2156-  v[6]  = rot7_avx2(v[6]);
2157-  v[7]  = rot7_avx2(v[7]);
2158-  v[4]  = rot7_avx2(v[4]);
2159-}
2160-
2161-static inline void
2162-transpose_vecs_avx2(__m256i vecs[DEGREE_AVX2])
2163-{
2164-  __m256i ab_0145 = _mm256_unpacklo_epi32(vecs[0], vecs[1]);
2165-  __m256i ab_2367 = _mm256_unpackhi_epi32(vecs[0], vecs[1]);
2166-  __m256i cd_0145 = _mm256_unpacklo_epi32(vecs[2], vecs[3]);
2167-  __m256i cd_2367 = _mm256_unpackhi_epi32(vecs[2], vecs[3]);
2168-  __m256i ef_0145 = _mm256_unpacklo_epi32(vecs[4], vecs[5]);
2169-  __m256i ef_2367 = _mm256_unpackhi_epi32(vecs[4], vecs[5]);
2170-  __m256i gh_0145 = _mm256_unpacklo_epi32(vecs[6], vecs[7]);
2171-  __m256i gh_2367 = _mm256_unpackhi_epi32(vecs[6], vecs[7]);
2172-
2173-  __m256i abcd_04 = _mm256_unpacklo_epi64(ab_0145, cd_0145);
2174-  __m256i abcd_15 = _mm256_unpackhi_epi64(ab_0145, cd_0145);
2175-  __m256i abcd_26 = _mm256_unpacklo_epi64(ab_2367, cd_2367);
2176-  __m256i abcd_37 = _mm256_unpackhi_epi64(ab_2367, cd_2367);
2177-  __m256i efgh_04 = _mm256_unpacklo_epi64(ef_0145, gh_0145);
2178-  __m256i efgh_15 = _mm256_unpackhi_epi64(ef_0145, gh_0145);
2179-  __m256i efgh_26 = _mm256_unpacklo_epi64(ef_2367, gh_2367);
2180-  __m256i efgh_37 = _mm256_unpackhi_epi64(ef_2367, gh_2367);
2181-
2182-  vecs[0] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x20);
2183-  vecs[1] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x20);
2184-  vecs[2] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x20);
2185-  vecs[3] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x20);
2186-  vecs[4] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x31);
2187-  vecs[5] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x31);
2188-  vecs[6] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x31);
2189-  vecs[7] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x31);
2190-}
2191-
2192-static inline void
2193-transpose_msg_vecs_avx2(const uint8_t *const *inputs, size_t block_offset, __m256i out_msg[16])
2194-{
2195-  size_t i;
2196-
2197-  out_msg[0]  = loadu_avx2(&inputs[0][block_offset + 0 * sizeof(__m256i)]);
2198-  out_msg[1]  = loadu_avx2(&inputs[1][block_offset + 0 * sizeof(__m256i)]);
2199-  out_msg[2]  = loadu_avx2(&inputs[2][block_offset + 0 * sizeof(__m256i)]);
2200-  out_msg[3]  = loadu_avx2(&inputs[3][block_offset + 0 * sizeof(__m256i)]);
2201-  out_msg[4]  = loadu_avx2(&inputs[4][block_offset + 0 * sizeof(__m256i)]);
2202-  out_msg[5]  = loadu_avx2(&inputs[5][block_offset + 0 * sizeof(__m256i)]);
2203-  out_msg[6]  = loadu_avx2(&inputs[6][block_offset + 0 * sizeof(__m256i)]);
2204-  out_msg[7]  = loadu_avx2(&inputs[7][block_offset + 0 * sizeof(__m256i)]);
2205-  out_msg[8]  = loadu_avx2(&inputs[0][block_offset + 1 * sizeof(__m256i)]);
2206-  out_msg[9]  = loadu_avx2(&inputs[1][block_offset + 1 * sizeof(__m256i)]);
2207-  out_msg[10] = loadu_avx2(&inputs[2][block_offset + 1 * sizeof(__m256i)]);
2208-  out_msg[11] = loadu_avx2(&inputs[3][block_offset + 1 * sizeof(__m256i)]);
2209-  out_msg[12] = loadu_avx2(&inputs[4][block_offset + 1 * sizeof(__m256i)]);
2210-  out_msg[13] = loadu_avx2(&inputs[5][block_offset + 1 * sizeof(__m256i)]);
2211-  out_msg[14] = loadu_avx2(&inputs[6][block_offset + 1 * sizeof(__m256i)]);
2212-  out_msg[15] = loadu_avx2(&inputs[7][block_offset + 1 * sizeof(__m256i)]);
2213-
2214-  for (i = 0; i < 8; i++) {
2215-    _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0);
2216-  }
2217-  transpose_vecs_avx2(&out_msg[0]);
2218-  transpose_vecs_avx2(&out_msg[8]);
2219-}
2220-
2221-static inline void
2222-load_counters_avx2(uint64_t counter, int increment_counter, __m256i *out_lo, __m256i *out_hi)
2223-{
2224-  const __m256i mask  = _mm256_set1_epi32(-increment_counter);
2225-  const __m256i add0  = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
2226-  const __m256i add1  = _mm256_and_si256(mask, add0);
2227-  __m256i       l     = _mm256_add_epi32(_mm256_set1_epi32((int32_t)counter), add1);
2228-  __m256i       carry = _mm256_xor_si256(add1, _mm256_set1_epi32(0x80000000));
2229-  __m256i       comp  = _mm256_xor_si256(l, _mm256_set1_epi32(0x80000000));
2230-  __m256i       gt    = _mm256_cmpgt_epi32(carry, comp);
2231-  __m256i       h     = _mm256_sub_epi32(_mm256_set1_epi32((int32_t)(counter >> 32)), gt);
2232-
2233-  *out_lo = l;
2234-  *out_hi = h;
2235-}
2236-
2237-void
2238-blake3_hash8_avx2(
2239-    const uint8_t *const *inputs,
2240-    size_t                blocks,
2241-    const uint32_t        key[8],
2242-    uint64_t              counter,
2243-    int                   increment_counter,
2244-    uint8_t               flags,
2245-    uint8_t               flags_start,
2246-    uint8_t               flags_end,
2247-    uint8_t              *out_bytes
2248-)
2249-{
2250-  __m256i h_vecs[8];
2251-  __m256i counter_low_vec, counter_high_vec;
2252-  uint8_t block_flags;
2253-  size_t  block;
2254-
2255-  h_vecs[0] = set1_avx2(key[0]);
2256-  h_vecs[1] = set1_avx2(key[1]);
2257-  h_vecs[2] = set1_avx2(key[2]);
2258-  h_vecs[3] = set1_avx2(key[3]);
2259-  h_vecs[4] = set1_avx2(key[4]);
2260-  h_vecs[5] = set1_avx2(key[5]);
2261-  h_vecs[6] = set1_avx2(key[6]);
2262-  h_vecs[7] = set1_avx2(key[7]);
2263-
2264-  load_counters_avx2(counter, increment_counter, &counter_low_vec, &counter_high_vec);
2265-  block_flags = flags | flags_start;
2266-
2267-  for (block = 0; block < blocks; block++) {
2268-    __m256i block_len_vec;
2269-    __m256i block_flags_vec;
2270-    __m256i msg_vecs[16];
2271-    __m256i v[16];
2272-
2273-    if (block + 1 == blocks) {
2274-      block_flags |= flags_end;
2275-    }
2276-    block_len_vec   = set1_avx2(BLAKE3_BLOCK_LEN);
2277-    block_flags_vec = set1_avx2(block_flags);
2278-    transpose_msg_vecs_avx2(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs);
2279-
2280-    v[0]  = h_vecs[0];
2281-    v[1]  = h_vecs[1];
2282-    v[2]  = h_vecs[2];
2283-    v[3]  = h_vecs[3];
2284-    v[4]  = h_vecs[4];
2285-    v[5]  = h_vecs[5];
2286-    v[6]  = h_vecs[6];
2287-    v[7]  = h_vecs[7];
2288-    v[8]  = set1_avx2(IV[0]);
2289-    v[9]  = set1_avx2(IV[1]);
2290-    v[10] = set1_avx2(IV[2]);
2291-    v[11] = set1_avx2(IV[3]);
2292-    v[12] = counter_low_vec;
2293-    v[13] = counter_high_vec;
2294-    v[14] = block_len_vec;
2295-    v[15] = block_flags_vec;
2296-
2297-    round_fn_avx2(v, msg_vecs, 0);
2298-    round_fn_avx2(v, msg_vecs, 1);
2299-    round_fn_avx2(v, msg_vecs, 2);
2300-    round_fn_avx2(v, msg_vecs, 3);
2301-    round_fn_avx2(v, msg_vecs, 4);
2302-    round_fn_avx2(v, msg_vecs, 5);
2303-    round_fn_avx2(v, msg_vecs, 6);
2304-
2305-    h_vecs[0] = xorv_avx2(v[0], v[8]);
2306-    h_vecs[1] = xorv_avx2(v[1], v[9]);
2307-    h_vecs[2] = xorv_avx2(v[2], v[10]);
2308-    h_vecs[3] = xorv_avx2(v[3], v[11]);
2309-    h_vecs[4] = xorv_avx2(v[4], v[12]);
2310-    h_vecs[5] = xorv_avx2(v[5], v[13]);
2311-    h_vecs[6] = xorv_avx2(v[6], v[14]);
2312-    h_vecs[7] = xorv_avx2(v[7], v[15]);
2313-
2314-    block_flags = flags;
2315-  }
2316-
2317-  transpose_vecs_avx2(h_vecs);
2318-  storeu_avx2(h_vecs[0], &out_bytes[0 * sizeof(__m256i)]);
2319-  storeu_avx2(h_vecs[1], &out_bytes[1 * sizeof(__m256i)]);
2320-  storeu_avx2(h_vecs[2], &out_bytes[2 * sizeof(__m256i)]);
2321-  storeu_avx2(h_vecs[3], &out_bytes[3 * sizeof(__m256i)]);
2322-  storeu_avx2(h_vecs[4], &out_bytes[4 * sizeof(__m256i)]);
2323-  storeu_avx2(h_vecs[5], &out_bytes[5 * sizeof(__m256i)]);
2324-  storeu_avx2(h_vecs[6], &out_bytes[6 * sizeof(__m256i)]);
2325-  storeu_avx2(h_vecs[7], &out_bytes[7 * sizeof(__m256i)]);
2326-}
2327-
2328-void
2329-blake3_hash_many_avx2(
2330-    const uint8_t *const *inputs,
2331-    size_t                num_inputs,
2332-    size_t                blocks,
2333-    const uint32_t        key[8],
2334-    uint64_t              counter,
2335-    int                   increment_counter,
2336-    uint8_t               flags,
2337-    uint8_t               flags_start,
2338-    uint8_t               flags_end,
2339-    uint8_t              *out_bytes
2340-)
2341-{
2342-  while (num_inputs >= DEGREE_AVX2) {
2343-    blake3_hash8_avx2(
2344-        inputs, blocks, key, counter, increment_counter, flags, flags_start, flags_end, out_bytes
2345-    );
2346-    if (increment_counter) {
2347-      counter += DEGREE_AVX2;
2348-    }
2349-    inputs += DEGREE_AVX2;
2350-    num_inputs -= DEGREE_AVX2;
2351-    out_bytes = &out_bytes[DEGREE_AVX2 * BLAKE3_OUT_LEN];
2352-  }
2353-  blake3_hash_many_sse41(
2354-      inputs,
2355-      num_inputs,
2356-      blocks,
2357-      key,
2358-      counter,
2359-      increment_counter,
2360-      flags,
2361-      flags_start,
2362-      flags_end,
2363-      out_bytes
2364-  );
2365-}
2366-#if defined(__clang__)
2367-#pragma clang attribute pop
2368-#elif defined(__GNUC__)
2369-#pragma GCC pop_options
2370-#endif
2371-
2372-#if defined(__clang__)
2373-#pragma clang attribute push(__attribute__((target("avx512f,avx512vl"))), apply_to = function)
2374-#elif defined(__GNUC__)
2375-#pragma GCC push_options
2376-#pragma GCC target("avx512f,avx512vl")
2377-#endif
2378-static inline __m128i
2379-loadu_128_avx512(const uint8_t src[16])
2380-{
2381-  return _mm_loadu_si128((const __m128i *)src);
2382-}
2383-
2384-static inline __m256i
2385-loadu_256_avx512(const uint8_t src[32])
2386-{
2387-  return _mm256_loadu_si256((const __m256i *)src);
2388-}
2389-
2390-static inline __m512i
2391-loadu_512_avx512(const uint8_t src[64])
2392-{
2393-  return _mm512_loadu_si512((const __m512i *)src);
2394-}
2395-
2396-static inline void
2397-storeu_128_avx512(__m128i src, uint8_t dest[16])
2398-{
2399-  _mm_storeu_si128((__m128i *)dest, src);
2400-}
2401-
2402-static inline void
2403-storeu_256_avx512(__m256i src, uint8_t dest[32])
2404-{
2405-  _mm256_storeu_si256((__m256i *)dest, src);
2406-}
2407-
2408-static inline __m128i
2409-add_128_avx512(__m128i a, __m128i b)
2410-{
2411-  return _mm_add_epi32(a, b);
2412-}
2413-
2414-static inline __m256i
2415-add_256_avx512(__m256i a, __m256i b)
2416-{
2417-  return _mm256_add_epi32(a, b);
2418-}
2419-
2420-static inline __m512i
2421-add_512_avx512(__m512i a, __m512i b)
2422-{
2423-  return _mm512_add_epi32(a, b);
2424-}
2425-
2426-static inline __m128i
2427-xor_128_avx512(__m128i a, __m128i b)
2428-{
2429-  return _mm_xor_si128(a, b);
2430-}
2431-
2432-static inline __m256i
2433-xor_256_avx512(__m256i a, __m256i b)
2434-{
2435-  return _mm256_xor_si256(a, b);
2436-}
2437-
2438-static inline __m512i
2439-xor_512_avx512(__m512i a, __m512i b)
2440-{
2441-  return _mm512_xor_si512(a, b);
2442-}
2443-
2444-static inline __m128i
2445-set1_128_avx512(uint32_t x)
2446-{
2447-  return _mm_set1_epi32((int32_t)x);
2448-}
2449-
2450-static inline __m256i
2451-set1_256_avx512(uint32_t x)
2452-{
2453-  return _mm256_set1_epi32((int32_t)x);
2454-}
2455-
2456-static inline __m512i
2457-set1_512_avx512(uint32_t x)
2458-{
2459-  return _mm512_set1_epi32((int32_t)x);
2460-}
2461-
2462-static inline __m128i
2463-set4_avx512(uint32_t a, uint32_t b, uint32_t c, uint32_t d)
2464-{
2465-  return _mm_setr_epi32((int32_t)a, (int32_t)b, (int32_t)c, (int32_t)d);
2466-}
2467-
2468-static inline __m128i
2469-rot16_128_avx512(__m128i x)
2470-{
2471-  return _mm_ror_epi32(x, 16);
2472-}
2473-
2474-static inline __m256i
2475-rot16_256_avx512(__m256i x)
2476-{
2477-  return _mm256_ror_epi32(x, 16);
2478-}
2479-
2480-static inline __m512i
2481-rot16_512_avx512(__m512i x)
2482-{
2483-  return _mm512_ror_epi32(x, 16);
2484-}
2485-
2486-static inline __m128i
2487-rot12_128_avx512(__m128i x)
2488-{
2489-  return _mm_ror_epi32(x, 12);
2490-}
2491-
2492-static inline __m256i
2493-rot12_256_avx512(__m256i x)
2494-{
2495-  return _mm256_ror_epi32(x, 12);
2496-}
2497-
2498-static inline __m512i
2499-rot12_512_avx512(__m512i x)
2500-{
2501-  return _mm512_ror_epi32(x, 12);
2502-}
2503-
2504-static inline __m128i
2505-rot8_128_avx512(__m128i x)
2506-{
2507-  return _mm_ror_epi32(x, 8);
2508-}
2509-
2510-static inline __m256i
2511-rot8_256_avx512(__m256i x)
2512-{
2513-  return _mm256_ror_epi32(x, 8);
2514-}
2515-
2516-static inline __m512i
2517-rot8_512_avx512(__m512i x)
2518-{
2519-  return _mm512_ror_epi32(x, 8);
2520-}
2521-
2522-static inline __m128i
2523-rot7_128_avx512(__m128i x)
2524-{
2525-  return _mm_ror_epi32(x, 7);
2526-}
2527-
2528-static inline __m256i
2529-rot7_256_avx512(__m256i x)
2530-{
2531-  return _mm256_ror_epi32(x, 7);
2532-}
2533-
2534-static inline __m512i
2535-rot7_512_avx512(__m512i x)
2536-{
2537-  return _mm512_ror_epi32(x, 7);
2538-}
2539-
2540-static inline void
2541-g1_avx512(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, __m128i m)
2542-{
2543-  *row0 = add_128_avx512(add_128_avx512(*row0, m), *row1);
2544-  *row3 = xor_128_avx512(*row3, *row0);
2545-  *row3 = rot16_128_avx512(*row3);
2546-  *row2 = add_128_avx512(*row2, *row3);
2547-  *row1 = xor_128_avx512(*row1, *row2);
2548-  *row1 = rot12_128_avx512(*row1);
2549-}
2550-
2551-static inline void
2552-g2_avx512(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, __m128i m)
2553-{
2554-  *row0 = add_128_avx512(add_128_avx512(*row0, m), *row1);
2555-  *row3 = xor_128_avx512(*row3, *row0);
2556-  *row3 = rot8_128_avx512(*row3);
2557-  *row2 = add_128_avx512(*row2, *row3);
2558-  *row1 = xor_128_avx512(*row1, *row2);
2559-  *row1 = rot7_128_avx512(*row1);
2560-}
2561-
2562-static inline void
2563-diagonalize_avx512(__m128i *row0, __m128i *row2, __m128i *row3)
2564-{
2565-  *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3));
2566-  *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2));
2567-  *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1));
2568-}
2569-
2570-static inline void
2571-undiagonalize_avx512(__m128i *row0, __m128i *row2, __m128i *row3)
2572-{
2573-  *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1));
2574-  *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2));
2575-  *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3));
2576-}
2577-
2578-static inline void
2579-compress_pre_avx512(
2580-    __m128i        rows[4],
2581-    const uint32_t cv[8],
2582-    const uint8_t  block[BLAKE3_BLOCK_LEN],
2583-    uint8_t        block_len,
2584-    uint64_t       counter,
2585-    uint8_t        flags
2586-)
2587-{
2588-  __m128i m0, m1, m2, m3;
2589-  __m128i t0, t1, t2, t3;
2590-
2591-  rows[0] = loadu_128_avx512((const uint8_t *)&cv[0]);
2592-  rows[1] = loadu_128_avx512((const uint8_t *)&cv[4]);
2593-  rows[2] = set4_avx512(IV[0], IV[1], IV[2], IV[3]);
2594-  rows[3] = set4_avx512(
2595-      counter_low(counter), counter_high(counter), (uint32_t)block_len, (uint32_t)flags
2596-  );
2597-
2598-  m0 = loadu_128_avx512(&block[sizeof(__m128i) * 0]);
2599-  m1 = loadu_128_avx512(&block[sizeof(__m128i) * 1]);
2600-  m2 = loadu_128_avx512(&block[sizeof(__m128i) * 2]);
2601-  m3 = loadu_128_avx512(&block[sizeof(__m128i) * 3]);
2602-
2603-  /* round 1 */
2604-  t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(2, 0, 2, 0));
2605-  g1_avx512(&rows[0], &rows[1], &rows[2], &rows[3], t0);
2606-  t1 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 3, 1));
2607-  g2_avx512(&rows[0], &rows[1], &rows[2], &rows[3], t1);
2608-  diagonalize_avx512(&rows[0], &rows[2], &rows[3]);
2609-  t2 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(2, 0, 2, 0));
2610-  t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3));
2611-  g1_avx512(&rows[0], &rows[1], &rows[2], &rows[3], t2);
2612-  t3 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 1, 3, 1));
2613-  t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3));
2614-  g2_avx512(&rows[0], &rows[1], &rows[2], &rows[3], t3);
2615-  undiagonalize_avx512(&rows[0], &rows[2], &rows[3]);
2616-  m0 = t0;
2617-  m1 = t1;
2618-  m2 = t2;
2619-  m3 = t3;
2620-
2621-  /* round 2 */
2622-  t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
2623-  t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
2624-  g1_avx512(&rows[0], &rows[1], &rows[2], &rows[3], t0);
2625-  t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
2626-  t1 = _mm_blend_epi16(m0, t1, 0xCC);
2627-  g2_avx512(&rows[0], &rows[1], &rows[2], &rows[3], t1);
2628-  diagonalize_avx512(&rows[0], &rows[2], &rows[3]);
2629-  t2 = _mm_unpacklo_epi64(m3, m1);
2630-  t2 = _mm_blend_epi16(t2, m2, 0xC0);
2631-  t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(1, 3, 2, 0));
2632-  g1_avx512(&rows[0], &rows[1], &rows[2], &rows[3], t2);
2633-  t3 = _mm_unpackhi_epi32(m1, m3);
2634-  t3 = _mm_unpacklo_epi32(m2, t3);
2635-  t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(0, 1, 3, 2));
2636-  g2_avx512(&rows[0], &rows[1], &rows[2], &rows[3], t3);
2637-  undiagonalize_avx512(&rows[0], &rows[2], &rows[3]);
2638-  m0 = t0;
2639-  m1 = t1;
2640-  m2 = t2;
2641-  m3 = t3;
2642-
2643-  /* round 3 */
2644-  t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
2645-  t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
2646-  g1_avx512(&rows[0], &rows[1], &rows[2], &rows[3], t0);
2647-  t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
2648-  t1 = _mm_blend_epi16(m0, t1, 0xCC);
2649-  g2_avx512(&rows[0], &rows[1], &rows[2], &rows[3], t1);
2650-  diagonalize_avx512(&rows[0], &rows[2], &rows[3]);
2651-  t2 = _mm_unpacklo_epi64(m3, m1);
2652-  t2 = _mm_blend_epi16(t2, m2, 0xC0);
2653-  t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(1, 3, 2, 0));
2654-  g1_avx512(&rows[0], &rows[1], &rows[2], &rows[3], t2);
2655-  t3 = _mm_unpackhi_epi32(m1, m3);
2656-  t3 = _mm_unpacklo_epi32(m2, t3);
2657-  t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(0, 1, 3, 2));
2658-  g2_avx512(&rows[0], &rows[1], &rows[2], &rows[3], t3);
2659-  undiagonalize_avx512(&rows[0], &rows[2], &rows[3]);
2660-  m0 = t0;
2661-  m1 = t1;
2662-  m2 = t2;
2663-  m3 = t3;
2664-
2665-  /* round 4 */
2666-  t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
2667-  t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
2668-  g1_avx512(&rows[0], &rows[1], &rows[2], &rows[3], t0);
2669-  t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
2670-  t1 = _mm_blend_epi16(m0, t1, 0xCC);
2671-  g2_avx512(&rows[0], &rows[1], &rows[2], &rows[3], t1);
2672-  diagonalize_avx512(&rows[0], &rows[2], &rows[3]);
2673-  t2 = _mm_unpacklo_epi64(m3, m1);
2674-  t2 = _mm_blend_epi16(t2, m2, 0xC0);
2675-  t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(1, 3, 2, 0));
2676-  g1_avx512(&rows[0], &rows[1], &rows[2], &rows[3], t2);
2677-  t3 = _mm_unpackhi_epi32(m1, m3);
2678-  t3 = _mm_unpacklo_epi32(m2, t3);
2679-  t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(0, 1, 3, 2));
2680-  g2_avx512(&rows[0], &rows[1], &rows[2], &rows[3], t3);
2681-  undiagonalize_avx512(&rows[0], &rows[2], &rows[3]);
2682-  m0 = t0;
2683-  m1 = t1;
2684-  m2 = t2;
2685-  m3 = t3;
2686-
2687-  /* round 5 */
2688-  t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
2689-  t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
2690-  g1_avx512(&rows[0], &rows[1], &rows[2], &rows[3], t0);
2691-  t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
2692-  t1 = _mm_blend_epi16(m0, t1, 0xCC);
2693-  g2_avx512(&rows[0], &rows[1], &rows[2], &rows[3], t1);
2694-  diagonalize_avx512(&rows[0], &rows[2], &rows[3]);
2695-  t2 = _mm_unpacklo_epi64(m3, m1);
2696-  t2 = _mm_blend_epi16(t2, m2, 0xC0);
2697-  t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(1, 3, 2, 0));
2698-  g1_avx512(&rows[0], &rows[1], &rows[2], &rows[3], t2);
2699-  t3 = _mm_unpackhi_epi32(m1, m3);
2700-  t3 = _mm_unpacklo_epi32(m2, t3);
2701-  t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(0, 1, 3, 2));
2702-  g2_avx512(&rows[0], &rows[1], &rows[2], &rows[3], t3);
2703-  undiagonalize_avx512(&rows[0], &rows[2], &rows[3]);
2704-  m0 = t0;
2705-  m1 = t1;
2706-  m2 = t2;
2707-  m3 = t3;
2708-
2709-  /* round 6 */
2710-  t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
2711-  t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
2712-  g1_avx512(&rows[0], &rows[1], &rows[2], &rows[3], t0);
2713-  t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
2714-  t1 = _mm_blend_epi16(m0, t1, 0xCC);
2715-  g2_avx512(&rows[0], &rows[1], &rows[2], &rows[3], t1);
2716-  diagonalize_avx512(&rows[0], &rows[2], &rows[3]);
2717-  t2 = _mm_unpacklo_epi64(m3, m1);
2718-  t2 = _mm_blend_epi16(t2, m2, 0xC0);
2719-  t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(1, 3, 2, 0));
2720-  g1_avx512(&rows[0], &rows[1], &rows[2], &rows[3], t2);
2721-  t3 = _mm_unpackhi_epi32(m1, m3);
2722-  t3 = _mm_unpacklo_epi32(m2, t3);
2723-  t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(0, 1, 3, 2));
2724-  g2_avx512(&rows[0], &rows[1], &rows[2], &rows[3], t3);
2725-  undiagonalize_avx512(&rows[0], &rows[2], &rows[3]);
2726-  m0 = t0;
2727-  m1 = t1;
2728-  m2 = t2;
2729-  m3 = t3;
2730-
2731-  /* round 7 */
2732-  t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
2733-  t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
2734-  g1_avx512(&rows[0], &rows[1], &rows[2], &rows[3], t0);
2735-  t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
2736-  t1 = _mm_blend_epi16(m0, t1, 0xCC);
2737-  g2_avx512(&rows[0], &rows[1], &rows[2], &rows[3], t1);
2738-  diagonalize_avx512(&rows[0], &rows[2], &rows[3]);
2739-  t2 = _mm_unpacklo_epi64(m3, m1);
2740-  t2 = _mm_blend_epi16(t2, m2, 0xC0);
2741-  t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(1, 3, 2, 0));
2742-  g1_avx512(&rows[0], &rows[1], &rows[2], &rows[3], t2);
2743-  t3 = _mm_unpackhi_epi32(m1, m3);
2744-  t3 = _mm_unpacklo_epi32(m2, t3);
2745-  t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(0, 1, 3, 2));
2746-  g2_avx512(&rows[0], &rows[1], &rows[2], &rows[3], t3);
2747-  undiagonalize_avx512(&rows[0], &rows[2], &rows[3]);
2748-}
2749-
2750-void
2751-blake3_compress_in_place_avx512(
2752-    uint32_t      cv[8],
2753-    const uint8_t block[BLAKE3_BLOCK_LEN],
2754-    uint8_t       block_len,
2755-    uint64_t      counter,
2756-    uint8_t       flags
2757-)
2758-{
2759-  __m128i rows[4];
2760-
2761-  compress_pre_avx512(rows, cv, block, block_len, counter, flags);
2762-  storeu_128_avx512(xor_128_avx512(rows[0], rows[2]), (uint8_t *)&cv[0]);
2763-  storeu_128_avx512(xor_128_avx512(rows[1], rows[3]), (uint8_t *)&cv[4]);
2764-}
2765-
2766-void
2767-blake3_compress_xof_avx512(
2768-    const uint32_t cv[8],
2769-    const uint8_t  block[BLAKE3_BLOCK_LEN],
2770-    uint8_t        block_len,
2771-    uint64_t       counter,
2772-    uint8_t        flags,
2773-    uint8_t        out_buf[64]
2774-)
2775-{
2776-  __m128i rows[4];
2777-
2778-  compress_pre_avx512(rows, cv, block, block_len, counter, flags);
2779-  storeu_128_avx512(xor_128_avx512(rows[0], rows[2]), &out_buf[0]);
2780-  storeu_128_avx512(xor_128_avx512(rows[1], rows[3]), &out_buf[16]);
2781-  storeu_128_avx512(
2782-      xor_128_avx512(rows[2], loadu_128_avx512((const uint8_t *)&cv[0])), &out_buf[32]
2783-  );
2784-  storeu_128_avx512(
2785-      xor_128_avx512(rows[3], loadu_128_avx512((const uint8_t *)&cv[4])), &out_buf[48]
2786-  );
2787-}
2788-
2789-static inline void
2790-round_fn4_avx512(__m128i v[16], __m128i m[16], size_t r)
2791-{
2792-  v[0]  = add_128_avx512(v[0], m[(size_t)MSG_SCHEDULE[r][0]]);
2793-  v[1]  = add_128_avx512(v[1], m[(size_t)MSG_SCHEDULE[r][2]]);
2794-  v[2]  = add_128_avx512(v[2], m[(size_t)MSG_SCHEDULE[r][4]]);
2795-  v[3]  = add_128_avx512(v[3], m[(size_t)MSG_SCHEDULE[r][6]]);
2796-  v[0]  = add_128_avx512(v[0], v[4]);
2797-  v[1]  = add_128_avx512(v[1], v[5]);
2798-  v[2]  = add_128_avx512(v[2], v[6]);
2799-  v[3]  = add_128_avx512(v[3], v[7]);
2800-  v[12] = xor_128_avx512(v[12], v[0]);
2801-  v[13] = xor_128_avx512(v[13], v[1]);
2802-  v[14] = xor_128_avx512(v[14], v[2]);
2803-  v[15] = xor_128_avx512(v[15], v[3]);
2804-  v[12] = rot16_128_avx512(v[12]);
2805-  v[13] = rot16_128_avx512(v[13]);
2806-  v[14] = rot16_128_avx512(v[14]);
2807-  v[15] = rot16_128_avx512(v[15]);
2808-  v[8]  = add_128_avx512(v[8], v[12]);
2809-  v[9]  = add_128_avx512(v[9], v[13]);
2810-  v[10] = add_128_avx512(v[10], v[14]);
2811-  v[11] = add_128_avx512(v[11], v[15]);
2812-  v[4]  = xor_128_avx512(v[4], v[8]);
2813-  v[5]  = xor_128_avx512(v[5], v[9]);
2814-  v[6]  = xor_128_avx512(v[6], v[10]);
2815-  v[7]  = xor_128_avx512(v[7], v[11]);
2816-  v[4]  = rot12_128_avx512(v[4]);
2817-  v[5]  = rot12_128_avx512(v[5]);
2818-  v[6]  = rot12_128_avx512(v[6]);
2819-  v[7]  = rot12_128_avx512(v[7]);
2820-  v[0]  = add_128_avx512(v[0], m[(size_t)MSG_SCHEDULE[r][1]]);
2821-  v[1]  = add_128_avx512(v[1], m[(size_t)MSG_SCHEDULE[r][3]]);
2822-  v[2]  = add_128_avx512(v[2], m[(size_t)MSG_SCHEDULE[r][5]]);
2823-  v[3]  = add_128_avx512(v[3], m[(size_t)MSG_SCHEDULE[r][7]]);
2824-  v[0]  = add_128_avx512(v[0], v[4]);
2825-  v[1]  = add_128_avx512(v[1], v[5]);
2826-  v[2]  = add_128_avx512(v[2], v[6]);
2827-  v[3]  = add_128_avx512(v[3], v[7]);
2828-  v[12] = xor_128_avx512(v[12], v[0]);
2829-  v[13] = xor_128_avx512(v[13], v[1]);
2830-  v[14] = xor_128_avx512(v[14], v[2]);
2831-  v[15] = xor_128_avx512(v[15], v[3]);
2832-  v[12] = rot8_128_avx512(v[12]);
2833-  v[13] = rot8_128_avx512(v[13]);
2834-  v[14] = rot8_128_avx512(v[14]);
2835-  v[15] = rot8_128_avx512(v[15]);
2836-  v[8]  = add_128_avx512(v[8], v[12]);
2837-  v[9]  = add_128_avx512(v[9], v[13]);
2838-  v[10] = add_128_avx512(v[10], v[14]);
2839-  v[11] = add_128_avx512(v[11], v[15]);
2840-  v[4]  = xor_128_avx512(v[4], v[8]);
2841-  v[5]  = xor_128_avx512(v[5], v[9]);
2842-  v[6]  = xor_128_avx512(v[6], v[10]);
2843-  v[7]  = xor_128_avx512(v[7], v[11]);
2844-  v[4]  = rot7_128_avx512(v[4]);
2845-  v[5]  = rot7_128_avx512(v[5]);
2846-  v[6]  = rot7_128_avx512(v[6]);
2847-  v[7]  = rot7_128_avx512(v[7]);
2848-
2849-  v[0]  = add_128_avx512(v[0], m[(size_t)MSG_SCHEDULE[r][8]]);
2850-  v[1]  = add_128_avx512(v[1], m[(size_t)MSG_SCHEDULE[r][10]]);
2851-  v[2]  = add_128_avx512(v[2], m[(size_t)MSG_SCHEDULE[r][12]]);
2852-  v[3]  = add_128_avx512(v[3], m[(size_t)MSG_SCHEDULE[r][14]]);
2853-  v[0]  = add_128_avx512(v[0], v[5]);
2854-  v[1]  = add_128_avx512(v[1], v[6]);
2855-  v[2]  = add_128_avx512(v[2], v[7]);
2856-  v[3]  = add_128_avx512(v[3], v[4]);
2857-  v[15] = xor_128_avx512(v[15], v[0]);
2858-  v[12] = xor_128_avx512(v[12], v[1]);
2859-  v[13] = xor_128_avx512(v[13], v[2]);
2860-  v[14] = xor_128_avx512(v[14], v[3]);
2861-  v[15] = rot16_128_avx512(v[15]);
2862-  v[12] = rot16_128_avx512(v[12]);
2863-  v[13] = rot16_128_avx512(v[13]);
2864-  v[14] = rot16_128_avx512(v[14]);
2865-  v[10] = add_128_avx512(v[10], v[15]);
2866-  v[11] = add_128_avx512(v[11], v[12]);
2867-  v[8]  = add_128_avx512(v[8], v[13]);
2868-  v[9]  = add_128_avx512(v[9], v[14]);
2869-  v[5]  = xor_128_avx512(v[5], v[10]);
2870-  v[6]  = xor_128_avx512(v[6], v[11]);
2871-  v[7]  = xor_128_avx512(v[7], v[8]);
2872-  v[4]  = xor_128_avx512(v[4], v[9]);
2873-  v[5]  = rot12_128_avx512(v[5]);
2874-  v[6]  = rot12_128_avx512(v[6]);
2875-  v[7]  = rot12_128_avx512(v[7]);
2876-  v[4]  = rot12_128_avx512(v[4]);
2877-  v[0]  = add_128_avx512(v[0], m[(size_t)MSG_SCHEDULE[r][9]]);
2878-  v[1]  = add_128_avx512(v[1], m[(size_t)MSG_SCHEDULE[r][11]]);
2879-  v[2]  = add_128_avx512(v[2], m[(size_t)MSG_SCHEDULE[r][13]]);
2880-  v[3]  = add_128_avx512(v[3], m[(size_t)MSG_SCHEDULE[r][15]]);
2881-  v[0]  = add_128_avx512(v[0], v[5]);
2882-  v[1]  = add_128_avx512(v[1], v[6]);
2883-  v[2]  = add_128_avx512(v[2], v[7]);
2884-  v[3]  = add_128_avx512(v[3], v[4]);
2885-  v[15] = xor_128_avx512(v[15], v[0]);
2886-  v[12] = xor_128_avx512(v[12], v[1]);
2887-  v[13] = xor_128_avx512(v[13], v[2]);
2888-  v[14] = xor_128_avx512(v[14], v[3]);
2889-  v[15] = rot8_128_avx512(v[15]);
2890-  v[12] = rot8_128_avx512(v[12]);
2891-  v[13] = rot8_128_avx512(v[13]);
2892-  v[14] = rot8_128_avx512(v[14]);
2893-  v[10] = add_128_avx512(v[10], v[15]);
2894-  v[11] = add_128_avx512(v[11], v[12]);
2895-  v[8]  = add_128_avx512(v[8], v[13]);
2896-  v[9]  = add_128_avx512(v[9], v[14]);
2897-  v[5]  = xor_128_avx512(v[5], v[10]);
2898-  v[6]  = xor_128_avx512(v[6], v[11]);
2899-  v[7]  = xor_128_avx512(v[7], v[8]);
2900-  v[4]  = xor_128_avx512(v[4], v[9]);
2901-  v[5]  = rot7_128_avx512(v[5]);
2902-  v[6]  = rot7_128_avx512(v[6]);
2903-  v[7]  = rot7_128_avx512(v[7]);
2904-  v[4]  = rot7_128_avx512(v[4]);
2905-}
2906-
2907-static inline void
2908-round_fn8_avx512(__m256i v[16], __m256i m[16], size_t r)
2909-{
2910-  v[0]  = add_256_avx512(v[0], m[(size_t)MSG_SCHEDULE[r][0]]);
2911-  v[1]  = add_256_avx512(v[1], m[(size_t)MSG_SCHEDULE[r][2]]);
2912-  v[2]  = add_256_avx512(v[2], m[(size_t)MSG_SCHEDULE[r][4]]);
2913-  v[3]  = add_256_avx512(v[3], m[(size_t)MSG_SCHEDULE[r][6]]);
2914-  v[0]  = add_256_avx512(v[0], v[4]);
2915-  v[1]  = add_256_avx512(v[1], v[5]);
2916-  v[2]  = add_256_avx512(v[2], v[6]);
2917-  v[3]  = add_256_avx512(v[3], v[7]);
2918-  v[12] = xor_256_avx512(v[12], v[0]);
2919-  v[13] = xor_256_avx512(v[13], v[1]);
2920-  v[14] = xor_256_avx512(v[14], v[2]);
2921-  v[15] = xor_256_avx512(v[15], v[3]);
2922-  v[12] = rot16_256_avx512(v[12]);
2923-  v[13] = rot16_256_avx512(v[13]);
2924-  v[14] = rot16_256_avx512(v[14]);
2925-  v[15] = rot16_256_avx512(v[15]);
2926-  v[8]  = add_256_avx512(v[8], v[12]);
2927-  v[9]  = add_256_avx512(v[9], v[13]);
2928-  v[10] = add_256_avx512(v[10], v[14]);
2929-  v[11] = add_256_avx512(v[11], v[15]);
2930-  v[4]  = xor_256_avx512(v[4], v[8]);
2931-  v[5]  = xor_256_avx512(v[5], v[9]);
2932-  v[6]  = xor_256_avx512(v[6], v[10]);
2933-  v[7]  = xor_256_avx512(v[7], v[11]);
2934-  v[4]  = rot12_256_avx512(v[4]);
2935-  v[5]  = rot12_256_avx512(v[5]);
2936-  v[6]  = rot12_256_avx512(v[6]);
2937-  v[7]  = rot12_256_avx512(v[7]);
2938-  v[0]  = add_256_avx512(v[0], m[(size_t)MSG_SCHEDULE[r][1]]);
2939-  v[1]  = add_256_avx512(v[1], m[(size_t)MSG_SCHEDULE[r][3]]);
2940-  v[2]  = add_256_avx512(v[2], m[(size_t)MSG_SCHEDULE[r][5]]);
2941-  v[3]  = add_256_avx512(v[3], m[(size_t)MSG_SCHEDULE[r][7]]);
2942-  v[0]  = add_256_avx512(v[0], v[4]);
2943-  v[1]  = add_256_avx512(v[1], v[5]);
2944-  v[2]  = add_256_avx512(v[2], v[6]);
2945-  v[3]  = add_256_avx512(v[3], v[7]);
2946-  v[12] = xor_256_avx512(v[12], v[0]);
2947-  v[13] = xor_256_avx512(v[13], v[1]);
2948-  v[14] = xor_256_avx512(v[14], v[2]);
2949-  v[15] = xor_256_avx512(v[15], v[3]);
2950-  v[12] = rot8_256_avx512(v[12]);
2951-  v[13] = rot8_256_avx512(v[13]);
2952-  v[14] = rot8_256_avx512(v[14]);
2953-  v[15] = rot8_256_avx512(v[15]);
2954-  v[8]  = add_256_avx512(v[8], v[12]);
2955-  v[9]  = add_256_avx512(v[9], v[13]);
2956-  v[10] = add_256_avx512(v[10], v[14]);
2957-  v[11] = add_256_avx512(v[11], v[15]);
2958-  v[4]  = xor_256_avx512(v[4], v[8]);
2959-  v[5]  = xor_256_avx512(v[5], v[9]);
2960-  v[6]  = xor_256_avx512(v[6], v[10]);
2961-  v[7]  = xor_256_avx512(v[7], v[11]);
2962-  v[4]  = rot7_256_avx512(v[4]);
2963-  v[5]  = rot7_256_avx512(v[5]);
2964-  v[6]  = rot7_256_avx512(v[6]);
2965-  v[7]  = rot7_256_avx512(v[7]);
2966-
2967-  v[0]  = add_256_avx512(v[0], m[(size_t)MSG_SCHEDULE[r][8]]);
2968-  v[1]  = add_256_avx512(v[1], m[(size_t)MSG_SCHEDULE[r][10]]);
2969-  v[2]  = add_256_avx512(v[2], m[(size_t)MSG_SCHEDULE[r][12]]);
2970-  v[3]  = add_256_avx512(v[3], m[(size_t)MSG_SCHEDULE[r][14]]);
2971-  v[0]  = add_256_avx512(v[0], v[5]);
2972-  v[1]  = add_256_avx512(v[1], v[6]);
2973-  v[2]  = add_256_avx512(v[2], v[7]);
2974-  v[3]  = add_256_avx512(v[3], v[4]);
2975-  v[15] = xor_256_avx512(v[15], v[0]);
2976-  v[12] = xor_256_avx512(v[12], v[1]);
2977-  v[13] = xor_256_avx512(v[13], v[2]);
2978-  v[14] = xor_256_avx512(v[14], v[3]);
2979-  v[15] = rot16_256_avx512(v[15]);
2980-  v[12] = rot16_256_avx512(v[12]);
2981-  v[13] = rot16_256_avx512(v[13]);
2982-  v[14] = rot16_256_avx512(v[14]);
2983-  v[10] = add_256_avx512(v[10], v[15]);
2984-  v[11] = add_256_avx512(v[11], v[12]);
2985-  v[8]  = add_256_avx512(v[8], v[13]);
2986-  v[9]  = add_256_avx512(v[9], v[14]);
2987-  v[5]  = xor_256_avx512(v[5], v[10]);
2988-  v[6]  = xor_256_avx512(v[6], v[11]);
2989-  v[7]  = xor_256_avx512(v[7], v[8]);
2990-  v[4]  = xor_256_avx512(v[4], v[9]);
2991-  v[5]  = rot12_256_avx512(v[5]);
2992-  v[6]  = rot12_256_avx512(v[6]);
2993-  v[7]  = rot12_256_avx512(v[7]);
2994-  v[4]  = rot12_256_avx512(v[4]);
2995-  v[0]  = add_256_avx512(v[0], m[(size_t)MSG_SCHEDULE[r][9]]);
2996-  v[1]  = add_256_avx512(v[1], m[(size_t)MSG_SCHEDULE[r][11]]);
2997-  v[2]  = add_256_avx512(v[2], m[(size_t)MSG_SCHEDULE[r][13]]);
2998-  v[3]  = add_256_avx512(v[3], m[(size_t)MSG_SCHEDULE[r][15]]);
2999-  v[0]  = add_256_avx512(v[0], v[5]);
3000-  v[1]  = add_256_avx512(v[1], v[6]);
3001-  v[2]  = add_256_avx512(v[2], v[7]);
3002-  v[3]  = add_256_avx512(v[3], v[4]);
3003-  v[15] = xor_256_avx512(v[15], v[0]);
3004-  v[12] = xor_256_avx512(v[12], v[1]);
3005-  v[13] = xor_256_avx512(v[13], v[2]);
3006-  v[14] = xor_256_avx512(v[14], v[3]);
3007-  v[15] = rot8_256_avx512(v[15]);
3008-  v[12] = rot8_256_avx512(v[12]);
3009-  v[13] = rot8_256_avx512(v[13]);
3010-  v[14] = rot8_256_avx512(v[14]);
3011-  v[10] = add_256_avx512(v[10], v[15]);
3012-  v[11] = add_256_avx512(v[11], v[12]);
3013-  v[8]  = add_256_avx512(v[8], v[13]);
3014-  v[9]  = add_256_avx512(v[9], v[14]);
3015-  v[5]  = xor_256_avx512(v[5], v[10]);
3016-  v[6]  = xor_256_avx512(v[6], v[11]);
3017-  v[7]  = xor_256_avx512(v[7], v[8]);
3018-  v[4]  = xor_256_avx512(v[4], v[9]);
3019-  v[5]  = rot7_256_avx512(v[5]);
3020-  v[6]  = rot7_256_avx512(v[6]);
3021-  v[7]  = rot7_256_avx512(v[7]);
3022-  v[4]  = rot7_256_avx512(v[4]);
3023-}
3024-
3025-static inline void
3026-round_fn16_avx512(__m512i v[16], __m512i m[16], size_t r)
3027-{
3028-  v[0]  = add_512_avx512(v[0], m[(size_t)MSG_SCHEDULE[r][0]]);
3029-  v[1]  = add_512_avx512(v[1], m[(size_t)MSG_SCHEDULE[r][2]]);
3030-  v[2]  = add_512_avx512(v[2], m[(size_t)MSG_SCHEDULE[r][4]]);
3031-  v[3]  = add_512_avx512(v[3], m[(size_t)MSG_SCHEDULE[r][6]]);
3032-  v[0]  = add_512_avx512(v[0], v[4]);
3033-  v[1]  = add_512_avx512(v[1], v[5]);
3034-  v[2]  = add_512_avx512(v[2], v[6]);
3035-  v[3]  = add_512_avx512(v[3], v[7]);
3036-  v[12] = xor_512_avx512(v[12], v[0]);
3037-  v[13] = xor_512_avx512(v[13], v[1]);
3038-  v[14] = xor_512_avx512(v[14], v[2]);
3039-  v[15] = xor_512_avx512(v[15], v[3]);
3040-  v[12] = rot16_512_avx512(v[12]);
3041-  v[13] = rot16_512_avx512(v[13]);
3042-  v[14] = rot16_512_avx512(v[14]);
3043-  v[15] = rot16_512_avx512(v[15]);
3044-  v[8]  = add_512_avx512(v[8], v[12]);
3045-  v[9]  = add_512_avx512(v[9], v[13]);
3046-  v[10] = add_512_avx512(v[10], v[14]);
3047-  v[11] = add_512_avx512(v[11], v[15]);
3048-  v[4]  = xor_512_avx512(v[4], v[8]);
3049-  v[5]  = xor_512_avx512(v[5], v[9]);
3050-  v[6]  = xor_512_avx512(v[6], v[10]);
3051-  v[7]  = xor_512_avx512(v[7], v[11]);
3052-  v[4]  = rot12_512_avx512(v[4]);
3053-  v[5]  = rot12_512_avx512(v[5]);
3054-  v[6]  = rot12_512_avx512(v[6]);
3055-  v[7]  = rot12_512_avx512(v[7]);
3056-  v[0]  = add_512_avx512(v[0], m[(size_t)MSG_SCHEDULE[r][1]]);
3057-  v[1]  = add_512_avx512(v[1], m[(size_t)MSG_SCHEDULE[r][3]]);
3058-  v[2]  = add_512_avx512(v[2], m[(size_t)MSG_SCHEDULE[r][5]]);
3059-  v[3]  = add_512_avx512(v[3], m[(size_t)MSG_SCHEDULE[r][7]]);
3060-  v[0]  = add_512_avx512(v[0], v[4]);
3061-  v[1]  = add_512_avx512(v[1], v[5]);
3062-  v[2]  = add_512_avx512(v[2], v[6]);
3063-  v[3]  = add_512_avx512(v[3], v[7]);
3064-  v[12] = xor_512_avx512(v[12], v[0]);
3065-  v[13] = xor_512_avx512(v[13], v[1]);
3066-  v[14] = xor_512_avx512(v[14], v[2]);
3067-  v[15] = xor_512_avx512(v[15], v[3]);
3068-  v[12] = rot8_512_avx512(v[12]);
3069-  v[13] = rot8_512_avx512(v[13]);
3070-  v[14] = rot8_512_avx512(v[14]);
3071-  v[15] = rot8_512_avx512(v[15]);
3072-  v[8]  = add_512_avx512(v[8], v[12]);
3073-  v[9]  = add_512_avx512(v[9], v[13]);
3074-  v[10] = add_512_avx512(v[10], v[14]);
3075-  v[11] = add_512_avx512(v[11], v[15]);
3076-  v[4]  = xor_512_avx512(v[4], v[8]);
3077-  v[5]  = xor_512_avx512(v[5], v[9]);
3078-  v[6]  = xor_512_avx512(v[6], v[10]);
3079-  v[7]  = xor_512_avx512(v[7], v[11]);
3080-  v[4]  = rot7_512_avx512(v[4]);
3081-  v[5]  = rot7_512_avx512(v[5]);
3082-  v[6]  = rot7_512_avx512(v[6]);
3083-  v[7]  = rot7_512_avx512(v[7]);
3084-
3085-  v[0]  = add_512_avx512(v[0], m[(size_t)MSG_SCHEDULE[r][8]]);
3086-  v[1]  = add_512_avx512(v[1], m[(size_t)MSG_SCHEDULE[r][10]]);
3087-  v[2]  = add_512_avx512(v[2], m[(size_t)MSG_SCHEDULE[r][12]]);
3088-  v[3]  = add_512_avx512(v[3], m[(size_t)MSG_SCHEDULE[r][14]]);
3089-  v[0]  = add_512_avx512(v[0], v[5]);
3090-  v[1]  = add_512_avx512(v[1], v[6]);
3091-  v[2]  = add_512_avx512(v[2], v[7]);
3092-  v[3]  = add_512_avx512(v[3], v[4]);
3093-  v[15] = xor_512_avx512(v[15], v[0]);
3094-  v[12] = xor_512_avx512(v[12], v[1]);
3095-  v[13] = xor_512_avx512(v[13], v[2]);
3096-  v[14] = xor_512_avx512(v[14], v[3]);
3097-  v[15] = rot16_512_avx512(v[15]);
3098-  v[12] = rot16_512_avx512(v[12]);
3099-  v[13] = rot16_512_avx512(v[13]);
3100-  v[14] = rot16_512_avx512(v[14]);
3101-  v[10] = add_512_avx512(v[10], v[15]);
3102-  v[11] = add_512_avx512(v[11], v[12]);
3103-  v[8]  = add_512_avx512(v[8], v[13]);
3104-  v[9]  = add_512_avx512(v[9], v[14]);
3105-  v[5]  = xor_512_avx512(v[5], v[10]);
3106-  v[6]  = xor_512_avx512(v[6], v[11]);
3107-  v[7]  = xor_512_avx512(v[7], v[8]);
3108-  v[4]  = xor_512_avx512(v[4], v[9]);
3109-  v[5]  = rot12_512_avx512(v[5]);
3110-  v[6]  = rot12_512_avx512(v[6]);
3111-  v[7]  = rot12_512_avx512(v[7]);
3112-  v[4]  = rot12_512_avx512(v[4]);
3113-  v[0]  = add_512_avx512(v[0], m[(size_t)MSG_SCHEDULE[r][9]]);
3114-  v[1]  = add_512_avx512(v[1], m[(size_t)MSG_SCHEDULE[r][11]]);
3115-  v[2]  = add_512_avx512(v[2], m[(size_t)MSG_SCHEDULE[r][13]]);
3116-  v[3]  = add_512_avx512(v[3], m[(size_t)MSG_SCHEDULE[r][15]]);
3117-  v[0]  = add_512_avx512(v[0], v[5]);
3118-  v[1]  = add_512_avx512(v[1], v[6]);
3119-  v[2]  = add_512_avx512(v[2], v[7]);
3120-  v[3]  = add_512_avx512(v[3], v[4]);
3121-  v[15] = xor_512_avx512(v[15], v[0]);
3122-  v[12] = xor_512_avx512(v[12], v[1]);
3123-  v[13] = xor_512_avx512(v[13], v[2]);
3124-  v[14] = xor_512_avx512(v[14], v[3]);
3125-  v[15] = rot8_512_avx512(v[15]);
3126-  v[12] = rot8_512_avx512(v[12]);
3127-  v[13] = rot8_512_avx512(v[13]);
3128-  v[14] = rot8_512_avx512(v[14]);
3129-  v[10] = add_512_avx512(v[10], v[15]);
3130-  v[11] = add_512_avx512(v[11], v[12]);
3131-  v[8]  = add_512_avx512(v[8], v[13]);
3132-  v[9]  = add_512_avx512(v[9], v[14]);
3133-  v[5]  = xor_512_avx512(v[5], v[10]);
3134-  v[6]  = xor_512_avx512(v[6], v[11]);
3135-  v[7]  = xor_512_avx512(v[7], v[8]);
3136-  v[4]  = xor_512_avx512(v[4], v[9]);
3137-  v[5]  = rot7_512_avx512(v[5]);
3138-  v[6]  = rot7_512_avx512(v[6]);
3139-  v[7]  = rot7_512_avx512(v[7]);
3140-  v[4]  = rot7_512_avx512(v[4]);
3141-}
3142-
3143-#define LO_IMM8 0x88
3144-#define HI_IMM8 0xdd
3145-
3146-static inline __m512i
3147-unpack_lo_128_avx512(__m512i a, __m512i b)
3148-{
3149-  return _mm512_shuffle_i32x4(a, b, LO_IMM8);
3150-}
3151-
3152-static inline __m512i
3153-unpack_hi_128_avx512(__m512i a, __m512i b)
3154-{
3155-  return _mm512_shuffle_i32x4(a, b, HI_IMM8);
3156-}
3157-
3158-static inline void
3159-transpose_vecs_512_avx512(__m512i vecs[16])
3160-{
3161-  __m512i ab_0 = _mm512_unpacklo_epi32(vecs[0], vecs[1]);
3162-  __m512i ab_2 = _mm512_unpackhi_epi32(vecs[0], vecs[1]);
3163-  __m512i cd_0 = _mm512_unpacklo_epi32(vecs[2], vecs[3]);
3164-  __m512i cd_2 = _mm512_unpackhi_epi32(vecs[2], vecs[3]);
3165-  __m512i ef_0 = _mm512_unpacklo_epi32(vecs[4], vecs[5]);
3166-  __m512i ef_2 = _mm512_unpackhi_epi32(vecs[4], vecs[5]);
3167-  __m512i gh_0 = _mm512_unpacklo_epi32(vecs[6], vecs[7]);
3168-  __m512i gh_2 = _mm512_unpackhi_epi32(vecs[6], vecs[7]);
3169-  __m512i ij_0 = _mm512_unpacklo_epi32(vecs[8], vecs[9]);
3170-  __m512i ij_2 = _mm512_unpackhi_epi32(vecs[8], vecs[9]);
3171-  __m512i kl_0 = _mm512_unpacklo_epi32(vecs[10], vecs[11]);
3172-  __m512i kl_2 = _mm512_unpackhi_epi32(vecs[10], vecs[11]);
3173-  __m512i mn_0 = _mm512_unpacklo_epi32(vecs[12], vecs[13]);
3174-  __m512i mn_2 = _mm512_unpackhi_epi32(vecs[12], vecs[13]);
3175-  __m512i op_0 = _mm512_unpacklo_epi32(vecs[14], vecs[15]);
3176-  __m512i op_2 = _mm512_unpackhi_epi32(vecs[14], vecs[15]);
3177-
3178-  __m512i abcd_0 = _mm512_unpacklo_epi64(ab_0, cd_0);
3179-  __m512i abcd_1 = _mm512_unpackhi_epi64(ab_0, cd_0);
3180-  __m512i abcd_2 = _mm512_unpacklo_epi64(ab_2, cd_2);
3181-  __m512i abcd_3 = _mm512_unpackhi_epi64(ab_2, cd_2);
3182-  __m512i efgh_0 = _mm512_unpacklo_epi64(ef_0, gh_0);
3183-  __m512i efgh_1 = _mm512_unpackhi_epi64(ef_0, gh_0);
3184-  __m512i efgh_2 = _mm512_unpacklo_epi64(ef_2, gh_2);
3185-  __m512i efgh_3 = _mm512_unpackhi_epi64(ef_2, gh_2);
3186-  __m512i ijkl_0 = _mm512_unpacklo_epi64(ij_0, kl_0);
3187-  __m512i ijkl_1 = _mm512_unpackhi_epi64(ij_0, kl_0);
3188-  __m512i ijkl_2 = _mm512_unpacklo_epi64(ij_2, kl_2);
3189-  __m512i ijkl_3 = _mm512_unpackhi_epi64(ij_2, kl_2);
3190-  __m512i mnop_0 = _mm512_unpacklo_epi64(mn_0, op_0);
3191-  __m512i mnop_1 = _mm512_unpackhi_epi64(mn_0, op_0);
3192-  __m512i mnop_2 = _mm512_unpacklo_epi64(mn_2, op_2);
3193-  __m512i mnop_3 = _mm512_unpackhi_epi64(mn_2, op_2);
3194-
3195-  __m512i abcdefgh_0 = unpack_lo_128_avx512(abcd_0, efgh_0);
3196-  __m512i abcdefgh_1 = unpack_lo_128_avx512(abcd_1, efgh_1);
3197-  __m512i abcdefgh_2 = unpack_lo_128_avx512(abcd_2, efgh_2);
3198-  __m512i abcdefgh_3 = unpack_lo_128_avx512(abcd_3, efgh_3);
3199-  __m512i abcdefgh_4 = unpack_hi_128_avx512(abcd_0, efgh_0);
3200-  __m512i abcdefgh_5 = unpack_hi_128_avx512(abcd_1, efgh_1);
3201-  __m512i abcdefgh_6 = unpack_hi_128_avx512(abcd_2, efgh_2);
3202-  __m512i abcdefgh_7 = unpack_hi_128_avx512(abcd_3, efgh_3);
3203-  __m512i ijklmnop_0 = unpack_lo_128_avx512(ijkl_0, mnop_0);
3204-  __m512i ijklmnop_1 = unpack_lo_128_avx512(ijkl_1, mnop_1);
3205-  __m512i ijklmnop_2 = unpack_lo_128_avx512(ijkl_2, mnop_2);
3206-  __m512i ijklmnop_3 = unpack_lo_128_avx512(ijkl_3, mnop_3);
3207-  __m512i ijklmnop_4 = unpack_hi_128_avx512(ijkl_0, mnop_0);
3208-  __m512i ijklmnop_5 = unpack_hi_128_avx512(ijkl_1, mnop_1);
3209-  __m512i ijklmnop_6 = unpack_hi_128_avx512(ijkl_2, mnop_2);
3210-  __m512i ijklmnop_7 = unpack_hi_128_avx512(ijkl_3, mnop_3);
3211-
3212-  vecs[0]  = unpack_lo_128_avx512(abcdefgh_0, ijklmnop_0);
3213-  vecs[1]  = unpack_lo_128_avx512(abcdefgh_1, ijklmnop_1);
3214-  vecs[2]  = unpack_lo_128_avx512(abcdefgh_2, ijklmnop_2);
3215-  vecs[3]  = unpack_lo_128_avx512(abcdefgh_3, ijklmnop_3);
3216-  vecs[4]  = unpack_lo_128_avx512(abcdefgh_4, ijklmnop_4);
3217-  vecs[5]  = unpack_lo_128_avx512(abcdefgh_5, ijklmnop_5);
3218-  vecs[6]  = unpack_lo_128_avx512(abcdefgh_6, ijklmnop_6);
3219-  vecs[7]  = unpack_lo_128_avx512(abcdefgh_7, ijklmnop_7);
3220-  vecs[8]  = unpack_hi_128_avx512(abcdefgh_0, ijklmnop_0);
3221-  vecs[9]  = unpack_hi_128_avx512(abcdefgh_1, ijklmnop_1);
3222-  vecs[10] = unpack_hi_128_avx512(abcdefgh_2, ijklmnop_2);
3223-  vecs[11] = unpack_hi_128_avx512(abcdefgh_3, ijklmnop_3);
3224-  vecs[12] = unpack_hi_128_avx512(abcdefgh_4, ijklmnop_4);
3225-  vecs[13] = unpack_hi_128_avx512(abcdefgh_5, ijklmnop_5);
3226-  vecs[14] = unpack_hi_128_avx512(abcdefgh_6, ijklmnop_6);
3227-  vecs[15] = unpack_hi_128_avx512(abcdefgh_7, ijklmnop_7);
3228-}
3229-
3230-static inline void
3231-transpose_msg_vecs16_avx512(const uint8_t *const *inputs, size_t block_offset, __m512i out_msg[16])
3232-{
3233-  size_t i;
3234-
3235-  out_msg[0]  = loadu_512_avx512(&inputs[0][block_offset]);
3236-  out_msg[1]  = loadu_512_avx512(&inputs[1][block_offset]);
3237-  out_msg[2]  = loadu_512_avx512(&inputs[2][block_offset]);
3238-  out_msg[3]  = loadu_512_avx512(&inputs[3][block_offset]);
3239-  out_msg[4]  = loadu_512_avx512(&inputs[4][block_offset]);
3240-  out_msg[5]  = loadu_512_avx512(&inputs[5][block_offset]);
3241-  out_msg[6]  = loadu_512_avx512(&inputs[6][block_offset]);
3242-  out_msg[7]  = loadu_512_avx512(&inputs[7][block_offset]);
3243-  out_msg[8]  = loadu_512_avx512(&inputs[8][block_offset]);
3244-  out_msg[9]  = loadu_512_avx512(&inputs[9][block_offset]);
3245-  out_msg[10] = loadu_512_avx512(&inputs[10][block_offset]);
3246-  out_msg[11] = loadu_512_avx512(&inputs[11][block_offset]);
3247-  out_msg[12] = loadu_512_avx512(&inputs[12][block_offset]);
3248-  out_msg[13] = loadu_512_avx512(&inputs[13][block_offset]);
3249-  out_msg[14] = loadu_512_avx512(&inputs[14][block_offset]);
3250-  out_msg[15] = loadu_512_avx512(&inputs[15][block_offset]);
3251-
3252-  for (i = 0; i < 16; i++) {
3253-    _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0);
3254-  }
3255-  transpose_vecs_512_avx512(out_msg);
3256-}
3257-
3258-static inline void
3259-load_counters16_avx512(uint64_t counter, int increment_counter, __m512i *out_lo, __m512i *out_hi)
3260-{
3261-  const __m512i mask   = _mm512_set1_epi32(-increment_counter);
3262-  const __m512i deltas = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
3263-  const __m512i masked_deltas = _mm512_and_si512(deltas, mask);
3264-  const __m512i low_words = _mm512_add_epi32(_mm512_set1_epi32((int32_t)counter), masked_deltas);
3265-  const __m512i carries =
3266-      _mm512_srli_epi32(_mm512_andnot_si512(low_words, _mm512_set1_epi32((int32_t)counter)), 31);
3267-  const __m512i high_words = _mm512_add_epi32(_mm512_set1_epi32((int32_t)(counter >> 32)), carries);
3268-
3269-  *out_lo = low_words;
3270-  *out_hi = high_words;
3271-}
3272-
3273-void
3274-blake3_hash16_avx512(
3275-    const uint8_t *const *inputs,
3276-    size_t                blocks,
3277-    const uint32_t        key[8],
3278-    uint64_t              counter,
3279-    int                   increment_counter,
3280-    uint8_t               flags,
3281-    uint8_t               flags_start,
3282-    uint8_t               flags_end,
3283-    uint8_t              *out_bytes
3284-)
3285-{
3286-  __m512i h_vecs[8];
3287-  __m512i counter_low_vec, counter_high_vec;
3288-  uint8_t block_flags;
3289-  size_t  block;
3290-
3291-  h_vecs[0] = set1_512_avx512(key[0]);
3292-  h_vecs[1] = set1_512_avx512(key[1]);
3293-  h_vecs[2] = set1_512_avx512(key[2]);
3294-  h_vecs[3] = set1_512_avx512(key[3]);
3295-  h_vecs[4] = set1_512_avx512(key[4]);
3296-  h_vecs[5] = set1_512_avx512(key[5]);
3297-  h_vecs[6] = set1_512_avx512(key[6]);
3298-  h_vecs[7] = set1_512_avx512(key[7]);
3299-
3300-  load_counters16_avx512(counter, increment_counter, &counter_low_vec, &counter_high_vec);
3301-  block_flags = flags | flags_start;
3302-
3303-  for (block = 0; block < blocks; block++) {
3304-    __m512i block_len_vec;
3305-    __m512i block_flags_vec;
3306-    __m512i msg_vecs[16];
3307-    __m512i v[16];
3308-
3309-    if (block + 1 == blocks) {
3310-      block_flags |= flags_end;
3311-    }
3312-    block_len_vec   = set1_512_avx512(BLAKE3_BLOCK_LEN);
3313-    block_flags_vec = set1_512_avx512(block_flags);
3314-    transpose_msg_vecs16_avx512(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs);
3315-
3316-    v[0]  = h_vecs[0];
3317-    v[1]  = h_vecs[1];
3318-    v[2]  = h_vecs[2];
3319-    v[3]  = h_vecs[3];
3320-    v[4]  = h_vecs[4];
3321-    v[5]  = h_vecs[5];
3322-    v[6]  = h_vecs[6];
3323-    v[7]  = h_vecs[7];
3324-    v[8]  = set1_512_avx512(IV[0]);
3325-    v[9]  = set1_512_avx512(IV[1]);
3326-    v[10] = set1_512_avx512(IV[2]);
3327-    v[11] = set1_512_avx512(IV[3]);
3328-    v[12] = counter_low_vec;
3329-    v[13] = counter_high_vec;
3330-    v[14] = block_len_vec;
3331-    v[15] = block_flags_vec;
3332-
3333-    round_fn16_avx512(v, msg_vecs, 0);
3334-    round_fn16_avx512(v, msg_vecs, 1);
3335-    round_fn16_avx512(v, msg_vecs, 2);
3336-    round_fn16_avx512(v, msg_vecs, 3);
3337-    round_fn16_avx512(v, msg_vecs, 4);
3338-    round_fn16_avx512(v, msg_vecs, 5);
3339-    round_fn16_avx512(v, msg_vecs, 6);
3340-
3341-    h_vecs[0] = xor_512_avx512(v[0], v[8]);
3342-    h_vecs[1] = xor_512_avx512(v[1], v[9]);
3343-    h_vecs[2] = xor_512_avx512(v[2], v[10]);
3344-    h_vecs[3] = xor_512_avx512(v[3], v[11]);
3345-    h_vecs[4] = xor_512_avx512(v[4], v[12]);
3346-    h_vecs[5] = xor_512_avx512(v[5], v[13]);
3347-    h_vecs[6] = xor_512_avx512(v[6], v[14]);
3348-    h_vecs[7] = xor_512_avx512(v[7], v[15]);
3349-
3350-    block_flags = flags;
3351-  }
3352-
3353-  __m512i padded[16] = {
3354-      h_vecs[0],
3355-      h_vecs[1],
3356-      h_vecs[2],
3357-      h_vecs[3],
3358-      h_vecs[4],
3359-      h_vecs[5],
3360-      h_vecs[6],
3361-      h_vecs[7],
3362-      set1_512_avx512(0),
3363-      set1_512_avx512(0),
3364-      set1_512_avx512(0),
3365-      set1_512_avx512(0),
3366-      set1_512_avx512(0),
3367-      set1_512_avx512(0),
3368-      set1_512_avx512(0),
3369-      set1_512_avx512(0),
3370-  };
3371-  transpose_vecs_512_avx512(padded);
3372-  _mm256_mask_storeu_epi32(
3373-      &out_bytes[0 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[0])
3374-  );
3375-  _mm256_mask_storeu_epi32(
3376-      &out_bytes[1 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[1])
3377-  );
3378-  _mm256_mask_storeu_epi32(
3379-      &out_bytes[2 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[2])
3380-  );
3381-  _mm256_mask_storeu_epi32(
3382-      &out_bytes[3 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[3])
3383-  );
3384-  _mm256_mask_storeu_epi32(
3385-      &out_bytes[4 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[4])
3386-  );
3387-  _mm256_mask_storeu_epi32(
3388-      &out_bytes[5 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[5])
3389-  );
3390-  _mm256_mask_storeu_epi32(
3391-      &out_bytes[6 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[6])
3392-  );
3393-  _mm256_mask_storeu_epi32(
3394-      &out_bytes[7 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[7])
3395-  );
3396-  _mm256_mask_storeu_epi32(
3397-      &out_bytes[8 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[8])
3398-  );
3399-  _mm256_mask_storeu_epi32(
3400-      &out_bytes[9 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[9])
3401-  );
3402-  _mm256_mask_storeu_epi32(
3403-      &out_bytes[10 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[10])
3404-  );
3405-  _mm256_mask_storeu_epi32(
3406-      &out_bytes[11 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[11])
3407-  );
3408-  _mm256_mask_storeu_epi32(
3409-      &out_bytes[12 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[12])
3410-  );
3411-  _mm256_mask_storeu_epi32(
3412-      &out_bytes[13 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[13])
3413-  );
3414-  _mm256_mask_storeu_epi32(
3415-      &out_bytes[14 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[14])
3416-  );
3417-  _mm256_mask_storeu_epi32(
3418-      &out_bytes[15 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[15])
3419-  );
3420-}
3421-
3422-static inline void
3423-transpose_msg_vecs8_avx512(const uint8_t *const *inputs, size_t block_offset, __m256i out_msg[16])
3424-{
3425-  out_msg[0]  = loadu_256_avx512(&inputs[0][block_offset]);
3426-  out_msg[1]  = loadu_256_avx512(&inputs[1][block_offset]);
3427-  out_msg[2]  = loadu_256_avx512(&inputs[2][block_offset]);
3428-  out_msg[3]  = loadu_256_avx512(&inputs[3][block_offset]);
3429-  out_msg[4]  = loadu_256_avx512(&inputs[4][block_offset]);
3430-  out_msg[5]  = loadu_256_avx512(&inputs[5][block_offset]);
3431-  out_msg[6]  = loadu_256_avx512(&inputs[6][block_offset]);
3432-  out_msg[7]  = loadu_256_avx512(&inputs[7][block_offset]);
3433-  out_msg[8]  = loadu_256_avx512(&inputs[0][block_offset + 32]);
3434-  out_msg[9]  = loadu_256_avx512(&inputs[1][block_offset + 32]);
3435-  out_msg[10] = loadu_256_avx512(&inputs[2][block_offset + 32]);
3436-  out_msg[11] = loadu_256_avx512(&inputs[3][block_offset + 32]);
3437-  out_msg[12] = loadu_256_avx512(&inputs[4][block_offset + 32]);
3438-  out_msg[13] = loadu_256_avx512(&inputs[5][block_offset + 32]);
3439-  out_msg[14] = loadu_256_avx512(&inputs[6][block_offset + 32]);
3440-  out_msg[15] = loadu_256_avx512(&inputs[7][block_offset + 32]);
3441-
3442-  transpose_vecs_avx2(out_msg);
3443-}
3444-
3445-static inline void
3446-load_counters8_avx512(uint64_t counter, int increment_counter, __m256i *out_lo, __m256i *out_hi)
3447-{
3448-  const __m256i mask = _mm256_set1_epi32(-increment_counter);
3449-  const __m256i add0 = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
3450-  const __m256i add1 = _mm256_and_si256(mask, add0);
3451-  __m256i       l    = _mm256_add_epi32(_mm256_set1_epi32((int32_t)counter), add1);
3452-  __m256i       carry =
3453-      _mm256_srli_epi32(_mm256_andnot_si256(l, _mm256_set1_epi32((int32_t)counter)), 31);
3454-  __m256i h = _mm256_add_epi32(_mm256_set1_epi32((int32_t)(counter >> 32)), carry);
3455-
3456-  *out_lo = l;
3457-  *out_hi = h;
3458-}
3459-
3460-void
3461-blake3_hash8_avx512(
3462-    const uint8_t *const *inputs,
3463-    size_t                blocks,
3464-    const uint32_t        key[8],
3465-    uint64_t              counter,
3466-    int                   increment_counter,
3467-    uint8_t               flags,
3468-    uint8_t               flags_start,
3469-    uint8_t               flags_end,
3470-    uint8_t              *out_bytes
3471-)
3472-{
3473-  __m256i h_vecs[8];
3474-  __m256i counter_low_vec, counter_high_vec;
3475-  uint8_t block_flags;
3476-  size_t  block;
3477-
3478-  h_vecs[0] = set1_256_avx512(key[0]);
3479-  h_vecs[1] = set1_256_avx512(key[1]);
3480-  h_vecs[2] = set1_256_avx512(key[2]);
3481-  h_vecs[3] = set1_256_avx512(key[3]);
3482-  h_vecs[4] = set1_256_avx512(key[4]);
3483-  h_vecs[5] = set1_256_avx512(key[5]);
3484-  h_vecs[6] = set1_256_avx512(key[6]);
3485-  h_vecs[7] = set1_256_avx512(key[7]);
3486-
3487-  load_counters8_avx512(counter, increment_counter, &counter_low_vec, &counter_high_vec);
3488-  block_flags = flags | flags_start;
3489-
3490-  for (block = 0; block < blocks; block++) {
3491-    __m256i block_len_vec;
3492-    __m256i block_flags_vec;
3493-    __m256i msg_vecs[16];
3494-    __m256i v[16];
3495-
3496-    if (block + 1 == blocks) {
3497-      block_flags |= flags_end;
3498-    }
3499-    block_len_vec   = set1_256_avx512(BLAKE3_BLOCK_LEN);
3500-    block_flags_vec = set1_256_avx512(block_flags);
3501-    transpose_msg_vecs8_avx512(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs);
3502-
3503-    v[0]  = h_vecs[0];
3504-    v[1]  = h_vecs[1];
3505-    v[2]  = h_vecs[2];
3506-    v[3]  = h_vecs[3];
3507-    v[4]  = h_vecs[4];
3508-    v[5]  = h_vecs[5];
3509-    v[6]  = h_vecs[6];
3510-    v[7]  = h_vecs[7];
3511-    v[8]  = set1_256_avx512(IV[0]);
3512-    v[9]  = set1_256_avx512(IV[1]);
3513-    v[10] = set1_256_avx512(IV[2]);
3514-    v[11] = set1_256_avx512(IV[3]);
3515-    v[12] = counter_low_vec;
3516-    v[13] = counter_high_vec;
3517-    v[14] = block_len_vec;
3518-    v[15] = block_flags_vec;
3519-
3520-    round_fn8_avx512(v, msg_vecs, 0);
3521-    round_fn8_avx512(v, msg_vecs, 1);
3522-    round_fn8_avx512(v, msg_vecs, 2);
3523-    round_fn8_avx512(v, msg_vecs, 3);
3524-    round_fn8_avx512(v, msg_vecs, 4);
3525-    round_fn8_avx512(v, msg_vecs, 5);
3526-    round_fn8_avx512(v, msg_vecs, 6);
3527-
3528-    h_vecs[0] = xor_256_avx512(v[0], v[8]);
3529-    h_vecs[1] = xor_256_avx512(v[1], v[9]);
3530-    h_vecs[2] = xor_256_avx512(v[2], v[10]);
3531-    h_vecs[3] = xor_256_avx512(v[3], v[11]);
3532-    h_vecs[4] = xor_256_avx512(v[4], v[12]);
3533-    h_vecs[5] = xor_256_avx512(v[5], v[13]);
3534-    h_vecs[6] = xor_256_avx512(v[6], v[14]);
3535-    h_vecs[7] = xor_256_avx512(v[7], v[15]);
3536-
3537-    block_flags = flags;
3538-  }
3539-
3540-  transpose_vecs_avx2(h_vecs);
3541-  storeu_256_avx512(h_vecs[0], &out_bytes[0 * sizeof(__m256i)]);
3542-  storeu_256_avx512(h_vecs[1], &out_bytes[1 * sizeof(__m256i)]);
3543-  storeu_256_avx512(h_vecs[2], &out_bytes[2 * sizeof(__m256i)]);
3544-  storeu_256_avx512(h_vecs[3], &out_bytes[3 * sizeof(__m256i)]);
3545-  storeu_256_avx512(h_vecs[4], &out_bytes[4 * sizeof(__m256i)]);
3546-  storeu_256_avx512(h_vecs[5], &out_bytes[5 * sizeof(__m256i)]);
3547-  storeu_256_avx512(h_vecs[6], &out_bytes[6 * sizeof(__m256i)]);
3548-  storeu_256_avx512(h_vecs[7], &out_bytes[7 * sizeof(__m256i)]);
3549-}
3550-
3551-static inline void
3552-transpose_msg_vecs4_avx512(const uint8_t *const *inputs, size_t block_offset, __m128i out_msg[16])
3553-{
3554-  out_msg[0]  = loadu_128_avx512(&inputs[0][block_offset]);
3555-  out_msg[1]  = loadu_128_avx512(&inputs[1][block_offset]);
3556-  out_msg[2]  = loadu_128_avx512(&inputs[2][block_offset]);
3557-  out_msg[3]  = loadu_128_avx512(&inputs[3][block_offset]);
3558-  out_msg[4]  = loadu_128_avx512(&inputs[0][block_offset + 16]);
3559-  out_msg[5]  = loadu_128_avx512(&inputs[1][block_offset + 16]);
3560-  out_msg[6]  = loadu_128_avx512(&inputs[2][block_offset + 16]);
3561-  out_msg[7]  = loadu_128_avx512(&inputs[3][block_offset + 16]);
3562-  out_msg[8]  = loadu_128_avx512(&inputs[0][block_offset + 32]);
3563-  out_msg[9]  = loadu_128_avx512(&inputs[1][block_offset + 32]);
3564-  out_msg[10] = loadu_128_avx512(&inputs[2][block_offset + 32]);
3565-  out_msg[11] = loadu_128_avx512(&inputs[3][block_offset + 32]);
3566-  out_msg[12] = loadu_128_avx512(&inputs[0][block_offset + 48]);
3567-  out_msg[13] = loadu_128_avx512(&inputs[1][block_offset + 48]);
3568-  out_msg[14] = loadu_128_avx512(&inputs[2][block_offset + 48]);
3569-  out_msg[15] = loadu_128_avx512(&inputs[3][block_offset + 48]);
3570-
3571-  transpose_vecs_sse2(out_msg);
3572-}
3573-
3574-static inline void
3575-load_counters4_avx512(uint64_t counter, int increment_counter, __m128i *out_lo, __m128i *out_hi)
3576-{
3577-  const __m128i mask  = _mm_set1_epi32(-increment_counter);
3578-  const __m128i add0  = _mm_set_epi32(3, 2, 1, 0);
3579-  const __m128i add1  = _mm_and_si128(mask, add0);
3580-  __m128i       l     = _mm_add_epi32(_mm_set1_epi32((int32_t)counter), add1);
3581-  __m128i       carry = _mm_srli_epi32(_mm_andnot_si128(l, _mm_set1_epi32((int32_t)counter)), 31);
3582-  __m128i       h     = _mm_add_epi32(_mm_set1_epi32((int32_t)(counter >> 32)), carry);
3583-
3584-  *out_lo = l;
3585-  *out_hi = h;
3586-}
3587-
3588-void
3589-blake3_hash4_avx512(
3590-    const uint8_t *const *inputs,
3591-    size_t                blocks,
3592-    const uint32_t        key[8],
3593-    uint64_t              counter,
3594-    int                   increment_counter,
3595-    uint8_t               flags,
3596-    uint8_t               flags_start,
3597-    uint8_t               flags_end,
3598-    uint8_t              *out_bytes
3599-)
3600-{
3601-  __m128i h_vecs[8];
3602-  __m128i counter_low_vec, counter_high_vec;
3603-  uint8_t block_flags;
3604-  size_t  block;
3605-
3606-  h_vecs[0] = set1_128_avx512(key[0]);
3607-  h_vecs[1] = set1_128_avx512(key[1]);
3608-  h_vecs[2] = set1_128_avx512(key[2]);
3609-  h_vecs[3] = set1_128_avx512(key[3]);
3610-  h_vecs[4] = set1_128_avx512(key[4]);
3611-  h_vecs[5] = set1_128_avx512(key[5]);
3612-  h_vecs[6] = set1_128_avx512(key[6]);
3613-  h_vecs[7] = set1_128_avx512(key[7]);
3614-
3615-  load_counters4_avx512(counter, increment_counter, &counter_low_vec, &counter_high_vec);
3616-  block_flags = flags | flags_start;
3617-
3618-  for (block = 0; block < blocks; block++) {
3619-    __m128i block_len_vec;
3620-    __m128i block_flags_vec;
3621-    __m128i msg_vecs[16];
3622-    __m128i v[16];
3623-
3624-    if (block + 1 == blocks) {
3625-      block_flags |= flags_end;
3626-    }
3627-    block_len_vec   = set1_128_avx512(BLAKE3_BLOCK_LEN);
3628-    block_flags_vec = set1_128_avx512(block_flags);
3629-    transpose_msg_vecs4_avx512(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs);
3630-
3631-    v[0]  = h_vecs[0];
3632-    v[1]  = h_vecs[1];
3633-    v[2]  = h_vecs[2];
3634-    v[3]  = h_vecs[3];
3635-    v[4]  = h_vecs[4];
3636-    v[5]  = h_vecs[5];
3637-    v[6]  = h_vecs[6];
3638-    v[7]  = h_vecs[7];
3639-    v[8]  = set1_128_avx512(IV[0]);
3640-    v[9]  = set1_128_avx512(IV[1]);
3641-    v[10] = set1_128_avx512(IV[2]);
3642-    v[11] = set1_128_avx512(IV[3]);
3643-    v[12] = counter_low_vec;
3644-    v[13] = counter_high_vec;
3645-    v[14] = block_len_vec;
3646-    v[15] = block_flags_vec;
3647-
3648-    round_fn4_avx512(v, msg_vecs, 0);
3649-    round_fn4_avx512(v, msg_vecs, 1);
3650-    round_fn4_avx512(v, msg_vecs, 2);
3651-    round_fn4_avx512(v, msg_vecs, 3);
3652-    round_fn4_avx512(v, msg_vecs, 4);
3653-    round_fn4_avx512(v, msg_vecs, 5);
3654-    round_fn4_avx512(v, msg_vecs, 6);
3655-
3656-    h_vecs[0] = xor_128_avx512(v[0], v[8]);
3657-    h_vecs[1] = xor_128_avx512(v[1], v[9]);
3658-    h_vecs[2] = xor_128_avx512(v[2], v[10]);
3659-    h_vecs[3] = xor_128_avx512(v[3], v[11]);
3660-    h_vecs[4] = xor_128_avx512(v[4], v[12]);
3661-    h_vecs[5] = xor_128_avx512(v[5], v[13]);
3662-    h_vecs[6] = xor_128_avx512(v[6], v[14]);
3663-    h_vecs[7] = xor_128_avx512(v[7], v[15]);
3664-
3665-    block_flags = flags;
3666-  }
3667-
3668-  transpose_vecs_sse2(h_vecs);
3669-  storeu_128_avx512(h_vecs[0], &out_bytes[0 * sizeof(__m128i)]);
3670-  storeu_128_avx512(h_vecs[1], &out_bytes[2 * sizeof(__m128i)]);
3671-  storeu_128_avx512(h_vecs[2], &out_bytes[4 * sizeof(__m128i)]);
3672-  storeu_128_avx512(h_vecs[3], &out_bytes[6 * sizeof(__m128i)]);
3673-  storeu_128_avx512(h_vecs[4], &out_bytes[1 * sizeof(__m128i)]);
3674-  storeu_128_avx512(h_vecs[5], &out_bytes[3 * sizeof(__m128i)]);
3675-  storeu_128_avx512(h_vecs[6], &out_bytes[5 * sizeof(__m128i)]);
3676-  storeu_128_avx512(h_vecs[7], &out_bytes[7 * sizeof(__m128i)]);
3677-}
3678-
3679-static inline void
3680-hash_one_avx512(
3681-    const uint8_t *input,
3682-    size_t         blocks,
3683-    const uint32_t key[8],
3684-    uint64_t       counter,
3685-    uint8_t        flags,
3686-    uint8_t        flags_start,
3687-    uint8_t        flags_end,
3688-    uint8_t        out_bytes[BLAKE3_OUT_LEN]
3689-)
3690-{
3691-  uint32_t cv[8];
3692-  uint8_t  block_flags;
3693-
3694-  memcpy(cv, key, BLAKE3_KEY_LEN);
3695-  block_flags = flags | flags_start;
3696-  while (blocks > 0) {
3697-    if (blocks == 1) {
3698-      block_flags |= flags_end;
3699-    }
3700-    blake3_compress_in_place_avx512(cv, input, BLAKE3_BLOCK_LEN, counter, block_flags);
3701-    input = &input[BLAKE3_BLOCK_LEN];
3702-    blocks -= 1;
3703-    block_flags = flags;
3704-  }
3705-  memcpy(out_bytes, cv, BLAKE3_OUT_LEN);
3706-}
3707-
3708-void
3709-blake3_hash_many_avx512(
3710-    const uint8_t *const *inputs,
3711-    size_t                num_inputs,
3712-    size_t                blocks,
3713-    const uint32_t        key[8],
3714-    uint64_t              counter,
3715-    int                   increment_counter,
3716-    uint8_t               flags,
3717-    uint8_t               flags_start,
3718-    uint8_t               flags_end,
3719-    uint8_t              *out_bytes
3720-)
3721-{
3722-  while (num_inputs >= 16) {
3723-    blake3_hash16_avx512(
3724-        inputs, blocks, key, counter, increment_counter, flags, flags_start, flags_end, out_bytes
3725-    );
3726-    if (increment_counter) {
3727-      counter += 16;
3728-    }
3729-    inputs += 16;
3730-    num_inputs -= 16;
3731-    out_bytes = &out_bytes[16 * BLAKE3_OUT_LEN];
3732-  }
3733-  while (num_inputs >= 8) {
3734-    blake3_hash8_avx512(
3735-        inputs, blocks, key, counter, increment_counter, flags, flags_start, flags_end, out_bytes
3736-    );
3737-    if (increment_counter) {
3738-      counter += 8;
3739-    }
3740-    inputs += 8;
3741-    num_inputs -= 8;
3742-    out_bytes = &out_bytes[8 * BLAKE3_OUT_LEN];
3743-  }
3744-  while (num_inputs >= 4) {
3745-    blake3_hash4_avx512(
3746-        inputs, blocks, key, counter, increment_counter, flags, flags_start, flags_end, out_bytes
3747-    );
3748-    if (increment_counter) {
3749-      counter += 4;
3750-    }
3751-    inputs += 4;
3752-    num_inputs -= 4;
3753-    out_bytes = &out_bytes[4 * BLAKE3_OUT_LEN];
3754-  }
3755-  while (num_inputs > 0) {
3756-    hash_one_avx512(inputs[0], blocks, key, counter, flags, flags_start, flags_end, out_bytes);
3757-    if (increment_counter) {
3758-      counter += 1;
3759-    }
3760-    inputs += 1;
3761-    num_inputs -= 1;
3762-    out_bytes = &out_bytes[BLAKE3_OUT_LEN];
3763-  }
3764-}
3765-#if defined(__clang__)
3766-#pragma clang attribute pop
3767-#elif defined(__GNUC__)
3768-#pragma GCC pop_options
3769-#endif
3770-#endif
3771-
3772-/* dispatch functions */
3773-void
3774-blake3_compress_in_place(
3775-    uint32_t      cv[8],
3776-    const uint8_t block[BLAKE3_BLOCK_LEN],
3777-    uint8_t       block_len,
3778-    uint64_t      counter,
3779-    uint8_t       flags
3780-)
3781-{
3782-#if BLAKE3_X86_SIMD
3783-  if (!blake3_cpu_detected)
3784-    blake3_detect_cpu_features();
3785-  if (blake3_cpu_features & AVX512) {
3786-    blake3_compress_in_place_avx512(cv, block, block_len, counter, flags);
3787-    return;
3788-  }
3789-  if (blake3_cpu_features & SSE41) {
3790-    blake3_compress_in_place_sse41(cv, block, block_len, counter, flags);
3791-    return;
3792-  }
3793-  if (blake3_cpu_features & SSE2) {
3794-    blake3_compress_in_place_sse2(cv, block, block_len, counter, flags);
3795-    return;
3796-  }
3797-#endif
3798-  blake3_compress_in_place_portable(cv, block, block_len, counter, flags);
3799-}
3800-
3801-void
3802-blake3_compress_xof(
3803-    const uint32_t cv[8],
3804-    const uint8_t  block[BLAKE3_BLOCK_LEN],
3805-    uint8_t        block_len,
3806-    uint64_t       counter,
3807-    uint8_t        flags,
3808-    uint8_t        out[64]
3809-)
3810-{
3811-#if BLAKE3_X86_SIMD
3812-  if (!blake3_cpu_detected)
3813-    blake3_detect_cpu_features();
3814-  if (blake3_cpu_features & AVX512) {
3815-    blake3_compress_xof_avx512(cv, block, block_len, counter, flags, out);
3816-    return;
3817-  }
3818-  if (blake3_cpu_features & SSE41) {
3819-    blake3_compress_xof_sse41(cv, block, block_len, counter, flags, out);
3820-    return;
3821-  }
3822-  if (blake3_cpu_features & SSE2) {
3823-    blake3_compress_xof_sse2(cv, block, block_len, counter, flags, out);
3824-    return;
3825-  }
3826-#endif
3827-  blake3_compress_xof_portable(cv, block, block_len, counter, flags, out);
3828-}
3829-
3830-void
3831-blake3_hash_many(
3832-    const uint8_t *const *inputs,
3833-    size_t                num_inputs,
3834-    size_t                blocks,
3835-    const uint32_t        key[8],
3836-    uint64_t              counter,
3837-    int                   increment_counter,
3838-    uint8_t               flags,
3839-    uint8_t               flags_start,
3840-    uint8_t               flags_end,
3841-    uint8_t              *out
3842-)
3843-{
3844-#if BLAKE3_X86_SIMD
3845-  if (!blake3_cpu_detected)
3846-    blake3_detect_cpu_features();
3847-  if (blake3_cpu_features & AVX512) {
3848-    blake3_hash_many_avx512(
3849-        inputs,
3850-        num_inputs,
3851-        blocks,
3852-        key,
3853-        counter,
3854-        increment_counter,
3855-        flags,
3856-        flags_start,
3857-        flags_end,
3858-        out
3859-    );
3860-    return;
3861-  }
3862-  if (blake3_cpu_features & AVX2) {
3863-    blake3_hash_many_avx2(
3864-        inputs,
3865-        num_inputs,
3866-        blocks,
3867-        key,
3868-        counter,
3869-        increment_counter,
3870-        flags,
3871-        flags_start,
3872-        flags_end,
3873-        out
3874-    );
3875-    return;
3876-  }
3877-  if (blake3_cpu_features & SSE41) {
3878-    blake3_hash_many_sse41(
3879-        inputs,
3880-        num_inputs,
3881-        blocks,
3882-        key,
3883-        counter,
3884-        increment_counter,
3885-        flags,
3886-        flags_start,
3887-        flags_end,
3888-        out
3889-    );
3890-    return;
3891-  }
3892-  if (blake3_cpu_features & SSE2) {
3893-    blake3_hash_many_sse2(
3894-        inputs,
3895-        num_inputs,
3896-        blocks,
3897-        key,
3898-        counter,
3899-        increment_counter,
3900-        flags,
3901-        flags_start,
3902-        flags_end,
3903-        out
3904-    );
3905-    return;
3906-  }
3907-#endif
3908-  blake3_hash_many_portable(
3909-      inputs,
3910-      num_inputs,
3911-      blocks,
3912-      key,
3913-      counter,
3914-      increment_counter,
3915-      flags,
3916-      flags_start,
3917-      flags_end,
3918-      out
3919-  );
3920-}
3921-
3922-size_t
3923-blake3_simd_degree(void)
3924-{
3925-#if BLAKE3_X86_SIMD
3926-  if (!blake3_cpu_detected)
3927-    blake3_detect_cpu_features();
3928-  if (blake3_cpu_features & AVX512)
3929-    return 16;
3930-  if (blake3_cpu_features & AVX2)
3931-    return 8;
3932-  if (blake3_cpu_features & SSE41)
3933-    return 4;
3934-  if (blake3_cpu_features & SSE2)
3935-    return 4;
3936-#endif
3937-  return 1;
3938-}
3939-
3940-/* core hasher implementation */
3941-const char *
3942-blake3_version(void)
3943-{
3944-  return BLAKE3_VERSION_STRING;
3945-}
3946-
3947-static inline void
3948-chunk_state_init(struct Blake3ChunkState *self, const uint32_t key[8], uint8_t flags)
3949-{
3950-  memcpy(self->cv, key, BLAKE3_KEY_LEN);
3951-  self->chunk_counter = 0;
3952-  memset(self->buf, 0, BLAKE3_BLOCK_LEN);
3953-  self->buf_len           = 0;
3954-  self->blocks_compressed = 0;
3955-  self->flags             = flags;
3956-}
3957-
3958-static inline void
3959-chunk_state_reset(struct Blake3ChunkState *self, const uint32_t key[8], uint64_t chunk_counter)
3960-{
3961-  memcpy(self->cv, key, BLAKE3_KEY_LEN);
3962-  self->chunk_counter     = chunk_counter;
3963-  self->blocks_compressed = 0;
3964-  memset(self->buf, 0, BLAKE3_BLOCK_LEN);
3965-  self->buf_len = 0;
3966-}
3967-
3968-static inline size_t
3969-chunk_state_len(const struct Blake3ChunkState *self)
3970-{
3971-  return (BLAKE3_BLOCK_LEN * (size_t)self->blocks_compressed) + ((size_t)self->buf_len);
3972-}
3973-
3974-static inline size_t
3975-chunk_state_fill_buf(struct Blake3ChunkState *self, const uint8_t *input, size_t input_len)
3976-{
3977-  size_t   take = BLAKE3_BLOCK_LEN - ((size_t)self->buf_len);
3978-  uint8_t *dest;
3979-
3980-  if (take > input_len) {
3981-    take = input_len;
3982-  }
3983-  dest = self->buf + ((size_t)self->buf_len);
3984-  memcpy(dest, input, take);
3985-  self->buf_len += (uint8_t)take;
3986-  return take;
3987-}
3988-
3989-static inline uint8_t
3990-chunk_state_maybe_start_flag(const struct Blake3ChunkState *self)
3991-{
3992-  if (self->blocks_compressed == 0) {
3993-    return CHUNK_START;
3994-  } else {
3995-    return 0;
3996-  }
3997-}
3998-
3999-static inline struct Output
4000-make_output(
4001-    const uint32_t input_cv[8],
4002-    const uint8_t  block[BLAKE3_BLOCK_LEN],
4003-    uint8_t        block_len,
4004-    uint64_t       counter,
4005-    uint8_t        flags
4006-)
4007-{
4008-  struct Output ret;
4009-
4010-  memcpy(ret.input_cv, input_cv, 32);
4011-  memcpy(ret.block, block, BLAKE3_BLOCK_LEN);
4012-  ret.block_len = block_len;
4013-  ret.counter   = counter;
4014-  ret.flags     = flags;
4015-  return ret;
4016-}
4017-
4018-static inline void
4019-output_chaining_value(const struct Output *self, uint8_t cv[32])
4020-{
4021-  uint32_t cv_words[8];
4022-
4023-  memcpy(cv_words, self->input_cv, 32);
4024-  blake3_compress_in_place(cv_words, self->block, self->block_len, self->counter, self->flags);
4025-  store_cv_words(cv, cv_words);
4026-}
4027-
4028-static inline void
4029-output_root_bytes(const struct Output *self, uint64_t seek, uint8_t *out, size_t out_len)
4030-{
4031-  uint64_t output_block_counter = seek / 64;
4032-  size_t   offset_within_block  = seek % 64;
4033-  uint8_t  wide_buf[64];
4034-  size_t   available_bytes;
4035-  size_t   memcpy_len;
4036-
4037-  while (out_len > 0) {
4038-    blake3_compress_xof(
4039-        self->input_cv,
4040-        self->block,
4041-        self->block_len,
4042-        output_block_counter,
4043-        self->flags | ROOT,
4044-        wide_buf
4045-    );
4046-    available_bytes = 64 - offset_within_block;
4047-    if (out_len > available_bytes) {
4048-      memcpy_len = available_bytes;
4049-    } else {
4050-      memcpy_len = out_len;
4051-    }
4052-    memcpy(out, wide_buf + offset_within_block, memcpy_len);
4053-    out += memcpy_len;
4054-    out_len -= memcpy_len;
4055-    output_block_counter += 1;
4056-    offset_within_block = 0;
4057-  }
4058-}
4059-
4060-static inline void
4061-chunk_state_update(struct Blake3ChunkState *self, const uint8_t *input, size_t input_len)
4062-{
4063-  size_t take;
4064-
4065-  if (self->buf_len > 0) {
4066-    take = chunk_state_fill_buf(self, input, input_len);
4067-    input += take;
4068-    input_len -= take;
4069-    if (input_len > 0) {
4070-      blake3_compress_in_place(
4071-          self->cv,
4072-          self->buf,
4073-          BLAKE3_BLOCK_LEN,
4074-          self->chunk_counter,
4075-          self->flags | chunk_state_maybe_start_flag(self)
4076-      );
4077-      self->blocks_compressed += 1;
4078-      self->buf_len = 0;
4079-      memset(self->buf, 0, BLAKE3_BLOCK_LEN);
4080-    }
4081-  }
4082-
4083-  while (input_len > BLAKE3_BLOCK_LEN) {
4084-    blake3_compress_in_place(
4085-        self->cv,
4086-        input,
4087-        BLAKE3_BLOCK_LEN,
4088-        self->chunk_counter,
4089-        self->flags | chunk_state_maybe_start_flag(self)
4090-    );
4091-    self->blocks_compressed += 1;
4092-    input += BLAKE3_BLOCK_LEN;
4093-    input_len -= BLAKE3_BLOCK_LEN;
4094-  }
4095-
4096-  take = chunk_state_fill_buf(self, input, input_len);
4097-  input += take;
4098-  input_len -= take;
4099-}
4100-
4101-static inline struct Output
4102-chunk_state_output(const struct Blake3ChunkState *self)
4103-{
4104-  uint8_t block_flags = self->flags | chunk_state_maybe_start_flag(self) | CHUNK_END;
4105-
4106-  return make_output(self->cv, self->buf, self->buf_len, self->chunk_counter, block_flags);
4107-}
4108-
4109-static inline struct Output
4110-parent_output(const uint8_t block[BLAKE3_BLOCK_LEN], const uint32_t key[8], uint8_t flags)
4111-{
4112-  return make_output(key, block, BLAKE3_BLOCK_LEN, 0, flags | PARENT);
4113-}
4114-
4115-static unsigned int
4116-highest_one(uint64_t x)
4117-{
4118-#if defined(__GNUC__) || defined(__clang__)
4119-  return 63 ^ __builtin_clzll(x);
4120-#else
4121-  unsigned int c = 0;
4122-  if (x & 0xffffffff00000000ULL) {
4123-    x >>= 32;
4124-    c += 32;
4125-  }
4126-  if (x & 0x00000000ffff0000ULL) {
4127-    x >>= 16;
4128-    c += 16;
4129-  }
4130-  if (x & 0x000000000000ff00ULL) {
4131-    x >>= 8;
4132-    c += 8;
4133-  }
4134-  if (x & 0x00000000000000f0ULL) {
4135-    x >>= 4;
4136-    c += 4;
4137-  }
4138-  if (x & 0x000000000000000cULL) {
4139-    x >>= 2;
4140-    c += 2;
4141-  }
4142-  if (x & 0x0000000000000002ULL) {
4143-    c += 1;
4144-  }
4145-  return c;
4146-#endif
4147-}
4148-
4149-static inline uint64_t
4150-round_down_to_power_of_2(uint64_t x)
4151-{
4152-  return 1ULL << highest_one(x | 1);
4153-}
4154-
4155-static inline size_t
4156-left_len(size_t content_len)
4157-{
4158-  size_t full_chunks = (content_len - 1) / BLAKE3_CHUNK_LEN;
4159-
4160-  return round_down_to_power_of_2(full_chunks) * BLAKE3_CHUNK_LEN;
4161-}
4162-
4163-static inline size_t
4164-compress_chunks_parallel(
4165-    const uint8_t *input,
4166-    size_t         input_len,
4167-    const uint32_t key[8],
4168-    uint64_t       chunk_counter,
4169-    uint8_t        flags,
4170-    uint8_t       *out
4171-)
4172-{
4173-  const uint8_t *chunks_array[MAX_SIMD_DEGREE];
4174-  size_t         input_position   = 0;
4175-  size_t         chunks_array_len = 0;
4176-
4177-  assert(0 < input_len);
4178-  assert(input_len <= MAX_SIMD_DEGREE * BLAKE3_CHUNK_LEN);
4179-
4180-  while (input_len - input_position >= BLAKE3_CHUNK_LEN) {
4181-    chunks_array[chunks_array_len] = &input[input_position];
4182-    input_position += BLAKE3_CHUNK_LEN;
4183-    chunks_array_len += 1;
4184-  }
4185-
4186-  blake3_hash_many(
4187-      chunks_array,
4188-      chunks_array_len,
4189-      BLAKE3_CHUNK_LEN / BLAKE3_BLOCK_LEN,
4190-      key,
4191-      chunk_counter,
4192-      1,
4193-      flags,
4194-      CHUNK_START,
4195-      CHUNK_END,
4196-      out
4197-  );
4198-
4199-  if (input_len > input_position) {
4200-    uint64_t                counter = chunk_counter + (uint64_t)chunks_array_len;
4201-    struct Blake3ChunkState chunk_state;
4202-    struct Output           output;
4203-
4204-    chunk_state_init(&chunk_state, key, flags);
4205-    chunk_state.chunk_counter = counter;
4206-    chunk_state_update(&chunk_state, &input[input_position], input_len - input_position);
4207-    output = chunk_state_output(&chunk_state);
4208-    output_chaining_value(&output, &out[chunks_array_len * BLAKE3_OUT_LEN]);
4209-    return chunks_array_len + 1;
4210-  } else {
4211-    return chunks_array_len;
4212-  }
4213-}
4214-
4215-static inline size_t
4216-compress_parents_parallel(
4217-    const uint8_t *child_chaining_values,
4218-    size_t         num_chaining_values,
4219-    const uint32_t key[8],
4220-    uint8_t        flags,
4221-    uint8_t       *out
4222-)
4223-{
4224-  const uint8_t *parents_array[MAX_SIMD_DEGREE_OR_2];
4225-  size_t         parents_array_len = 0;
4226-
4227-  assert(2 <= num_chaining_values);
4228-  assert(num_chaining_values <= 2 * MAX_SIMD_DEGREE_OR_2);
4229-
4230-  while (num_chaining_values - (2 * parents_array_len) >= 2) {
4231-    parents_array[parents_array_len] =
4232-        &child_chaining_values[2 * parents_array_len * BLAKE3_OUT_LEN];
4233-    parents_array_len += 1;
4234-  }
4235-
4236-  blake3_hash_many(parents_array, parents_array_len, 1, key, 0, 0, flags | PARENT, 0, 0, out);
4237-
4238-  if (num_chaining_values > 2 * parents_array_len) {
4239-    memcpy(
4240-        &out[parents_array_len * BLAKE3_OUT_LEN],
4241-        &child_chaining_values[2 * parents_array_len * BLAKE3_OUT_LEN],
4242-        BLAKE3_OUT_LEN
4243-    );
4244-    return parents_array_len + 1;
4245-  } else {
4246-    return parents_array_len;
4247-  }
4248-}
4249-
4250-static inline size_t
4251-blake3_compress_subtree_wide(
4252-    const uint8_t *input,
4253-    size_t         input_len,
4254-    const uint32_t key[8],
4255-    uint64_t       chunk_counter,
4256-    uint8_t        flags,
4257-    uint8_t       *out
4258-)
4259-{
4260-  size_t degree;
4261-
4262-  if (input_len <= (size_t)blake3_simd_degree() * BLAKE3_CHUNK_LEN) {
4263-    return compress_chunks_parallel(input, input_len, key, chunk_counter, flags, out);
4264-  }
4265-
4266-  degree = blake3_simd_degree();
4267-  if (degree > 1) {
4268-    size_t child_len   = round_down_to_power_of_2(input_len - 1) / degree;
4269-    size_t cvs_written = 0;
4270-
4271-    if (child_len < BLAKE3_CHUNK_LEN) {
4272-      child_len = BLAKE3_CHUNK_LEN;
4273-    }
4274-
4275-    while (input_len > 0) {
4276-      size_t take = child_len;
4277-      size_t sub_cvs;
4278-
4279-      if (take > input_len) {
4280-        take = input_len;
4281-      }
4282-      sub_cvs = blake3_compress_subtree_wide(
4283-          input, take, key, chunk_counter, flags, &out[cvs_written * BLAKE3_OUT_LEN]
4284-      );
4285-      cvs_written += sub_cvs;
4286-      chunk_counter += take / BLAKE3_CHUNK_LEN;
4287-      input += take;
4288-      input_len -= take;
4289-    }
4290-
4291-    while (cvs_written > 2) {
4292-      cvs_written = compress_parents_parallel(out, cvs_written, key, flags, out);
4293-    }
4294-    return cvs_written;
4295-  } else {
4296-    /* fallback when simd degree is 1 */
4297-    size_t left      = left_len(input_len);
4298-    size_t right     = input_len - left;
4299-    size_t left_cvs  = blake3_compress_subtree_wide(input, left, key, chunk_counter, flags, out);
4300-    size_t right_cvs = blake3_compress_subtree_wide(
4301-        input + left,
4302-        right,
4303-        key,
4304-        chunk_counter + (left / BLAKE3_CHUNK_LEN),
4305-        flags,
4306-        &out[left_cvs * BLAKE3_OUT_LEN]
4307-    );
4308-
4309-    return left_cvs + right_cvs;
4310-  }
4311-}
4312-
4313-static void
4314-compress_subtree_to_parent_node(
4315-    const uint8_t *input,
4316-    size_t         input_len,
4317-    const uint32_t key[8],
4318-    uint64_t       chunk_counter,
4319-    uint8_t        flags,
4320-    uint8_t        out[2 * BLAKE3_OUT_LEN]
4321-)
4322-{
4323-  uint8_t cv_array[2 * MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN];
4324-  size_t  num_cvs =
4325-      blake3_compress_subtree_wide(input, input_len, key, chunk_counter, flags, cv_array);
4326-
4327-  assert(num_cvs >= 2);
4328-  while (num_cvs > 2) {
4329-    num_cvs = compress_parents_parallel(cv_array, num_cvs, key, flags, cv_array);
4330-  }
4331-  memcpy(out, cv_array, 2 * BLAKE3_OUT_LEN);
4332-}
4333-
4334-static inline void
4335-hasher_init_base(struct Blake3Hasher *self, const uint32_t key[8], uint8_t flags)
4336-{
4337-  memcpy(self->key, key, BLAKE3_KEY_LEN);
4338-  chunk_state_init(&self->chunk, key, flags);
4339-  self->cv_stack_len = 0;
4340-}
4341-
4342-void
4343-blake3_hasher_init(struct Blake3Hasher *self)
4344-{
4345-  if (!blake3_cpu_detected)
4346-    blake3_detect_cpu_features();
4347-  hasher_init_base(self, IV, 0);
4348-}
4349-
4350-static inline void
4351-load_key_words(const uint8_t key[BLAKE3_KEY_LEN], uint32_t key_words[8])
4352-{
4353-  key_words[0] = load32(&key[0 * 4]);
4354-  key_words[1] = load32(&key[1 * 4]);
4355-  key_words[2] = load32(&key[2 * 4]);
4356-  key_words[3] = load32(&key[3 * 4]);
4357-  key_words[4] = load32(&key[4 * 4]);
4358-  key_words[5] = load32(&key[5 * 4]);
4359-  key_words[6] = load32(&key[6 * 4]);
4360-  key_words[7] = load32(&key[7 * 4]);
4361-}
4362-
4363-void
4364-blake3_hasher_init_keyed(struct Blake3Hasher *self, const uint8_t key[BLAKE3_KEY_LEN])
4365-{
4366-  uint32_t key_words[8];
4367-
4368-  load_key_words(key, key_words);
4369-  hasher_init_base(self, key_words, KEYED_HASH);
4370-}
4371-
4372-void
4373-blake3_hasher_init_derive_key_raw(
4374-    struct Blake3Hasher *self, const void *context, size_t context_len
4375-)
4376-{
4377-  struct Blake3Hasher context_hasher;
4378-  uint8_t             context_key[BLAKE3_KEY_LEN];
4379-  uint32_t            context_key_words[8];
4380-
4381-  hasher_init_base(&context_hasher, IV, DERIVE_KEY_CONTEXT);
4382-  blake3_hasher_update(&context_hasher, context, context_len);
4383-  blake3_hasher_finalize(&context_hasher, context_key, BLAKE3_KEY_LEN);
4384-  load_key_words(context_key, context_key_words);
4385-  hasher_init_base(self, context_key_words, DERIVE_KEY_MATERIAL);
4386-}
4387-
4388-static inline unsigned int
4389-popcnt(uint64_t x)
4390-{
4391-#if defined(__GNUC__) || defined(__clang__)
4392-  return __builtin_popcountll(x);
4393-#else
4394-  unsigned int count = 0;
4395-  while (x != 0) {
4396-    count += 1;
4397-    x &= x - 1;
4398-  }
4399-  return count;
4400-#endif
4401-}
4402-
4403-void
4404-blake3_hasher_init_derive_key(struct Blake3Hasher *self, const char *context)
4405-{
4406-  blake3_hasher_init_derive_key_raw(self, context, strlen(context));
4407-}
4408-
4409-static inline void
4410-hasher_merge_cv_stack(struct Blake3Hasher *self, uint64_t total_len)
4411-{
4412-  size_t post_merge_stack_len = (size_t)popcnt(total_len);
4413-
4414-  while (self->cv_stack_len > post_merge_stack_len) {
4415-    uint8_t      *parent_node = &self->cv_stack[(self->cv_stack_len - 2) * BLAKE3_OUT_LEN];
4416-    struct Output output      = parent_output(parent_node, self->key, self->chunk.flags);
4417-
4418-    output_chaining_value(&output, parent_node);
4419-    self->cv_stack_len -= 1;
4420-  }
4421-}
4422-
4423-static inline void
4424-hasher_push_cv(struct Blake3Hasher *self, uint8_t new_cv[BLAKE3_OUT_LEN], uint64_t chunk_counter)
4425-{
4426-  hasher_merge_cv_stack(self, chunk_counter);
4427-  memcpy(&self->cv_stack[self->cv_stack_len * BLAKE3_OUT_LEN], new_cv, BLAKE3_OUT_LEN);
4428-  self->cv_stack_len += 1;
4429-}
4430-
4431-void
4432-blake3_hasher_update(struct Blake3Hasher *self, const void *input, size_t input_len)
4433-{
4434-  const uint8_t *input_bytes;
4435-
4436-  if (input_len == 0) {
4437-    return;
4438-  }
4439-
4440-  input_bytes = (const uint8_t *)input;
4441-
4442-  if (chunk_state_len(&self->chunk) > 0) {
4443-    size_t take = BLAKE3_CHUNK_LEN - chunk_state_len(&self->chunk);
4444-
4445-    if (take > input_len) {
4446-      take = input_len;
4447-    }
4448-    chunk_state_update(&self->chunk, input_bytes, take);
4449-    input_bytes += take;
4450-    input_len -= take;
4451-    if (input_len > 0) {
4452-      struct Output output = chunk_state_output(&self->chunk);
4453-      uint8_t       chunk_cv[32];
4454-
4455-      output_chaining_value(&output, chunk_cv);
4456-      hasher_push_cv(self, chunk_cv, self->chunk.chunk_counter);
4457-      chunk_state_reset(&self->chunk, self->key, self->chunk.chunk_counter + 1);
4458-    } else {
4459-      return;
4460-    }
4461-  }
4462-
4463-  while (input_len > BLAKE3_CHUNK_LEN) {
4464-    size_t   subtree_len  = round_down_to_power_of_2(input_len);
4465-    uint64_t count_so_far = self->chunk.chunk_counter * BLAKE3_CHUNK_LEN;
4466-    uint64_t subtree_chunks;
4467-
4468-    while ((((uint64_t)(subtree_len - 1)) & count_so_far) != 0) {
4469-      subtree_len /= 2;
4470-    }
4471-    subtree_chunks = subtree_len / BLAKE3_CHUNK_LEN;
4472-    if (subtree_len <= BLAKE3_CHUNK_LEN) {
4473-      struct Blake3ChunkState chunk_state;
4474-      struct Output           output;
4475-      uint8_t                 cv[BLAKE3_OUT_LEN];
4476-
4477-      chunk_state_init(&chunk_state, self->key, self->chunk.flags);
4478-      chunk_state.chunk_counter = self->chunk.chunk_counter;
4479-      chunk_state_update(&chunk_state, input_bytes, subtree_len);
4480-      output = chunk_state_output(&chunk_state);
4481-      output_chaining_value(&output, cv);
4482-      hasher_push_cv(self, cv, chunk_state.chunk_counter);
4483-    } else {
4484-      uint8_t cv_pair[2 * BLAKE3_OUT_LEN];
4485-
4486-      compress_subtree_to_parent_node(
4487-          input_bytes, subtree_len, self->key, self->chunk.chunk_counter, self->chunk.flags, cv_pair
4488-      );
4489-      hasher_push_cv(self, cv_pair, self->chunk.chunk_counter);
4490-      hasher_push_cv(
4491-          self, &cv_pair[BLAKE3_OUT_LEN], self->chunk.chunk_counter + (subtree_chunks / 2)
4492-      );
4493-    }
4494-    self->chunk.chunk_counter += subtree_chunks;
4495-    input_bytes += subtree_len;
4496-    input_len -= subtree_len;
4497-  }
4498-
4499-  if (input_len > 0) {
4500-    chunk_state_update(&self->chunk, input_bytes, input_len);
4501-    hasher_merge_cv_stack(self, self->chunk.chunk_counter);
4502-  }
4503-}
4504-
4505-void
4506-blake3_hasher_finalize_seek(
4507-    const struct Blake3Hasher *self, uint64_t seek, uint8_t *out, size_t out_len
4508-)
4509-{
4510-  struct Output output;
4511-  size_t        cvs_remaining;
4512-
4513-  if (out_len == 0) {
4514-    return;
4515-  }
4516-
4517-  if (self->cv_stack_len == 0) {
4518-    output = chunk_state_output(&self->chunk);
4519-    output_root_bytes(&output, seek, out, out_len);
4520-    return;
4521-  }
4522-
4523-  if (chunk_state_len(&self->chunk) > 0) {
4524-    cvs_remaining = self->cv_stack_len;
4525-    output        = chunk_state_output(&self->chunk);
4526-  } else {
4527-    cvs_remaining = self->cv_stack_len - 2;
4528-    output = parent_output(&self->cv_stack[cvs_remaining * 32], self->key, self->chunk.flags);
4529-  }
4530-
4531-  while (cvs_remaining > 0) {
4532-    uint8_t parent_block[BLAKE3_BLOCK_LEN];
4533-
4534-    cvs_remaining -= 1;
4535-    memcpy(parent_block, &self->cv_stack[cvs_remaining * 32], 32);
4536-    output_chaining_value(&output, &parent_block[32]);
4537-    output = parent_output(parent_block, self->key, self->chunk.flags);
4538-  }
4539-  output_root_bytes(&output, seek, out, out_len);
4540-}
4541-
4542-void
4543-blake3_hasher_finalize(const struct Blake3Hasher *self, uint8_t *out, size_t out_len)
4544-{
4545-  blake3_hasher_finalize_seek(self, 0, out, out_len);
4546-}
4547+/* see LICENSE file for copyright and license details */
4548+#include "arg.h"
4549+#include "util.h"
4550+#include "../../shared/libblake3/blake3.h"
4551 
4552-void
4553-blake3_hasher_reset(struct Blake3Hasher *self)
4554-{
4555-  chunk_state_reset(&self->chunk, self->key, 0);
4556-  self->cv_stack_len = 0;
4557-}
4558+#include <stdio.h>
4559+#include <stdlib.h>
4560+#include <string.h>
4561 
4562-/* utility implementation */
4563 static unsigned char *out;
4564 static size_t         outlen = BLAKE3_OUT_LEN;
4565 
4566@@ -4598,18 +53,6 @@ sum(const char *name, FILE *file)
4567   return 0;
4568 }
4569 
4570-static int
4571-hexval(int c)
4572-{
4573-  if ('0' <= c && c <= '9')
4574-    return c - '0';
4575-  if ('a' <= c && c <= 'f')
4576-    return c - 'a' + 10;
4577-  if ('A' <= c && c <= 'F')
4578-    return c - 'A' + 10;
4579-  return -1;
4580-}
4581-
4582 static int
4583 checkfile(const char *name, const char *mode, const char *str, unsigned char *out_buf, size_t len)
4584 {