Check in pre-generated perlasm and error data files

This adds a tool for managing pre-generated files, aligning our CMake
and non-CMake builds. The plan is roughly:

The source of truth for the file lists will (eventually) be build.json.
This describes the build in terms of the files that we directly edit.

However, we have a two-phase build. First a pregeneration step
transforms some of the less convenient inputs into checked in files.
Notably perlasm files get expanded. This produces an equivalent JSON
structure with fewer inputs. The same tool then outputs that structure
into whatever build systems we want.

This initial version pre-generates err_data.c and perlasm files. I've
not wired up the various build formats, except for CMake (for the CMake
build to consume) and JSON (for generate_build_files.py to parse).
build.json is also, for now, only a subset of the build. Later changes

The upshot of all this is we no longer have a Perl build dependency!
Perl is now only needed when working on BoringSSL. It nearly removes the
Go one, but Go is still needed to run and (for now) build the tests.

To keep the generated files up-to-date, once this lands, I'll update our
CI to run `go run ./util/pregenerate -check` which asserts that all
generated files are correct. From there we can land the later changes in
this patch series that uses this more extensively. My eventual goal is
to replace generate_build_files.py altogether and the
"master-with-bazel" branch. Instead we'll just have sources.bzl,
sources.gni, etc. all checked into the tree directly. And then the
normal branch will just have both a CMake and Bazel build in it.

Update-Note: generate_build_files.py no longer generates assembly files
or err_data.c. Those are now checked into the tree directly.

Bug: 542
Change-Id: I71f5ff7417be811f8b7888b345279474e6b38ee9
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/67288
Reviewed-by: Bob Beck <bbe@google.com>
Commit-Queue: David Benjamin <davidben@google.com>

5 files changed, 856 insertions, 154 deletions

diff --git a/util/generate_build_files.py b/util/generate_build_files.py
index 1b34dc2..d564a17 100644
--- a/util/generate_build_files.py
+++ b/util/generate_build_files.py
@@ -23,46 +23,6 @@ import sys
 import json
 
 
-# OS_ARCH_COMBOS maps from OS and platform to the OpenSSL assembly "style" for
-# that platform and the extension used by asm files.
-#
-# TODO(https://crbug.com/boringssl/542): This probably should be a map, but some
-# downstream scripts import this to find what folders to add/remove from git.
-OS_ARCH_COMBOS = [
-    ('apple', 'aarch64', 'ios64', [], 'S'),
-    ('apple', 'x86', 'macosx', ['-fPIC'], 'S'),
-    ('apple', 'x86_64', 'macosx', [], 'S'),
-    ('linux', 'arm', 'linux32', [], 'S'),
-    ('linux', 'aarch64', 'linux64', [], 'S'),
-    ('linux', 'x86', 'elf', ['-fPIC'], 'S'),
-    ('linux', 'x86_64', 'elf', [], 'S'),
-    ('win', 'x86', 'win32n', [], 'asm'),
-    ('win', 'x86_64', 'nasm', [], 'asm'),
-    ('win', 'aarch64', 'win64', [], 'S'),
-]
-
-# NON_PERL_FILES enumerates assembly files that are not processed by the
-# perlasm system.
-NON_PERL_FILES = {
-    ('apple', 'x86_64'): [
-        'src/third_party/fiat/asm/fiat_curve25519_adx_mul.S',
-        'src/third_party/fiat/asm/fiat_curve25519_adx_square.S',
-        'src/third_party/fiat/asm/fiat_p256_adx_mul.S',
-        'src/third_party/fiat/asm/fiat_p256_adx_sqr.S',
-    ],
-    ('linux', 'arm'): [
-        'src/crypto/curve25519/asm/x25519-asm-arm.S',
-        'src/crypto/poly1305/poly1305_arm_asm.S',
-    ],
-    ('linux', 'x86_64'): [
-        'src/crypto/hrss/asm/poly_rq_mul.S',
-        'src/third_party/fiat/asm/fiat_curve25519_adx_mul.S',
-        'src/third_party/fiat/asm/fiat_curve25519_adx_square.S',
-        'src/third_party/fiat/asm/fiat_p256_adx_mul.S',
-        'src/third_party/fiat/asm/fiat_p256_adx_sqr.S',
-    ],
-}
-
 PREFIX = None
 EMBED_TEST_DATA = True
 
@@ -569,17 +529,6 @@ class JSON(object):
     with open('sources.json', 'w+') as f:
       json.dump(files, f, sort_keys=True, indent=2)
 
-def FindCMakeFiles(directory):
-  """Returns list of all CMakeLists.txt files recursively in directory."""
-  cmakefiles = []
-
-  for (path, _, filenames) in os.walk(directory):
-    for filename in filenames:
-      if filename == 'CMakeLists.txt':
-        cmakefiles.append(os.path.join(path, filename))
-
-  return cmakefiles
-
 def OnlyFIPSFragments(path, dent, is_dir):
   return is_dir or (path.startswith(
       os.path.join('src', 'crypto', 'fipsmodule', '')) and
@@ -679,85 +628,6 @@ def FindHeaderFiles(directory, filter_func):
   return hfiles
 
 
-def ExtractPerlAsmFromCMakeFile(cmakefile):
-  """Parses the contents of the CMakeLists.txt file passed as an argument and
-  returns a list of all the perlasm() directives found in the file."""
-  perlasms = []
-  with open(cmakefile) as f:
-    for line in f:
-      line = line.strip()
-      if not line.startswith('perlasm('):
-        continue
-      if not line.endswith(')'):
-        raise ValueError('Bad perlasm line in %s' % cmakefile)
-      # Remove "perlasm(" from start and ")" from end
-      params = line[8:-1].split()
-      if len(params) < 4:
-        raise ValueError('Bad perlasm line in %s' % cmakefile)
-      perlasms.append({
-          'arch': params[1],
-          'output': os.path.join(os.path.dirname(cmakefile), params[2]),
-          'input': os.path.join(os.path.dirname(cmakefile), params[3]),
-          'extra_args': params[4:],
-      })
-
-  return perlasms
-
-
-def ReadPerlAsmOperations():
-  """Returns a list of all perlasm() directives found in CMake config files in
-  src/."""
-  perlasms = []
-  cmakefiles = FindCMakeFiles('src')
-
-  for cmakefile in cmakefiles:
-    perlasms.extend(ExtractPerlAsmFromCMakeFile(cmakefile))
-
-  return perlasms
-
-
-def PerlAsm(output_filename, input_filename, perlasm_style, extra_args):
-  """Runs the a perlasm script and puts the output into output_filename."""
-  base_dir = os.path.dirname(output_filename)
-  if not os.path.isdir(base_dir):
-    os.makedirs(base_dir)
-  subprocess.check_call(
-      ['perl', input_filename, perlasm_style] + extra_args + [output_filename])
-
-
-def WriteAsmFiles(perlasms):
-  """Generates asm files from perlasm directives for each supported OS x
-  platform combination."""
-  asmfiles = {}
-
-  for perlasm in perlasms:
-    for (osname, arch, perlasm_style, extra_args, asm_ext) in OS_ARCH_COMBOS:
-      if arch != perlasm['arch']:
-        continue
-      # TODO(https://crbug.com/boringssl/542): Now that we incorporate osname in
-      # the output filename, the asm files can just go in a single directory.
-      # For now, we keep them in target-specific directories to avoid breaking
-      # downstream scripts.
-      key = (osname, arch)
-      outDir = '%s-%s' % key
-      output = perlasm['output']
-      if not output.startswith('src'):
-        raise ValueError('output missing src: %s' % output)
-      output = os.path.join(outDir, output[4:])
-      output = '%s-%s.%s' % (output, osname, asm_ext)
-      PerlAsm(output, perlasm['input'], perlasm_style,
-              extra_args + perlasm['extra_args'])
-      asmfiles.setdefault(key, []).append(output)
-
-  for (key, non_perl_asm_files) in NON_PERL_FILES.items():
-    asmfiles.setdefault(key, []).extend(non_perl_asm_files)
-
-  for files in asmfiles.values():
-    files.sort()
-
-  return asmfiles
-
-
 def ExtractVariablesFromCMakeFile(cmakefile):
   """Parses the contents of the CMakeLists.txt file passed as an argument and
   returns a dictionary of exported source lists."""
@@ -792,7 +662,12 @@ def PrefixWithSrc(files):
 
 
 def main(platforms):
+  # TODO(crbug.com/boringssl/542): Move everything to util/pregenerate and the
+  # new JSON file.
   cmake = ExtractVariablesFromCMakeFile(os.path.join('src', 'sources.cmake'))
+  with open(os.path.join('src', 'gen', 'sources.json')) as f:
+    sources = json.load(f)
+
   crypto_c_files = (FindCFiles(os.path.join('src', 'crypto'), NoTestsNorFIPSFragments) +
                     FindCFiles(os.path.join('src', 'third_party', 'fiat'), NoTestsNorFIPSFragments))
   fips_fragments = FindCFiles(os.path.join('src', 'crypto', 'fipsmodule'), OnlyFIPSFragments)
@@ -805,12 +680,7 @@ def main(platforms):
       os.path.join('src', 'crypto', 'fipsmodule', 'bcm.c')
   ]
 
-  # Generate err_data.c
-  with open('err_data.c', 'w+') as err_data:
-    subprocess.check_call(['go', 'run', 'err_data_generate.go'],
-                          cwd=os.path.join('src', 'crypto', 'err'),
-                          stdout=err_data)
-  crypto_c_files.append('err_data.c')
+  crypto_c_files += PrefixWithSrc(sources['crypto']['srcs'])
   crypto_c_files.sort()
 
   test_support_h_files = (
@@ -847,28 +717,19 @@ def main(platforms):
       FindHeaderFiles(os.path.join('src', 'crypto'), NoTests) +
       FindHeaderFiles(os.path.join('src', 'third_party', 'fiat'), NoTests))
 
-  asm_outputs = sorted(WriteAsmFiles(ReadPerlAsmOperations()).items())
-
-  # Generate combined source lists for gas and nasm. Some files appear in
-  # multiple per-platform lists, so we de-duplicate.
-  #
-  # TODO(https://crbug.com/boringssl/542): It would be simpler to build the
-  # combined source lists directly. This is a remnant of the previous assembly
-  # strategy. When we move to pre-generated assembly files, this will be
-  # removed.
-  asm_sources = set()
-  nasm_sources = set()
-  for ((osname, arch), asm_files) in asm_outputs:
-    if (osname, arch) in (('win', 'x86'), ('win', 'x86_64')):
-      nasm_sources.update(asm_files)
-    else:
-      asm_sources.update(asm_files)
+  # TODO(crbug.com/boringssl/542): generate_build_files.py historically reported
+  # all the assembly files as part of libcrypto. Merge them for now, but we
+  # should split them out later.
+  crypto_asm = sorted(sources['bcm']['asm'] + sources['crypto']['asm'] +
+                      sources['test_support']['asm'])
+  crypto_nasm = sorted(sources['bcm']['nasm'] + sources['crypto']['nasm'] +
+                       sources['test_support']['nasm'])
 
   files = {
       'bcm_crypto': bcm_crypto_c_files,
       'crypto': crypto_c_files,
-      'crypto_asm': sorted(list(asm_sources)),
-      'crypto_nasm': sorted(list(nasm_sources)),
+      'crypto_asm': PrefixWithSrc(crypto_asm),
+      'crypto_nasm': PrefixWithSrc(crypto_nasm),
       'crypto_headers': crypto_h_files,
       'crypto_internal_headers': crypto_internal_h_files,
       'crypto_test': crypto_test_files,
diff --git a/util/pregenerate/build.go b/util/pregenerate/build.go
new file mode 100644
index 0000000..5f60960
--- /dev/null
+++ b/util/pregenerate/build.go
@@ -0,0 +1,284 @@
+// Copyright (c) 2024, Google Inc.
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+package main
+
+import (
+	"bytes"
+	"cmp"
+	"encoding/json"
+	"fmt"
+	"path"
+	"slices"
+	"strings"
+)
+
+// An OutputTarget is a build target for consumption by the downstream build
+// systems. All pre-generated files are baked input its source lists.
+type OutputTarget struct {
+	// Srcs is the list of C or C++ files (determined by file extension) that are
+	// built into the target.
+	Srcs []string `json:"srcs,omitempty"`
+	// Hdrs is the list public headers that should be available to external
+	// projects using this target.
+	Hdrs []string `json:"hdrs,omitempty"`
+	// InternalHdrs is the list of internal headers that should be available to
+	// this target, as well as any internal targets using this target.
+	InternalHdrs []string `json:"internal_hdrs,omitempty"`
+	// Asm is the a list of assembly files to be passed to a gas-compatible
+	// assembler.
+	Asm []string `json:"asm,omitempty"`
+	// Nasm is the a list of assembly files to be passed to a nasm-compatible
+	// assembler.
+	Nasm []string `json:"nasm,omitempty"`
+	// Data is a list of test data files that should be available when the test is
+	// run.
+	Data []string `json:"data,omitempty"`
+}
+
+// An InputTarget is a build target with build inputs that still need to be
+// pregenerated.
+type InputTarget struct {
+	OutputTarget
+	// ErrData contains a list of errordata files to combine into err_data.c.
+	ErrData []string `json:"err_data,omitempty"`
+	// The following fields define perlasm sources for the corresponding
+	// architecture.
+	PerlasmAarch64 []PerlasmSource `json:"perlasm_aarch64,omitempty"`
+	PerlasmArm     []PerlasmSource `json:"perlasm_arm,omitempty"`
+	PerlasmX86     []PerlasmSource `json:"perlasm_x86,omitempty"`
+	PerlasmX86_64  []PerlasmSource `json:"perlasm_x86_64,omitempty"`
+}
+
+type PerlasmSource struct {
+	// Src the path to the input perlasm file.
+	Src string `json:"src"`
+	// Dst, if not empty, is base name of the destination file. If empty, this
+	// is determined from Src by default. It should be overriden if a single
+	// source file generates multiple functions (e.g. SHA-256 vs SHA-512) or
+	// multiple architectures (e.g. the "armx" files).
+	Dst string `json:"dst,omitempty"`
+	// Args is a list of extra parameters to pass to the script.
+	Args []string `json:"args,omitempty"`
+}
+
+// Pregenerate converts an input target to an output target. It returns the
+// result alongside a list of tasks that must be run to build the referenced
+// files.
+func (in *InputTarget) Pregenerate(name string) (out OutputTarget, tasks []Task) {
+	out = in.OutputTarget
+
+	// Make copies of any fields we will write to.
+	out.Srcs = slices.Clone(out.Srcs)
+	out.Asm = slices.Clone(out.Asm)
+	out.Nasm = slices.Clone(out.Nasm)
+
+	addTask := func(list *[]string, t Task) {
+		tasks = append(tasks, t)
+		*list = append(*list, t.Destination())
+	}
+
+	if len(in.ErrData) != 0 {
+		addTask(&out.Srcs, &ErrDataTask{TargetName: name, Inputs: in.ErrData})
+	}
+
+	addPerlasmTask := func(list *[]string, p *PerlasmSource, fileSuffix string, args []string) {
+		dst := p.Dst
+		if len(p.Dst) == 0 {
+			dst = strings.TrimSuffix(path.Base(p.Src), ".pl")
+		}
+		dst = path.Join("gen", name, dst+fileSuffix)
+		args = append(slices.Clone(args), p.Args...)
+		addTask(list, &PerlasmTask{Src: p.Src, Dst: dst, Args: args})
+	}
+
+	for _, p := range in.PerlasmAarch64 {
+		addPerlasmTask(&out.Asm, &p, "-apple.S", []string{"ios64"})
+		addPerlasmTask(&out.Asm, &p, "-linux.S", []string{"linux64"})
+		addPerlasmTask(&out.Asm, &p, "-win.S", []string{"win64"})
+	}
+	for _, p := range in.PerlasmArm {
+		addPerlasmTask(&out.Asm, &p, "-linux.S", []string{"linux32"})
+	}
+	for _, p := range in.PerlasmX86 {
+		addPerlasmTask(&out.Asm, &p, "-apple.S", []string{"macosx", "-fPIC", "-DOPENSSL_IA32_SSE2"})
+		addPerlasmTask(&out.Asm, &p, "-linux.S", []string{"elf", "-fPIC", "-DOPENSSL_IA32_SSE2"})
+		addPerlasmTask(&out.Nasm, &p, "-win.asm", []string{"win32n", "-fPIC", "-DOPENSSL_IA32_SSE2"})
+	}
+	for _, p := range in.PerlasmX86_64 {
+		addPerlasmTask(&out.Asm, &p, "-apple.S", []string{"macosx"})
+		addPerlasmTask(&out.Asm, &p, "-linux.S", []string{"elf"})
+		addPerlasmTask(&out.Nasm, &p, "-win.asm", []string{"nasm"})
+	}
+
+	// Re-sort the modified fields.
+	slices.Sort(out.Srcs)
+	slices.Sort(out.Asm)
+	slices.Sort(out.Nasm)
+
+	return
+}
+
+func sortedKeys[K cmp.Ordered, V any](m map[K]V) []K {
+	keys := make([]K, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	slices.Sort(keys)
+	return keys
+}
+
+func writeHeader(b *bytes.Buffer, comment string) {
+	fmt.Fprintf(b, "%s Copyright (c) 2024, Google Inc.\n", comment)
+	fmt.Fprintf(b, "%s\n", comment)
+	fmt.Fprintf(b, "%s Permission to use, copy, modify, and/or distribute this software for any\n", comment)
+	fmt.Fprintf(b, "%s purpose with or without fee is hereby granted, provided that the above\n", comment)
+	fmt.Fprintf(b, "%s copyright notice and this permission notice appear in all copies.\n", comment)
+	fmt.Fprintf(b, "%s\n", comment)
+	fmt.Fprintf(b, "%s THE SOFTWARE IS PROVIDED \"AS IS\" AND THE AUTHOR DISCLAIMS ALL WARRANTIES\n", comment)
+	fmt.Fprintf(b, "%s WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF\n", comment)
+	fmt.Fprintf(b, "%s MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY\n", comment)
+	fmt.Fprintf(b, "%s SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES\n", comment)
+	fmt.Fprintf(b, "%s WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION\n", comment)
+	fmt.Fprintf(b, "%s OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN\n", comment)
+	fmt.Fprintf(b, "%s CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.\n", comment)
+	fmt.Fprintf(b, "%s\n", comment)
+	fmt.Fprintf(b, "%s Generated by go ./util/pregenerate. Do not edit manually.\n", comment)
+}
+
+func buildVariablesTask(targets map[string]OutputTarget, dst, comment string, writeVariable func(b *bytes.Buffer, name string, val []string)) Task {
+	return NewSimpleTask(dst, func() ([]byte, error) {
+		var b bytes.Buffer
+		writeHeader(&b, comment)
+
+		for _, name := range sortedKeys(targets) {
+			target := targets[name]
+			if len(target.Srcs) != 0 {
+				writeVariable(&b, name+"_sources", target.Srcs)
+			}
+			if len(target.Hdrs) != 0 {
+				writeVariable(&b, name+"_headers", target.Hdrs)
+			}
+			if len(target.InternalHdrs) != 0 {
+				writeVariable(&b, name+"_internal_headers", target.InternalHdrs)
+			}
+			if len(target.Asm) != 0 {
+				writeVariable(&b, name+"_sources_asm", target.Asm)
+			}
+			if len(target.Nasm) != 0 {
+				writeVariable(&b, name+"_sources_nasm", target.Nasm)
+			}
+			if len(target.Data) != 0 {
+				writeVariable(&b, name+"_data", target.Data)
+			}
+		}
+
+		return b.Bytes(), nil
+	})
+}
+
+func writeBazelVariable(b *bytes.Buffer, name string, val []string) {
+	fmt.Fprintf(b, "\n%s = [\n", name)
+	for _, v := range val {
+		fmt.Fprintf(b, "  %q,\n", v)
+	}
+	fmt.Fprintf(b, "]\n")
+}
+
+func writeCMakeVariable(b *bytes.Buffer, name string, val []string) {
+	fmt.Fprintf(b, "\nset(\n")
+	fmt.Fprintf(b, "  %s\n\n", strings.ToUpper(name))
+	for _, v := range val {
+		fmt.Fprintf(b, "  %s\n", v)
+	}
+	fmt.Fprintf(b, ")\n")
+}
+
+func writeMakeVariable(b *bytes.Buffer, name string, val []string) {
+	fmt.Fprintf(b, "\n%s := \\\n", name)
+	for i, v := range val {
+		if i == len(val)-1 {
+			fmt.Fprintf(b, "  %s\n", v)
+		} else {
+			fmt.Fprintf(b, "  %s \\\n", v)
+		}
+	}
+}
+
+func writeGNVariable(b *bytes.Buffer, name string, val []string) {
+	// Bazel and GN have the same syntax similar syntax.
+	writeBazelVariable(b, name, val)
+}
+
+func jsonTask(targets map[string]OutputTarget, dst string) Task {
+	return NewSimpleTask(dst, func() ([]byte, error) {
+		return json.MarshalIndent(targets, "", "  ")
+	})
+}
+
+func soongTask(targets map[string]OutputTarget, dst string) Task {
+	return NewSimpleTask(dst, func() ([]byte, error) {
+		var b bytes.Buffer
+		writeHeader(&b, "//")
+
+		writeAttribute := func(indent, name string, val []string) {
+			fmt.Fprintf(&b, "%s%s: [\n", indent, name)
+			for _, v := range val {
+				fmt.Fprintf(&b, "%s    %q,\n", indent, v)
+			}
+			fmt.Fprintf(&b, "%s],\n", indent)
+
+		}
+
+		for _, name := range sortedKeys(targets) {
+			target := targets[name]
+			fmt.Fprintf(&b, "\ncc_defaults {\n")
+			fmt.Fprintf(&b, "    name: %q\n", "boringssl_"+name+"_sources")
+			if len(target.Srcs) != 0 {
+				writeAttribute("    ", "srcs", target.Srcs)
+			}
+			if len(target.Data) != 0 {
+				writeAttribute("    ", "data", target.Data)
+			}
+			if len(target.Asm) != 0 {
+				fmt.Fprintf(&b, "    target: {\n")
+				// Only emit asm for Linux. On Windows, BoringSSL requires NASM, which is
+				// not available in AOSP. On Darwin, the assembly works fine, but it
+				// conflicts with Android's FIPS build. See b/294399371.
+				fmt.Fprintf(&b, "        linux: {\n")
+				writeAttribute("            ", "srcs", target.Asm)
+				fmt.Fprintf(&b, "        },\n")
+				fmt.Fprintf(&b, "        darwin: {\n")
+				fmt.Fprintf(&b, "            cflags: [\"-DOPENSSL_NO_ASM\"],\n")
+				fmt.Fprintf(&b, "        },\n")
+				fmt.Fprintf(&b, "        windows: {\n")
+				fmt.Fprintf(&b, "            cflags: [\"-DOPENSSL_NO_ASM\"],\n")
+				fmt.Fprintf(&b, "        },\n")
+				fmt.Fprintf(&b, "    },\n")
+			}
+			fmt.Fprintf(&b, "},\n")
+		}
+
+		return b.Bytes(), nil
+	})
+}
+
+func MakeBuildFiles(targets map[string]OutputTarget) []Task {
+	// TODO(crbug.com/boringssl/542): Generate the build files for the other
+	// types as well.
+	return []Task{
+		buildVariablesTask(targets, "gen/sources.cmake", "#", writeCMakeVariable),
+		jsonTask(targets, "gen/sources.json"),
+	}
+}
diff --git a/util/pregenerate/err_data.go b/util/pregenerate/err_data.go
new file mode 100644
index 0000000..8d89d99
--- /dev/null
+++ b/util/pregenerate/err_data.go
@@ -0,0 +1,257 @@
+// Copyright (c) 2015, Google Inc.
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"sort"
+	"strconv"
+)
+
+// libraryNames must be kept in sync with the enum in err.h. The generated code
+// will contain static assertions to enforce this.
+var libraryNames = []string{
+	"NONE",
+	"SYS",
+	"BN",
+	"RSA",
+	"DH",
+	"EVP",
+	"BUF",
+	"OBJ",
+	"PEM",
+	"DSA",
+	"X509",
+	"ASN1",
+	"CONF",
+	"CRYPTO",
+	"EC",
+	"SSL",
+	"BIO",
+	"PKCS7",
+	"PKCS8",
+	"X509V3",
+	"RAND",
+	"ENGINE",
+	"OCSP",
+	"UI",
+	"COMP",
+	"ECDSA",
+	"ECDH",
+	"HMAC",
+	"DIGEST",
+	"CIPHER",
+	"HKDF",
+	"TRUST_TOKEN",
+	"USER",
+}
+
+// stringList is a map from uint32 -> string which can output data for a sorted
+// list as C literals.
+type stringList struct {
+	// entries is an array of keys and offsets into |stringData|. The
+	// offsets are in the bottom 15 bits of each uint32 and the key is the
+	// top 17 bits.
+	entries []uint32
+	// internedStrings contains the same strings as are in |stringData|,
+	// but allows for easy deduplication. It maps a string to its offset in
+	// |stringData|.
+	internedStrings map[string]uint32
+	stringData      []byte
+}
+
+func newStringList() *stringList {
+	return &stringList{
+		internedStrings: make(map[string]uint32),
+	}
+}
+
+// offsetMask is the bottom 15 bits. It's a mask that selects the offset from a
+// uint32 in entries.
+const offsetMask = 0x7fff
+
+func (st *stringList) Add(key uint32, value string) error {
+	if key&offsetMask != 0 {
+		return errors.New("need bottom 15 bits of the key for the offset")
+	}
+	offset, ok := st.internedStrings[value]
+	if !ok {
+		offset = uint32(len(st.stringData))
+		if offset&offsetMask != offset {
+			return errors.New("stringList overflow")
+		}
+		st.stringData = append(st.stringData, []byte(value)...)
+		st.stringData = append(st.stringData, 0)
+		st.internedStrings[value] = offset
+	}
+
+	for _, existing := range st.entries {
+		if existing>>15 == key>>15 {
+			panic("duplicate entry")
+		}
+	}
+	st.entries = append(st.entries, key|offset)
+	return nil
+}
+
+func (st *stringList) buildList() []uint32 {
+	sort.Slice(st.entries, func(i, j int) bool { return (st.entries[i] >> 15) < (st.entries[j] >> 15) })
+	return st.entries
+}
+
+type stringWriter interface {
+	io.Writer
+	WriteString(string) (int, error)
+}
+
+func (st *stringList) WriteTo(out stringWriter, name string) {
+	list := st.buildList()
+	values := "kOpenSSL" + name + "Values"
+	out.WriteString("const uint32_t " + values + "[] = {\n")
+	for _, v := range list {
+		fmt.Fprintf(out, "    0x%x,\n", v)
+	}
+	out.WriteString("};\n\n")
+	out.WriteString("const size_t " + values + "Len = sizeof(" + values + ") / sizeof(" + values + "[0]);\n\n")
+
+	stringData := "kOpenSSL" + name + "StringData"
+	out.WriteString("const char " + stringData + "[] =\n    \"")
+	for i, c := range st.stringData {
+		if c == 0 {
+			out.WriteString("\\0\"\n    \"")
+			continue
+		}
+		out.Write(st.stringData[i : i+1])
+	}
+	out.WriteString("\";\n\n")
+}
+
+type errorData struct {
+	reasons    *stringList
+	libraryMap map[string]uint32
+}
+
+func (e *errorData) readErrorDataFile(filename string) error {
+	inFile, err := os.Open(filename)
+	if err != nil {
+		return err
+	}
+	defer inFile.Close()
+
+	scanner := bufio.NewScanner(inFile)
+	comma := []byte(",")
+
+	lineNo := 0
+	for scanner.Scan() {
+		lineNo++
+
+		line := scanner.Bytes()
+		if len(line) == 0 {
+			continue
+		}
+		parts := bytes.Split(line, comma)
+		if len(parts) != 3 {
+			return fmt.Errorf("bad line %d in %s: found %d values but want 3", lineNo, filename, len(parts))
+		}
+		libNum, ok := e.libraryMap[string(parts[0])]
+		if !ok {
+			return fmt.Errorf("bad line %d in %s: unknown library", lineNo, filename)
+		}
+		if libNum >= 64 {
+			return fmt.Errorf("bad line %d in %s: library value too large", lineNo, filename)
+		}
+		key, err := strconv.ParseUint(string(parts[1]), 10 /* base */, 32 /* bit size */)
+		if err != nil {
+			return fmt.Errorf("bad line %d in %s: %s", lineNo, filename, err)
+		}
+		if key >= 2048 {
+			return fmt.Errorf("bad line %d in %s: key too large", lineNo, filename)
+		}
+		value := string(parts[2])
+
+		listKey := libNum<<26 | uint32(key)<<15
+
+		err = e.reasons.Add(listKey, value)
+		if err != nil {
+			return err
+		}
+	}
+
+	return scanner.Err()
+}
+
+type ErrDataTask struct {
+	TargetName string
+	Inputs     []string
+}
+
+func (t *ErrDataTask) Destination() string {
+	return path.Join("gen", t.TargetName, "err_data.c")
+}
+
+func (t *ErrDataTask) Run() ([]byte, error) {
+	e := &errorData{
+		reasons:    newStringList(),
+		libraryMap: make(map[string]uint32),
+	}
+	for i, name := range libraryNames {
+		e.libraryMap[name] = uint32(i) + 1
+	}
+
+	for _, input := range t.Inputs {
+		if err := e.readErrorDataFile(input); err != nil {
+			return nil, err
+		}
+	}
+
+	var out bytes.Buffer
+	out.WriteString(`/* Copyright (c) 2015, Google Inc.
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+ * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
+
+ /* This file was generated by go run ./util/pregenerate. */
+
+#include <openssl/base.h>
+#include <openssl/err.h>
+
+#include <assert.h>
+
+`)
+
+	for i, name := range libraryNames {
+		fmt.Fprintf(&out, "static_assert(ERR_LIB_%s == %d, \"library value changed\");\n", name, i+1)
+	}
+	fmt.Fprintf(&out, "static_assert(ERR_NUM_LIBS == %d, \"number of libraries changed\");\n", len(libraryNames)+1)
+	out.WriteString("\n")
+
+	e.reasons.WriteTo(&out, "Reason")
+	return out.Bytes(), nil
+}
diff --git a/util/pregenerate/pregenerate.go b/util/pregenerate/pregenerate.go
new file mode 100644
index 0000000..ba062c7
--- /dev/null
+++ b/util/pregenerate/pregenerate.go
@@ -0,0 +1,218 @@
+// Copyright (c) 2024, Google Inc.
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+// pregenerate manages generated files in BoringSSL
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"slices"
+	"strings"
+	"sync"
+)
+
+var (
+	check      = flag.Bool("check", false, "Check whether any files need to be updated, without actually updating them")
+	numWorkers = flag.Int("num-workers", runtime.NumCPU(), "Runs the given number of workers")
+	dryRun     = flag.Bool("dry-run", false, "Skip actually writing any files")
+	perlPath   = flag.String("perl", "perl", "Path to the perl command")
+	list       = flag.Bool("list", false, "List all generated files, rather than actually run them")
+)
+
+func runTask(t Task) error {
+	expected, err := t.Run()
+	if err != nil {
+		return err
+	}
+
+	dst := t.Destination()
+	dstPath := filepath.FromSlash(dst)
+	if *check {
+		actual, err := os.ReadFile(dstPath)
+		if err != nil {
+			if os.IsNotExist(err) {
+				err = errors.New("missing file")
+			}
+			return err
+		}
+
+		if !bytes.Equal(expected, actual) {
+			return errors.New("file out of date")
+		}
+		return nil
+	}
+
+	if *dryRun {
+		fmt.Printf("Would write %d bytes to %q\n", len(expected), dst)
+		return nil
+	}
+
+	if err := os.MkdirAll(filepath.Dir(dstPath), 0777); err != nil {
+		return err
+	}
+	return os.WriteFile(dstPath, expected, 0666)
+}
+
+type taskError struct {
+	dst string
+	err error
+}
+
+func worker(taskChan <-chan Task, errorChan chan<- taskError, wg *sync.WaitGroup) {
+	defer wg.Done()
+	for t := range taskChan {
+		if err := runTask(t); err != nil {
+			errorChan <- taskError{t.Destination(), err}
+		}
+	}
+}
+
+func run() error {
+	if _, err := os.Stat("BUILDING.md"); err != nil {
+		return fmt.Errorf("must be run from BoringSSL source root")
+	}
+
+	buildJSON, err := os.ReadFile("build.json")
+	if err != nil {
+		return err
+	}
+
+	// Remove comments. For now, just do a very basic preprocessing step. If
+	// needed, we can switch to something well-defined like one of the many
+	// dozen different extended JSONs like JSON5.
+	lines := bytes.Split(buildJSON, []byte("\n"))
+	for i := range lines {
+		if idx := bytes.Index(lines[i], []byte("//")); idx >= 0 {
+			lines[i] = lines[i][:idx]
+		}
+	}
+	buildJSON = bytes.Join(lines, []byte("\n"))
+
+	var targetsIn map[string]InputTarget
+	if err := json.Unmarshal(buildJSON, &targetsIn); err != nil {
+		return fmt.Errorf("error decoding build config: %s", err)
+	}
+
+	var tasks []Task
+	targetsOut := make(map[string]OutputTarget)
+	for name, targetIn := range targetsIn {
+		targetOut, targetTasks := targetIn.Pregenerate(name)
+		targetsOut[name] = targetOut
+		tasks = append(tasks, targetTasks...)
+	}
+
+	tasks = append(tasks, MakeBuildFiles(targetsOut)...)
+	tasks = append(tasks, NewSimpleTask("gen/README.md", func() ([]byte, error) {
+		return []byte(readme), nil
+	}))
+
+	// Filter tasks by command-line argument.
+	if args := flag.Args(); len(args) != 0 {
+		var filtered []Task
+		for _, t := range tasks {
+			dst := t.Destination()
+			for _, arg := range args {
+				if strings.Contains(dst, arg) {
+					filtered = append(filtered, t)
+					break
+				}
+			}
+		}
+		tasks = filtered
+	}
+
+	if *list {
+		paths := make([]string, len(tasks))
+		for i, t := range tasks {
+			paths[i] = t.Destination()
+		}
+		slices.Sort(paths)
+		for _, p := range paths {
+			fmt.Println(p)
+		}
+		return nil
+	}
+
+	// Schedule tasks in parallel. Perlasm benefits from running in parallel. The
+	// others likely do not, but it is simpler to parallelize them all.
+	var wg sync.WaitGroup
+	taskChan := make(chan Task, *numWorkers)
+	errorChan := make(chan taskError, *numWorkers)
+	for i := 0; i < *numWorkers; i++ {
+		wg.Add(1)
+		go worker(taskChan, errorChan, &wg)
+	}
+
+	go func() {
+		for _, t := range tasks {
+			taskChan <- t
+		}
+		close(taskChan)
+		wg.Wait()
+		close(errorChan)
+	}()
+
+	var failed bool
+	for err := range errorChan {
+		fmt.Fprintf(os.Stderr, "Error in file %q: %s\n", err.dst, err.err)
+		failed = true
+	}
+	if failed {
+		return errors.New("some files had errors")
+	}
+	return nil
+}
+
+func main() {
+	flag.Parse()
+	if err := run(); err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %s\n", err)
+		os.Exit(1)
+	}
+}
+
+const readme = `# Pre-generated files
+
+This directory contains a number of pre-generated build artifacts. To simplify
+downstream builds, they are checked into the repository, rather than dynamically
+generated as part of the build.
+
+When developing on BoringSSL, if any inputs to these files are modified, callers
+must run the following command to update the generated files:
+
+    go run ./util/pregenerate
+
+To check that files are up-to-date without updating files, run:
+
+    go run ./util/pregenerate -check
+
+This is run on CI to ensure the generated files remain up-to-date.
+
+To speed up local iteration, the tool accepts additional arguments to filter the
+files generated. For example, if editing ` + "`aesni-x86_64.pl`" + `, this
+command will only update files with "aesni-x86_64" as a substring.
+
+    go run ./util/pregenerate aesni-x86_64
+
+For convenience, all files in this directory, including this README, are managed
+by the tool. This means the whole directory may be deleted and regenerated from
+scratch at any time.
+`
diff --git a/util/pregenerate/task.go b/util/pregenerate/task.go
new file mode 100644
index 0000000..f04fc43
--- /dev/null
+++ b/util/pregenerate/task.go
@@ -0,0 +1,82 @@
+// Copyright (c) 2024, Google Inc.
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+package main
+
+import (
+	"bytes"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+)
+
+type Task interface {
+	// Destination returns the destination path for this task, using forward
+	// slashes and relative to the source directory. That is, use the "path"
+	// package, not "path/filepath".
+	Destination() string
+
+	// Run computes the output for this task. It should be written to the
+	// destination path.
+	Run() ([]byte, error)
+}
+
+type SimpleTask struct {
+	Dst     string
+	RunFunc func() ([]byte, error)
+}
+
+func (t *SimpleTask) Destination() string  { return t.Dst }
+func (t *SimpleTask) Run() ([]byte, error) { return t.RunFunc() }
+
+func NewSimpleTask(dst string, runFunc func() ([]byte, error)) *SimpleTask {
+	return &SimpleTask{Dst: dst, RunFunc: runFunc}
+}
+
+type PerlasmTask struct {
+	Src, Dst string
+	Args     []string
+}
+
+func (t *PerlasmTask) Destination() string { return t.Dst }
+func (t *PerlasmTask) Run() ([]byte, error) {
+	base := path.Base(t.Dst)
+	out, err := os.CreateTemp("", "*."+base)
+	if err != nil {
+		return nil, err
+	}
+	defer os.Remove(out.Name())
+
+	args := make([]string, 0, 2+len(t.Args))
+	args = append(args, filepath.FromSlash(t.Src))
+	args = append(args, t.Args...)
+	args = append(args, out.Name())
+	cmd := exec.Command(*perlPath, args...)
+	cmd.Stderr = os.Stderr
+	cmd.Stdout = os.Stdout
+	if err := cmd.Run(); err != nil {
+		return nil, err
+	}
+
+	data, err := os.ReadFile(out.Name())
+	if err != nil {
+		return nil, err
+	}
+
+	// On Windows, perl emits CRLF line endings. Normalize this so that the tool
+	// can be run on Windows too.
+	data = bytes.ReplaceAll(data, []byte("\r\n"), []byte("\n"))
+	return data, nil
+}