Use source lists to find pki_test data in run_android_tests.go

We now have (some of) our test sources in an easily parseable form.
run_android_tests.go no longer needs to crawl the source tree.

Note this required fixing the .gitignore rules. If a .gitignore line
doesn't have a slash at the start or middle, it applies to
subdirectories as well. This is confusing, so I just stuck a leading
slash in front of all of them.

Bug: 681
Change-Id: I389c2a0560594fbd23c60b5b614b0ccfedf28926
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/67293
Reviewed-by: Bob Beck <bbe@google.com>
Commit-Queue: David Benjamin <davidben@google.com>

5 files changed, 91 insertions, 68 deletions

diff --git a/.gitignore b/.gitignore
index 3e22141..d1cc3b8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,30 +1,32 @@
-build/
-build32/
-build64/
-ssl/test/runner/runner
+/build
+/build32
+/build64
+/ssl/test/runner/runner
 *.pyc
 *.swp
 *.swo
-doc/*.html
-doc/doc.css
-rust/Cargo.lock
-rust/bssl-crypto/Cargo.lock
-rust/target
+/doc/*.html
+/doc/doc.css
+/rust/Cargo.lock
+/rust/bssl-crypto/Cargo.lock
+/rust/target
 
-util/bot/android_ndk
-util/bot/android_sdk/public
-util/bot/cmake
-util/bot/golang
-util/bot/libFuzzer
-util/bot/libcxx
-util/bot/libcxxabi
-util/bot/llvm-build
-util/bot/nasm-win32.exe
-util/bot/ninja
-util/bot/perl-win32
-util/bot/sde-linux64
-util/bot/sde-linux64.tar.xz
-util/bot/sde-win32
-util/bot/sde-win32.tar.xz
-util/bot/win_toolchain.json
+/util/bot/android_ndk
+/util/bot/android_sdk/public
+/util/bot/cmake
+/util/bot/golang
+/util/bot/libFuzzer
+/util/bot/libcxx
+/util/bot/libcxxabi
+/util/bot/llvm-build
+/util/bot/nasm-win32.exe
+/util/bot/ninja
+/util/bot/perl-win32
+/util/bot/sde-linux64
+/util/bot/sde-linux64.tar.xz
+/util/bot/sde-win32
+/util/bot/sde-win32.tar.xz
+/util/bot/win_toolchain.json
+
+# Ignore target under any directory.
 target/
diff --git a/util/build/build.go b/util/build/build.go
new file mode 100644
index 0000000..277bfdc
--- /dev/null
+++ b/util/build/build.go
@@ -0,0 +1,38 @@
+// Copyright (c) 2024, Google Inc.
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+package build
+
+// A Target is a build target for consumption by the downstream build systems.
+// All pre-generated files are baked input its source lists.
+type Target struct {
+	// Srcs is the list of C or C++ files (determined by file extension) that are
+	// built into the target.
+	Srcs []string `json:"srcs,omitempty"`
+	// Hdrs is the list public headers that should be available to external
+	// projects using this target.
+	Hdrs []string `json:"hdrs,omitempty"`
+	// InternalHdrs is the list of internal headers that should be available to
+	// this target, as well as any internal targets using this target.
+	InternalHdrs []string `json:"internal_hdrs,omitempty"`
+	// Asm is the a list of assembly files to be passed to a gas-compatible
+	// assembler.
+	Asm []string `json:"asm,omitempty"`
+	// Nasm is the a list of assembly files to be passed to a nasm-compatible
+	// assembler.
+	Nasm []string `json:"nasm,omitempty"`
+	// Data is a list of test data files that should be available when the test is
+	// run.
+	Data []string `json:"data,omitempty"`
+}
diff --git a/util/pregenerate/build.go b/util/pregenerate/build.go
index 999a00f..192a1ea 100644
--- a/util/pregenerate/build.go
+++ b/util/pregenerate/build.go
@@ -23,36 +23,15 @@ import (
 	"path/filepath"
 	"slices"
 	"strings"
-)
 
-// An OutputTarget is a build target for consumption by the downstream build
-// systems. All pre-generated files are baked input its source lists.
-type OutputTarget struct {
-	// Srcs is the list of C or C++ files (determined by file extension) that are
-	// built into the target.
-	Srcs []string `json:"srcs,omitempty"`
-	// Hdrs is the list public headers that should be available to external
-	// projects using this target.
-	Hdrs []string `json:"hdrs,omitempty"`
-	// InternalHdrs is the list of internal headers that should be available to
-	// this target, as well as any internal targets using this target.
-	InternalHdrs []string `json:"internal_hdrs,omitempty"`
-	// Asm is the a list of assembly files to be passed to a gas-compatible
-	// assembler.
-	Asm []string `json:"asm,omitempty"`
-	// Nasm is the a list of assembly files to be passed to a nasm-compatible
-	// assembler.
-	Nasm []string `json:"nasm,omitempty"`
-	// Data is a list of test data files that should be available when the test is
-	// run.
-	Data []string `json:"data,omitempty"`
-}
+	"boringssl.googlesource.com/boringssl/util/build"
+)
 
 // An InputTarget is a build target with build inputs that still need to be
 // pregenerated. All file lists in InputTarget are interpreted with glob
 // patterns as in filepath.Glob.
 type InputTarget struct {
-	OutputTarget
+	build.Target
 	// ErrData contains a list of errordata files to combine into err_data.c.
 	ErrData []string `json:"err_data,omitempty"`
 	// The following fields define perlasm sources for the corresponding
@@ -78,7 +57,7 @@ type PerlasmSource struct {
 // Pregenerate converts an input target to an output target. It returns the
 // result alongside a list of tasks that must be run to build the referenced
 // files.
-func (in *InputTarget) Pregenerate(name string) (out OutputTarget, tasks []Task, err error) {
+func (in *InputTarget) Pregenerate(name string) (out build.Target, tasks []Task, err error) {
 	// Expand wildcards.
 	out.Srcs, err = glob(in.Srcs)
 	if err != nil {
@@ -206,7 +185,7 @@ func writeHeader(b *bytes.Buffer, comment string) {
 	fmt.Fprintf(b, "%s Generated by go ./util/pregenerate. Do not edit manually.\n", comment)
 }
 
-func buildVariablesTask(targets map[string]OutputTarget, dst, comment string, writeVariable func(b *bytes.Buffer, name string, val []string)) Task {
+func buildVariablesTask(targets map[string]build.Target, dst, comment string, writeVariable func(b *bytes.Buffer, name string, val []string)) Task {
 	return NewSimpleTask(dst, func() ([]byte, error) {
 		var b bytes.Buffer
 		writeHeader(&b, comment)
@@ -270,13 +249,13 @@ func writeGNVariable(b *bytes.Buffer, name string, val []string) {
 	writeBazelVariable(b, name, val)
 }
 
-func jsonTask(targets map[string]OutputTarget, dst string) Task {
+func jsonTask(targets map[string]build.Target, dst string) Task {
 	return NewSimpleTask(dst, func() ([]byte, error) {
 		return json.MarshalIndent(targets, "", "  ")
 	})
 }
 
-func soongTask(targets map[string]OutputTarget, dst string) Task {
+func soongTask(targets map[string]build.Target, dst string) Task {
 	return NewSimpleTask(dst, func() ([]byte, error) {
 		var b bytes.Buffer
 		writeHeader(&b, "//")
@@ -323,7 +302,7 @@ func soongTask(targets map[string]OutputTarget, dst string) Task {
 	})
 }
 
-func MakeBuildFiles(targets map[string]OutputTarget) []Task {
+func MakeBuildFiles(targets map[string]build.Target) []Task {
 	// TODO(crbug.com/boringssl/542): Generate the build files for the other
 	// types as well.
 	return []Task{
diff --git a/util/pregenerate/pregenerate.go b/util/pregenerate/pregenerate.go
index 0f30503..1a3710e 100644
--- a/util/pregenerate/pregenerate.go
+++ b/util/pregenerate/pregenerate.go
@@ -27,6 +27,8 @@ import (
 	"slices"
 	"strings"
 	"sync"
+
+	"boringssl.googlesource.com/boringssl/util/build"
 )
 
 var (
@@ -112,7 +114,7 @@ func run() error {
 	}
 
 	var tasks []Task
-	targetsOut := make(map[string]OutputTarget)
+	targetsOut := make(map[string]build.Target)
 	for name, targetIn := range targetsIn {
 		targetOut, targetTasks, err := targetIn.Pregenerate(name)
 		if err != nil {
diff --git a/util/run_android_tests.go b/util/run_android_tests.go
index 4b5a18a..165ee83 100644
--- a/util/run_android_tests.go
+++ b/util/run_android_tests.go
@@ -19,6 +19,7 @@ package main
 import (
 	"bufio"
 	"bytes"
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
@@ -30,6 +31,7 @@ import (
 	"strconv"
 	"strings"
 
+	"boringssl.googlesource.com/boringssl/util/build"
 	"boringssl.googlesource.com/boringssl/util/testconfig"
 )
 
@@ -288,6 +290,17 @@ func main() {
 		os.Exit(1)
 	}
 
+	targetsJSON, err := os.ReadFile("gen/sources.json")
+	if err != nil {
+		fmt.Printf("Error reading sources.json: %s.\n", err)
+		os.Exit(1)
+	}
+	var targets map[string]build.Target
+	if err := json.Unmarshal(targetsJSON, &targets); err != nil {
+		fmt.Printf("Error reading sources.json: %s.\n", err)
+		os.Exit(1)
+	}
+
 	// Clear the target directory.
 	if err := adb("shell", "rm -Rf /data/local/tmp/boringssl-tmp"); err != nil {
 		fmt.Printf("Failed to clear target directory: %s\n", err)
@@ -309,19 +322,8 @@ func main() {
 			"util/all_tests.json",
 			"BUILDING.md",
 		)
-
-		err := filepath.Walk("pki/testdata/", func(path string, info os.FileInfo, err error) error {
-			if err != nil {
-				return err
-			}
-			if info.Mode().IsRegular() {
-				files = append(files, path)
-			}
-			return nil
-		})
-		if err != nil {
-			fmt.Printf("Can't walk pki/testdata: %s\n", err)
-			os.Exit(1)
+		for _, target := range targets {
+			files = append(files, target.Data...)
 		}
 
 		tests, err := testconfig.ParseTestConfig("util/all_tests.json")