unified: Always run corpus tests

MODULE.bazel

@@ -228,13 +228,8 @@ use_repo(
 # auto-registers `xcode_swift_toolchain` (host Xcode + OS-provided Swift
 # runtime), which is not hermetic.
 #
-# The version is pinned as a literal rather than read from
-# `unified/swift-syntax-rs/.swift-version` via `swift_version_file`: the latter
-# makes the extension `module_ctx.read` a `//unified/...` label, which fails to
-# resolve when this repo is consumed as a dependency module (`@@ql+`) whose
-# `unified/swift-syntax-rs` package is not loadable in that context. Keep this
-# in sync with `unified/swift-syntax-rs/.swift-version` (used by the `cargo`
-# build) and the `swift-syntax` release in `swift/Package.swift`.
+# This and the `swift-syntax` version above are the only pins: the extractor is
+# built solely by Bazel, so there is no second Swift build to keep in step.
 swift = use_extension("@rules_swift//swift:extensions.bzl", "swift")
 swift.toolchain(
     name = "swift_toolchain",

unified/AGENTS.md

@@ -4,13 +4,18 @@ This is a CodeQL extractor that maps a language's parse tree onto a shared AST
 using the `yeast` desugaring engine. Swift, the only language so far, is parsed
 by Apple's swift-syntax rather than by tree-sitter.
 
+Everything is built and tested with Bazel, whose Swift toolchain is hermetic on
+Linux, so nothing needs to be installed locally. `cargo build`/`cargo test` do
+not work here: the extractor links a Swift FFI shim that only Bazel builds.
+(`cargo check` still works, so rust-analyzer is unaffected.)
+
 ## Building
-- To build the extractor, run `scripts/create-extractor-pack.sh`
+- To build the extractor pack, run `scripts/create-extractor-pack.sh`.
 
 ## Swift Parser
-- Swift source is parsed by `swift-syntax-parse`, a small Swift/Rust binary in
-  `swift-syntax-rs` that wraps Apple's swift-syntax and emits the parse tree as
-  JSON. There is no grammar in this repository to edit.
+- Swift source is parsed by the `swift-syntax-rs` crate, which wraps Apple's
+  swift-syntax and emits the parse tree as JSON. The extractor links it and
+  calls it in-process. There is no grammar in this repository to edit.
 
 - `extractor/src/languages/swift/adapter.rs` converts that JSON into a yeast AST.
 
@@ -22,11 +27,7 @@ by Apple's swift-syntax rather than by tree-sitter.
 
 - The mapping from the parse tree to the target AST is found in `extractor/src/languages/swift/swift.rs`
 
-- To run tests for the parser and mapping, run `cargo test` in the `extractor`
-  directory. The tests need the `swift-syntax-parse` binary: point
-  `CODEQL_EXTRACTOR_UNIFIED_SWIFT_SYNTAX_PARSE` at it, or put it on `PATH`.
-  Corpus tests skip themselves when it cannot be found, so check for skips
-  before concluding a change is clean.
+- To run tests for the parser and mapping, run `bazel test //unified/extractor:all_tests`.
 
 - Extractor test cases are located at `extractor/tests/corpus/swift/*/*.swift`.
 

unified/BUILD.bazel

@@ -47,15 +47,13 @@ codeql_pkg_files(
     prefix = "tools/{CODEQL_PLATFORM}",
 )
 
-# The Swift front-end parser (wrapper + real binary + bundled Swift runtime),
-# shipped next to the extractor. Only on platforms where swift-syntax builds
-# (Linux/macOS); elsewhere the group is empty so the pack still builds (Swift
-# extraction is simply unavailable there).
-pkg_filegroup(
-    name = "swift-syntax-parse-arch",
-    srcs = select_os(
+# The Swift runtime, which the extractor loads at startup. Linux only: macOS
+# provides it with the OS.
+codeql_pkg_files(
+    name = "swift-runtime-arch",
+    exes = select_os(
+        linux = ["//unified/swift-syntax-rs:swift_runtime_libs"],
         otherwise = [],
-        posix = ["//unified/swift-syntax-rs:swift-syntax-parse-pkg"],
     ),
     prefix = "tools/{CODEQL_PLATFORM}",
 )
@@ -66,7 +64,7 @@ codeql_pack(
         ":codeql-extractor-yml",
         ":dbscheme-group",
         ":extractor-arch",
-        ":swift-syntax-parse-arch",
+        ":swift-runtime-arch",
         "//unified/tools",
     ],
 )

unified/extractor/BUILD.bazel

@@ -1,8 +1,16 @@
+load("@rules_rust//rust:defs.bzl", "rust_test")
 load("//misc/bazel:rust.bzl", "codeql_rust_binary")
 load("//misc/bazel/3rdparty/tree_sitter_extractors_deps:defs.bzl", "aliases", "all_crate_deps")
 
 exports_files(["Cargo.toml"])
 
+# swift-syntax builds on Linux and macOS only.
+_SWIFT_SUPPORTED_PLATFORMS = select({
+    "@platforms//os:linux": [],
+    "@platforms//os:macos": [],
+    "//conditions:default": ["@platforms//:incompatible"],
+})
+
 codeql_rust_binary(
     name = "extractor",
     srcs = glob(["src/**/*.rs"]),
@@ -11,14 +19,75 @@ codeql_rust_binary(
         "ast_types.yml",
         "swift_node_types.yml",
     ],
+    # Only for running from the build tree: there the Swift runtime is resolved
+    # through the toolchain-relative part of the runpath, which needs the
+    # libraries in the runfiles tree. An installed pack uses `$ORIGIN` instead.
+    data = select({
+        "@platforms//os:linux": ["//unified/swift-syntax-rs:swift_runtime_libs"],
+        "//conditions:default": [],
+    }),
     proc_macro_deps = all_crate_deps(
         proc_macro = True,
     ),
+    target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS,
     visibility = ["//visibility:public"],
     deps = all_crate_deps(
         normal = True,
     ) + [
         "//shared/tree-sitter-extractor",
         "//shared/yeast",
+        "//unified/swift-syntax-rs:swift_syntax_rs",
     ],
 )
+
+# One target per file in `tests/`. Each pulls in `src/**` too, because the tests
+# reach into the crate's modules with `#[path]`.
+_TESTS = {
+    "corpus_tests": {
+        "data": glob(["tests/corpus/**"]),
+        "compile_data": [],
+        "size": "medium",
+    },
+    # `include_str!`s a checked-in `parse_to_json` dump.
+    "swift_syntax_pipeline": {
+        "data": [],
+        "compile_data": glob(["tests/fixtures/**"]),
+        "size": "small",
+    },
+}
+
+[
+    rust_test(
+        name = test_name,
+        size = spec["size"],
+        srcs = ["tests/%s.rs" % test_name] + glob(["src/**/*.rs"]),
+        aliases = aliases(),
+        compile_data = [
+            "ast_types.yml",
+            "swift_node_types.yml",
+        ] + spec["compile_data"],
+        crate_root = "tests/%s.rs" % test_name,
+        data = spec["data"] + select({
+            "@platforms//os:linux": ["//unified/swift-syntax-rs:swift_runtime_libs"],
+            "//conditions:default": [],
+        }),
+        edition = "2024",
+        proc_macro_deps = all_crate_deps(
+            proc_macro = True,
+        ),
+        target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS,
+        deps = all_crate_deps(
+            normal = True,
+        ) + [
+            "//shared/tree-sitter-extractor",
+            "//shared/yeast",
+            "//unified/swift-syntax-rs:swift_syntax_rs",
+        ],
+    )
+    for test_name, spec in _TESTS.items()
+]
+
+test_suite(
+    name = "all_tests",
+    tests = [":%s" % test_name for test_name in _TESTS],
+)

unified/extractor/Cargo.toml

@@ -18,3 +18,8 @@ serde_json = "1.0.145"
 
 codeql-extractor = { path = "../../shared/tree-sitter-extractor" }
 yeast = { path = "../../shared/yeast" }
+# The Swift front-end links swift-syntax through this crate's FFI shim. Its
+# Swift half is built by Bazel, so `cargo build`/`test` cannot link the
+# extractor: use `bazel build //unified/extractor` and
+# `bazel test //unified/extractor:all_tests`.
+swift-syntax-rs = { path = "../swift-syntax-rs" }

unified/extractor/src/languages/swift/parse.rs

@@ -1,121 +1,22 @@
-//! Swift front-end parser: shells out to the separate `swift-syntax-parse`
-//! binary (which links swift-syntax) to obtain a JSON syntax tree, then adapts
-//! that JSON into a `yeast::Ast` via the pure-Rust [`swift_adapter`] module.
-//!
-//! Running the parser in a separate process keeps the Swift toolchain out of
-//! the extractor's own build: the extractor never links Swift, so working on
-//! other (e.g. tree-sitter based) languages needs no Swift toolchain. Each call
-//! spawns the parser afresh; a longer-lived parser process could be swapped in
-//! behind this same seam later without touching the extraction pipeline.
-
-use std::io::Write;
-use std::process::{Command, Stdio};
+//! Swift front-end parser: calls into the `swift-syntax-rs` crate (which links
+//! swift-syntax) to obtain a JSON syntax tree, then adapts that JSON into a
+//! `yeast::Ast` via the pure-Rust [`swift_adapter`] module.
 
 use codeql_extractor::extractor::ParsedTree;
 
 use super::swift_adapter;
 
-/// Environment variable naming the `swift-syntax-parse` executable. When unset,
-/// the parser is resolved next to the extractor executable, then on `PATH`.
-const PARSE_BIN_ENV: &str = "CODEQL_EXTRACTOR_UNIFIED_SWIFT_SYNTAX_PARSE";
-
-/// Base name of the `swift-syntax-parse` executable as shipped / looked up.
-const PARSE_BIN_NAME: &str = "swift-syntax-parse";
-
 /// Parse Swift `source` into a [`ParsedTree`] (a raw `yeast::Ast` plus
 /// side-channel `extra` tokens), ready to be desugared via `run_from_ast`.
 pub fn parse(source: &[u8]) -> Result<ParsedTree, String> {
     let source =
         std::str::from_utf8(source).map_err(|e| format!("Swift source is not valid UTF-8: {e}"))?;
-    let json = run_parser(source)?;
+    let json =
+        swift_syntax_rs::parse_to_json(source).map_err(|e| format!("Swift parser failed: {e}"))?;
     let mut adapted = swift_adapter::json_to_ast(&json)?;
     adapted.ast.set_source(source.as_bytes().to_vec());
     Ok(ParsedTree {
         ast: adapted.ast,
         extras: adapted.extras,
     })
 }
-
-/// The `swift-syntax-parse` executable to invoke, resolved in priority order:
-///
-/// 1. the `CODEQL_EXTRACTOR_UNIFIED_SWIFT_SYNTAX_PARSE` override, if set;
-/// 2. a copy shipped next to the extractor executable — this is how the CodeQL
-///    extractor pack lays it out (`tools/<platform>/{extractor,
-///    swift-syntax-parse}`), so a packaged extractor is self-contained with no
-///    environment setup;
-/// 3. a bare `swift-syntax-parse`, looked up on `PATH`.
-fn parse_bin() -> String {
-    if let Ok(bin) = std::env::var(PARSE_BIN_ENV) {
-        if !bin.is_empty() {
-            return bin;
-        }
-    }
-    if let Ok(exe) = std::env::current_exe() {
-        if let Some(sibling) = exe.parent().map(|dir| dir.join(PARSE_BIN_NAME)) {
-            if sibling.is_file() {
-                return sibling.to_string_lossy().into_owned();
-            }
-        }
-    }
-    PARSE_BIN_NAME.to_string()
-}
-
-/// Whether the `swift-syntax-parse` executable can be launched at all.
-///
-/// This reports availability of the *executable*, deliberately not whether
-/// parsing succeeds: a binary that launches but then crashes or emits invalid
-/// JSON is still "available", so callers run and surface the failure rather
-/// than silently skipping. Only a genuinely missing/unlaunchable binary (e.g.
-/// no Swift toolchain is installed) reports `false`.
-pub fn binary_available() -> bool {
-    match Command::new(parse_bin())
-        .stdin(Stdio::null())
-        .stdout(Stdio::null())
-        .stderr(Stdio::null())
-        .spawn()
-    {
-        Ok(mut child) => {
-            let _ = child.wait();
-            true
-        }
-        Err(e) if e.kind() == std::io::ErrorKind::NotFound => false,
-        // Any other spawn failure (e.g. a permissions problem) is a genuine
-        // issue worth surfacing, so treat the parser as available and let the
-        // caller fail rather than masking it as "unavailable".
-        Err(_) => true,
-    }
-}
-
-/// Run the external parser, feeding `source` on stdin and returning its JSON
-/// stdout.
-fn run_parser(source: &str) -> Result<String, String> {
-    let bin = parse_bin();
-    let mut child = Command::new(&bin)
-        .stdin(Stdio::piped())
-        .stdout(Stdio::piped())
-        .stderr(Stdio::piped())
-        .spawn()
-        .map_err(|e| format!("failed to spawn Swift parser `{bin}`: {e}"))?;
-
-    // The parser reads all of stdin before writing any stdout, so writing the
-    // whole source and then closing stdin (by dropping it) cannot deadlock.
-    child
-        .stdin
-        .take()
-        .expect("child stdin was piped")
-        .write_all(source.as_bytes())
-        .map_err(|e| format!("failed to write source to Swift parser `{bin}`: {e}"))?;
-
-    let output = child
-        .wait_with_output()
-        .map_err(|e| format!("failed to run Swift parser `{bin}`: {e}"))?;
-    if !output.status.success() {
-        return Err(format!(
-            "Swift parser `{bin}` failed ({}): {}",
-            output.status,
-            String::from_utf8_lossy(&output.stderr).trim()
-        ));
-    }
-    String::from_utf8(output.stdout)
-        .map_err(|e| format!("Swift parser produced non-UTF-8 output: {e}"))
-}