Skip to content

armgit / dbx

Public Imported from GitHub
0

25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。

feat(ci): split dependency-aware checks and agent matrices

t8y2c8966a3de98a
diff --git a/.github/scripts/ci-agent-go.sh b/.github/scripts/ci-agent-go.sh
new file mode 100644
index 0000000000..e72f1bc696
--- /dev/null
+++ b/.github/scripts/ci-agent-go.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+driver="${1:?driver is required}"
+binary="${2:?binary name is required}"
+race="${3:?race mode is required}"
+root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "$root/agents/drivers/$driver"
+if [ "$driver" = xugu ]; then export GONOSUMDB=gitee.com/XuguDB/go-xugu-driver; fi
+
+if [ "$race" = true ]; then go test -race ./...; else go test ./...; fi
+if [ "$driver" = zookeeper ]; then
+ for target in darwin/arm64 darwin/amd64 linux/arm64 linux/amd64 windows/arm64 windows/amd64; do
+ IFS=/ read -r goos goarch <<< "$target"
+ output="/tmp/dbx-agent-zookeeper-${goos}-${goarch}"
+ [ "$goos" = windows ] && output="${output}.exe"
+ CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -trimpath -ldflags="-s -w" -o "$output" .
+ done
+else
+ CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o "/tmp/dbx-agent-${binary}-linux-x64" .
+fi
diff --git a/.github/scripts/ci-agent-integration.sh b/.github/scripts/ci-agent-integration.sh
new file mode 100644
index 0000000000..7ff5ac2367
--- /dev/null
+++ b/.github/scripts/ci-agent-integration.sh
@@ -0,0 +1,192 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+scenario="${1:?scenario is required}"
+version="${2:?version is required}"
+image="${3:-}"
+root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+case "$scenario" in
+ zookeeper)
+ cd "$root/agents/drivers/zookeeper"
+ name="dbx-zookeeper-${version//./-}"
+ docker rm -fv "$name" >/dev/null 2>&1 || true
+ docker run -d --name "$name" -p 2181:2181 "zookeeper:$version"
+ cleanup() {
+ docker rm -fv "$name" >/dev/null 2>&1 || true
+ }
+ trap cleanup EXIT
+ ready=false
+ for _ in $(seq 1 90); do
+ if timeout 1 bash -c '</dev/tcp/127.0.0.1/2181' >/dev/null 2>&1; then
+ ready=true
+ break
+ fi
+ sleep 2
+ done
+ if [ "$ready" != true ]; then
+ docker logs "$name"
+ exit 1
+ fi
+ DBX_ZOOKEEPER_TEST_CONNECT_STRING=127.0.0.1:2181 \
+ go test -run '^TestZooKeeperIntegration$' -count=1 ./...
+ if [ "$version" = "3.7.0" ]; then
+ DBX_ZOOKEEPER_TEST_CONNECT_STRING=127.0.0.1:2181 \
+ go test -run '^TestZooKeeperLargeChildrenIntegration$' -count=1 ./...
+ fi
+ ;;
+ zookeeper-sasl)
+ cd "$root/agents/drivers/zookeeper"
+ name="dbx-zookeeper-sasl-3-7-0"
+ jaas_file=$(mktemp)
+ cat > "$jaas_file" <<'EOF'
+ Server {
+ org.apache.zookeeper.server.auth.DigestLoginModule required
+ user_dbx="dbx-secret";
+ };
+EOF
+ chmod 0644 "$jaas_file"
+ docker rm -fv "$name" >/dev/null 2>&1 || true
+ docker run -d --name "$name" -p 2181:2181 \
+ -v "$jaas_file:/conf/server-jaas.conf:ro" \
+ -e ZOO_CFG_EXTRA="authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider" \
+ -e JVMFLAGS="-Djava.security.auth.login.config=/conf/server-jaas.conf -Dzookeeper.sasl.serverconfig=Server -Dzookeeper.sessionRequireClientSASLAuth=true" \
+ zookeeper:3.7.0
+ cleanup() {
+ docker rm -fv "$name" >/dev/null 2>&1 || true
+ }
+ trap cleanup EXIT
+ ready=false
+ for _ in $(seq 1 90); do
+ if timeout 1 bash -c '</dev/tcp/127.0.0.1/2181' >/dev/null 2>&1; then
+ ready=true
+ break
+ fi
+ sleep 2
+ done
+ if [ "$ready" != true ]; then
+ docker logs "$name"
+ exit 1
+ fi
+ DBX_ZOOKEEPER_TEST_CONNECT_STRING=127.0.0.1:2181 \
+ DBX_ZOOKEEPER_TEST_AUTH_SCHEME=sasl_digest \
+ DBX_ZOOKEEPER_TEST_USERNAME=dbx \
+ DBX_ZOOKEEPER_TEST_PASSWORD=dbx-secret \
+ go test -run '^TestZooKeeperIntegration$' -count=1 ./...
+ ;;
+ rocketmq)
+ cd "$root/agents/drivers/rocketmq"
+ ./scripts/run-integration.sh "$version"
+ ;;
+ tdengine)
+ cd "$root/agents"
+ name="dbx-tdengine-${version//./-}"
+ docker rm -fv "$name" >/dev/null 2>&1 || true
+ docker run -d --name "$name" \
+ --memory 4g \
+ --ulimit nofile=65535:65535 \
+ -p 6030:6030 \
+ -p 6041:6041 \
+ "$image"
+ cleanup() {
+ docker rm -fv "$name" >/dev/null 2>&1 || true
+ }
+ trap cleanup EXIT
+ ready=false
+ dnodes=""
+ for _ in $(seq 1 90); do
+ dnodes="$(docker exec "$name" taos -s 'SHOW DNODES' 2>/dev/null || true)"
+ if grep -Eq '\|[[:space:]]*ready[[:space:]]*\|' <<<"$dnodes"; then
+ ready=true
+ break
+ fi
+ sleep 2
+ done
+ if [ "$ready" != "true" ]; then
+ printf '%s\n' "$dnodes"
+ docker logs "$name"
+ exit 1
+ fi
+ TDENGINE_INTEGRATION=1 \
+ TDENGINE_TEST_HOST=127.0.0.1 \
+ TDENGINE_TEST_PORT=6041 \
+ cargo test --manifest-path drivers/tdengine/Cargo.toml --locked --test live -- --nocapture
+ ;;
+ cassandra)
+ cd "$root/agents/drivers/cassandra-go"
+ name="dbx-cassandra-${version//./-}"
+ docker rm -fv "$name" >/dev/null 2>&1 || true
+ docker run -d --name "$name" \
+ -e CASSANDRA_CLUSTER_NAME="DBX Cassandra CI $version" \
+ -e CASSANDRA_DC=dc1 \
+ -e CASSANDRA_RACK=rack1 \
+ -e CASSANDRA_ENDPOINT_SNITCH=GossipingPropertyFileSnitch \
+ -e CASSANDRA_NUM_TOKENS=16 \
+ -e MAX_HEAP_SIZE=512M \
+ -e HEAP_NEWSIZE=100M \
+ -p 9042:9042 \
+ "cassandra:$version"
+ cleanup() {
+ docker rm -fv "$name" >/dev/null 2>&1 || true
+ }
+ trap cleanup EXIT
+ ready=false
+ for _ in $(seq 1 100); do
+ if docker exec "$name" cqlsh -e 'SELECT release_version FROM system.local' >/dev/null 2>&1; then
+ ready=true
+ break
+ fi
+ sleep 3
+ done
+ if [ "$ready" != "true" ]; then
+ docker logs "$name"
+ exit 1
+ fi
+ CASSANDRA_TEST_HOST=127.0.0.1 \
+ CASSANDRA_TEST_PORT=9042 \
+ go test -run '^TestCassandraIntegration$' -count=1 ./...
+ ;;
+ rabbitmq)
+ cd "$root/agents/drivers/rabbitmq"
+ name="dbx-rabbitmq-${version//./-}"
+ cookie="dbx-ci-${GITHUB_RUN_ID:-local}-${version//./-}"
+ # RabbitMQ is disposable in CI. Keep its data on a fresh tmpfs
+ # owned by the image's rabbitmq user so .erlang.cookie is readable.
+ docker rm -fv "$name" >/dev/null 2>&1 || true
+ docker run -d --name "$name" \
+ --user 999:999 \
+ --tmpfs /var/lib/rabbitmq:rw,exec,uid=999,gid=999,mode=700 \
+ -e RABBITMQ_DEFAULT_USER=dbx \
+ -e RABBITMQ_DEFAULT_PASS=dbx-password \
+ -e RABBITMQ_ERLANG_COOKIE="$cookie" \
+ -p 5672:5672 -p 15672:15672 \
+ "rabbitmq:${version}-management"
+ cleanup() {
+ docker rm -fv "$name" >/dev/null 2>&1 || true
+ }
+ trap cleanup EXIT
+ ready=false
+ for _ in $(seq 1 60); do
+ if docker exec "$name" rabbitmq-diagnostics -q check_running >/dev/null 2>&1 \
+ && curl --fail --silent --noproxy '*' --user dbx:dbx-password \
+ http://127.0.0.1:15672/api/overview >/dev/null; then
+ ready=true
+ break
+ fi
+ sleep 2
+ done
+ if [ "$ready" != "true" ]; then
+ docker inspect "$name" --format 'image={{.Config.Image}} user={{.Config.User}} status={{.State.Status}} exit={{.State.ExitCode}}' || true
+ docker logs "$name"
+ exit 1
+ fi
+ RABBITMQ_INTEGRATION=1 \
+ RABBITMQ_USERNAME=dbx \
+ RABBITMQ_PASSWORD=dbx-password \
+ go test -run '^TestRabbitMQIntegration$' -count=1 ./...
+ ;;
+ *)
+ echo "Unknown integration scenario: $scenario" >&2
+ exit 2
+ ;;
+esac
diff --git a/.github/scripts/ci-config.mjs b/.github/scripts/ci-config.mjs
new file mode 100644
index 0000000000..0283dbb63b
--- /dev/null
+++ b/.github/scripts/ci-config.mjs
@@ -0,0 +1,32 @@
+export const rustGroups = {
+ foundation: ["dbx-types", "dbx-platform", "dbx-sql", "dbx-formats", "dbx-ai-provider", "dbx-plugin-runtime"],
+ drivers: ["dbx-drivers", "dbx-sqlite-worker"],
+ application: ["dbx", "dbx-core", "dbx-web", "dbx-cli", "dbx-mcp"],
+};
+
+export const goAgents = [
+ { driver: "oracle-go", binary: "oracle", race: false },
+ { driver: "xugu", binary: "xugu", race: false },
+ { driver: "rabbitmq", binary: "rabbitmq", race: false },
+ { driver: "rocketmq", binary: "rocketmq", race: true },
+ { driver: "zookeeper", binary: "zookeeper", race: true },
+ { driver: "cassandra-go", binary: "cassandra", race: false },
+ { driver: "hive-go", binary: "hive", race: false },
+ { driver: "vastbase-go", binary: "vastbase", race: false },
+ { driver: "neo4j-go", binary: "neo4j", race: false },
+ { driver: "iotdb", binary: "iotdb", race: false },
+];
+
+export const rustAgents = ["duckdb", "tdengine"];
+
+export const integrationCases = [
+ ...["3.4.14", "3.5.5", "3.7.0", "3.9.5"].map((version) => ({ driver: "zookeeper", scenario: "zookeeper", version })),
+ { driver: "zookeeper", scenario: "zookeeper-sasl", version: "3.7.0" },
+ ...["4.9.8", "5.3.1"].map((version) => ({ driver: "rocketmq", scenario: "rocketmq", version })),
+ ...["2.4.0.14", "2.6.0.34", "3.0.7.1", "3.3.6.13", "3.4.2.2"].map((version) => ({
+ driver: "tdengine", scenario: "tdengine", version,
+ image: `tdengine/${version === "3.4.2.2" ? "tsdb" : "tdengine"}:${version}`,
+ })),
+ ...["3.11.19", "5.0.6"].map((version) => ({ driver: "cassandra-go", scenario: "cassandra", version })),
+ ...["3.13", "4.3"].map((version) => ({ driver: "rabbitmq", scenario: "rabbitmq", version })),
+];
diff --git a/.github/scripts/ci-execution.test.mjs b/.github/scripts/ci-execution.test.mjs
new file mode 100644
index 0000000000..b8d37a853e
--- /dev/null
+++ b/.github/scripts/ci-execution.test.mjs
@@ -0,0 +1,121 @@
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import test from "node:test";
+import { goAgents, integrationCases } from "./ci-config.mjs";
+import { lockfileManifests } from "./ci-lockfiles.mjs";
+
+const root = fileURLToPath(new URL("../../", import.meta.url));
+const environmentKeys = ["CGO_ENABLED", "GOOS", "GOARCH", "GONOSUMDB", "DBX_ZOOKEEPER_TEST_CONNECT_STRING",
+ "DBX_ZOOKEEPER_TEST_AUTH_SCHEME", "DBX_ZOOKEEPER_TEST_USERNAME", "DBX_ZOOKEEPER_TEST_PASSWORD", "TDENGINE_INTEGRATION",
+ "TDENGINE_TEST_HOST", "TDENGINE_TEST_PORT", "CASSANDRA_TEST_HOST", "CASSANDRA_TEST_PORT", "RABBITMQ_INTEGRATION",
+ "RABBITMQ_USERNAME", "RABBITMQ_PASSWORD"];
+
+function withToolStubs(run, extraEnv = {}) {
+ const directory = mkdtempSync(path.join(tmpdir(), "dbx-ci-execution-"));
+ const binaries = path.join(directory, "bin");
+ mkdirSync(binaries);
+ const log = path.join(directory, "commands.jsonl");
+ const stub = path.join(binaries, "stub.cjs");
+ writeFileSync(stub, `#!/usr/bin/env node
+const fs = require('node:fs');
+const path = require('node:path');
+const tool = path.basename(process.argv[1]);
+const args = process.argv.slice(2);
+const env = Object.fromEntries(${JSON.stringify(environmentKeys)}.filter(name => process.env[name] !== undefined).map(name => [name, process.env[name]]));
+fs.appendFileSync(process.env.CI_STUB_LOG, JSON.stringify({tool,args,cwd:process.cwd(),env})+'\\n');
+if (process.env.CI_STUB_FAIL === tool) process.exit(23);
+if (tool === 'docker' && args.includes('taos')) console.log('| ready |');
+`, { mode: 0o755 });
+ for (const name of ["docker", "go", "cargo", "timeout", "curl", "sleep", "bash"]) symlinkSync(stub, path.join(binaries, name));
+ try {
+ const env = { ...process.env, ...extraEnv, TMPDIR: directory, CI_STUB_LOG: log, PATH: `${binaries}${path.delimiter}${process.env.PATH}` };
+ return run(env, () => readFileSync(log, "utf8").trim().split("\n").filter(Boolean).map((line) => JSON.parse(line)));
+ } finally { rmSync(directory, { recursive: true, force: true }); }
+}
+
+for (const agent of goAgents) {
+ test(`Go job preserves ${agent.driver} tests and target builds without executing compilers`, () => withToolStubs((env, commands) => {
+ const result = spawnSync("/bin/bash", [".github/scripts/ci-agent-go.sh", agent.driver, agent.binary, String(agent.race)], { cwd: root, env, encoding: "utf8" });
+ assert.equal(result.status, 0, result.stderr);
+ const invocations = commands();
+ assert.deepEqual(invocations[0].args, agent.race ? ["test", "-race", "./..."] : ["test", "./..."]);
+ assert.ok(invocations.every((entry) => entry.cwd === path.join(root, "agents/drivers", agent.driver)));
+ const builds = invocations.slice(1);
+ assert.equal(builds.length, agent.driver === "zookeeper" ? 6 : 1);
+ for (const build of builds) {
+ assert.deepEqual(build.args.slice(0, 3), ["build", "-trimpath", "-ldflags=-s -w"]);
+ assert.equal(build.env.CGO_ENABLED, "0");
+ }
+ if (agent.driver === "zookeeper") assert.deepEqual(builds.map((entry) => `${entry.env.GOOS}/${entry.env.GOARCH}`),
+ ["darwin/arm64", "darwin/amd64", "linux/arm64", "linux/amd64", "windows/arm64", "windows/amd64"]);
+ else assert.ok(builds[0].args.includes(`/tmp/dbx-agent-${agent.binary}-linux-x64`));
+ if (agent.driver === "xugu") assert.ok(invocations.every((entry) => entry.env.GONOSUMDB === "gitee.com/XuguDB/go-xugu-driver"));
+ }));
+}
+
+for (const entry of integrationCases) {
+ test(`live matrix retains ${entry.scenario}/${entry.version} commands without starting any container`, () => withToolStubs((env, commands) => {
+ const result = spawnSync("/bin/bash", [".github/scripts/ci-agent-integration.sh", entry.scenario, entry.version, entry.image ?? ""], { cwd: root, env, encoding: "utf8" });
+ assert.equal(result.status, 0, result.stderr);
+ const invocations = commands();
+ if (entry.scenario === "rocketmq") {
+ assert.ok(invocations.some((call) => call.tool === "bash" && call.args.at(-1) === entry.version));
+ return;
+ }
+ const start = invocations.find((call) => call.tool === "docker" && call.args[0] === "run");
+ assert.ok(start, "starts the selected container");
+ const expectedImage = entry.image ?? (entry.scenario.startsWith("zookeeper") ? `zookeeper:${entry.version}`
+ : entry.scenario === "rabbitmq" ? `rabbitmq:${entry.version}-management` : `cassandra:${entry.version}`);
+ assert.equal(start.args.at(-1), expectedImage);
+ const last = invocations.at(-1);
+ assert.equal(last.tool, "docker");
+ assert.deepEqual(last.args.slice(0, 2), ["rm", "-fv"]);
+ const tests = invocations.filter((call) => ["go", "cargo"].includes(call.tool));
+ assert.ok(tests.length);
+ if (entry.scenario === "zookeeper") {
+ assert.equal(tests.length, entry.version === "3.7.0" ? 2 : 1);
+ assert.ok(tests[0].args.includes("^TestZooKeeperIntegration$"));
+ if (entry.version === "3.7.0") assert.ok(tests[1].args.includes("^TestZooKeeperLargeChildrenIntegration$"));
+ } else if (entry.scenario === "zookeeper-sasl") {
+ assert.equal(tests[0].env.DBX_ZOOKEEPER_TEST_AUTH_SCHEME, "sasl_digest");
+ assert.equal(tests[0].env.DBX_ZOOKEEPER_TEST_USERNAME, "dbx");
+ assert.ok(start.args.some((arg) => arg.includes("sessionRequireClientSASLAuth=true")));
+ } else if (entry.scenario === "tdengine") {
+ assert.deepEqual(tests[0].args, ["test", "--manifest-path", "drivers/tdengine/Cargo.toml", "--locked", "--test", "live", "--", "--nocapture"]);
+ assert.equal(tests[0].env.TDENGINE_INTEGRATION, "1");
+ assert.equal(tests[0].env.TDENGINE_TEST_PORT, "6041");
+ } else if (entry.scenario === "cassandra") assert.ok(tests[0].args.includes("^TestCassandraIntegration$"));
+ else {
+ assert.ok(tests[0].args.includes("^TestRabbitMQIntegration$"));
+ assert.equal(tests[0].env.RABBITMQ_INTEGRATION, "1");
+ assert.ok(start.args.includes("999:999"));
+ assert.ok(start.args.includes("/var/lib/rabbitmq:rw,exec,uid=999,gid=999,mode=700"));
+ }
+ }));
+}
+
+test("failed live tests stay failed and still clean their container", () => withToolStubs((env, commands) => {
+ const result = spawnSync("/bin/bash", [".github/scripts/ci-agent-integration.sh", "cassandra", "5.0.6"], { cwd: root, env, encoding: "utf8" });
+ assert.equal(result.status, 23);
+ assert.deepEqual(commands().at(-1).args.slice(0, 2), ["rm", "-fv"]);
+}, { CI_STUB_FAIL: "go" }));
+
+test("lock preflight resolves full graphs, keeps --locked and propagates failures", () => {
+ const plan = { rust: true, agent_rust: { include: [{ driver: "duckdb" }, { driver: "tdengine" }] } };
+ assert.deepEqual(lockfileManifests(plan), ["Cargo.toml", "agents/drivers/duckdb/Cargo.toml", "agents/drivers/tdengine/Cargo.toml"]);
+ assert.throws(() => lockfileManifests({ rust: false, agent_rust: { include: [{ driver: "../../unknown" }] } }));
+ for (const failure of ["", "cargo"]) withToolStubs((env, commands) => {
+ const result = spawnSync(process.execPath, [".github/scripts/ci-lockfiles.mjs"], { cwd: root, env, encoding: "utf8" });
+ assert.equal(result.status, failure ? 23 : 0, result.stderr);
+ assert.equal(commands().length, failure ? 1 : 3);
+ for (const command of commands()) {
+ assert.equal(command.args[0], "metadata");
+ assert.ok(command.args.includes("--locked"));
+ assert.ok(!command.args.includes("--no-deps"));
+ }
+ }, { CI_PLAN: JSON.stringify(plan), CI_STUB_FAIL: failure });
+});
diff --git a/.github/scripts/ci-gate.mjs b/.github/scripts/ci-gate.mjs
new file mode 100644
index 0000000000..46ee5891e5
--- /dev/null
+++ b/.github/scripts/ci-gate.mjs
@@ -0,0 +1,40 @@
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+
+export function gateFailures(needs, mode) {
+ if (needs.changes?.result !== "success") return ["changes did not succeed"];
+ let plan;
+ try { plan = JSON.parse(needs.changes.outputs.plan); } catch { return ["missing or invalid CI plan"]; }
+ if (!plan || typeof plan !== "object" || Array.isArray(plan)) return ["incomplete CI plan"];
+ const flags = ["rust", "agents", "fast", "agent_java", "agent_go_changed", "agent_rust_changed", "agent_integration_changed"];
+ if (!flags.every((flag) => typeof plan[flag] === "boolean")) return ["incomplete CI plan"];
+ const routedJobs = { frontend: "frontend", packages: "packages", "github-scripts": "github_scripts",
+ "windows-win7-bundle": "windows_win7_bundle", "duckdb-windows-driver": "duckdb_windows",
+ jdbc: "jdbc", "offline-jdbc-release": "offline_jdbc", "nix-packaging": "nix" };
+ if (mode === "all" && !Object.values(routedJobs).every((output) => ["true", "false"].includes(needs.changes.outputs[output]))) {
+ return ["missing or invalid job selection outputs"];
+ }
+ const expected = mode === "rust" ? {
+ "fast-checks": plan.fast, "rust-fmt-clippy": plan.rust, "rust-test": plan.rust,
+ } : mode === "agents" ? {
+ "fast-checks": plan.fast, "agent-checks": plan.agents, "agent-java": plan.agent_java, "agent-go": plan.agent_go_changed,
+ "agent-rust": plan.agent_rust_changed, "agent-integration": plan.agent_integration_changed,
+ } : mode === "all" ? {
+ rust: true, agents: true, "fast-checks": plan.fast,
+ ...Object.fromEntries(Object.entries(routedJobs)
+ .map(([job, output]) => [job, needs.changes.outputs[output] === "true"])),
+ } : null;
+ if (!expected) return [`unknown gate: ${mode}`];
+ return Object.entries(expected).flatMap(([job, required]) => {
+ const result = needs[job]?.result;
+ return result === "success" || (!required && result === "skipped") ? [] : [`${job}: ${result ?? "missing"} (required=${required})`];
+ });
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
+ const failures = gateFailures(JSON.parse(process.env.NEEDS_JSON), process.argv[2]);
+ if (failures.length) {
+ console.error(failures.join("\n"));
+ process.exitCode = 1;
+ } else console.log("All selected CI jobs succeeded; only unselected jobs may be skipped.");
+}
diff --git a/.github/scripts/ci-guide.md b/.github/scripts/ci-guide.md
new file mode 100644
index 0000000000..d393757471
--- /dev/null
+++ b/.github/scripts/ci-guide.md
@@ -0,0 +1,108 @@
+# CI job boundaries
+
+Paths in this guide are relative to `.github/`.
+
+`workflows/ci.yml` retains one workflow and one cancellation group. Splitting jobs
+does not change release workflows or branch-protection settings. The existing
+`rust` and `agents` aggregate check names remain; `ci` additionally summarizes all
+selected jobs. None of these gates accepts a failed, cancelled, missing, or
+unexpectedly skipped prerequisite.
+
+## Selection
+
+`scripts/ci-plan.mjs` reads Cargo workspace metadata and the exact event base/head
+diff. Renames include both paths and deleted paths remain visible. Rust package
+selection follows reverse dependencies, including build and test dependencies.
+Connection-type descriptors belong to `dbx-types`; dialect descriptors belong to
+`dbx-sql`. A changed core dependency also selects the standalone DuckDB Agent tests,
+whose development dependency is `dbx-core`.
+
+- Ordinary Rust PRs select foundation, driver, and application test groups from
+ `scripts/ci-config.mjs`. Each group contains several packages, not one runner per
+ crate. At most three Rust test groups run concurrently.
+- Main pushes, shared Cargo manifests/locks, toolchain/config/vendor changes, CI
+ infrastructure changes, an unknown workspace member, or an unavailable diff
+ select the original full-workspace test lane instead of duplicating it with all
+ three groups.
+- Strict clippy still checks the complete workspace and all targets. The existing
+ full/fast feature distinction is retained: fast skips system-font discovery,
+ not the other previously enabled capabilities.
+- Native Agent source changes select the affected native driver and its live
+ cases. Shared Agent inputs conservatively select all existing Agent checks.
+ Windows DuckDB packaging is selected by its own inputs or shared build/lock
+ changes, not every change to its development dependency's source.
+- Existing frontend, package, Windows, JDBC, offline-payload and Nix jobs retain
+ their commands and path filters.
+
+Changes to the planner itself select the broad checks. Missing/invalid planning
+data fails the aggregate gates rather than producing a successful skip.
+
+## Fast checks and Rust feature coverage
+
+`fast-checks` runs formatting and CI/architecture contracts before dependency
+resolution or compilation. It checks generated connection types without rewriting
+them and validates the relevant root and independent Agent Cargo lockfiles with
+full `cargo metadata --locked`; `--no-deps` is intentionally not used for that
+validation.
+
+Cargo may unify features supplied by different workspace consumers. Grouped tests
+therefore explicitly retain the features previously enabled on each package,
+including plugin runtime defaults and the driver's bundled SQLite capability.
+`scripts/ci-rust-coverage.mjs` compares each selected workspace package's enabled
+features with the full-workspace invocation and verifies that the groups cover
+every workspace member exactly once. It fails if a selected package loses a
+feature. An unassigned new workspace member forces the full-workspace lane and
+bypasses only the group-specific audit until it is assigned a group. This does not substitute for
+the full-workspace integration lane, and does not use `--all-features` to combine
+incompatible SQLite backends.
+
+## Agent jobs
+
+| Job | Coverage | Maximum parallelism |
+| --- | --- | --- |
+| `agent-checks` | Existing Python/script tests and Agent source validation | 1 |
+| `agent-java` | Existing Java tests, `shadowJar`, and JAR validation | 1 |
+| `agent-rust` | DuckDB and TDengine tests; TDengine release build | 2 |
+| `agent-go` | Ten existing native Go test/build targets, original race flags, six ZooKeeper cross-build targets | 4 |
+| `agent-integration` | Sixteen existing engine/version/authentication cases | 4 |
+
+Matrix jobs set `fail-fast: false`, so one failure does not conceal results from
+the other selected cases. Live test predicates, readiness checks, time limits,
+credentials for disposable fixtures, and failure cleanup are retained.
+Native, Java and live jobs wait for fast checks and Agent source validation before
+starting expensive builds or containers.
+RocketMQ's existing integration harness starts official server JARs through Maven
+and Java rather than Docker; those matrix entries explicitly retain Java 21.
+
+| Integration scenario | Versions |
+| --- | --- |
+| ZooKeeper | 3.4.14, 3.5.5, 3.7.0, 3.9.5; large-child-list coverage remains on 3.7.0 |
+| ZooKeeper SASL DIGEST-MD5 | 3.7.0 |
+| RocketMQ | 4.9.8, 5.3.1 |
+| TDengine | 2.4.0.14, 2.6.0.34, 3.0.7.1, 3.3.6.13, 3.4.2.2; the last uses `tdengine/tsdb` |
+| Cassandra | 3.11.19, 5.0.6 |
+| RabbitMQ | 3.13, 4.3 |
+
+## Validation
+
+Run these from the repository root:
+
+```sh
+node --test .github/scripts/ci-*.test.mjs
+node .github/scripts/ci-rust-coverage.mjs
+actionlint .github/workflows/ci.yml
+bash -n .github/scripts/ci-agent-go.sh
+bash -n .github/scripts/ci-agent-integration.sh
+```
+
+The shell-command tests replace Docker, compilers, readiness probes and delegated
+integration scripts with recording stubs. They verify all configured cases and
+failure propagation without starting an engine; they are not live database tests.
+Do not run the integration shell script on a developer machine. It targets
+disposable Actions runners; equivalent validation on the designated test server
+must follow that server's isolated-container naming and port policies.
+
+The split aims to isolate failures, return small-check results earlier, and remove
+unrelated work. Rust groups can still repeat dependency compilation across runners;
+compare cold/warm build, link, test and queue time on actual runs before claiming a
+speedup or changing group sizes. Cancelled runs are not successful benchmarks.
diff --git a/.github/scripts/ci-lockfiles.mjs b/.github/scripts/ci-lockfiles.mjs
new file mode 100644
index 0000000000..b15b786aa4
--- /dev/null
+++ b/.github/scripts/ci-lockfiles.mjs
@@ -0,0 +1,23 @@
+import { spawnSync } from "node:child_process";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+
+export function lockfileManifests(plan) {
+ return [...(plan.rust ? ["Cargo.toml"] : []),
+ ...plan.agent_rust.include.map(({ driver }) => {
+ if (!["duckdb", "tdengine"].includes(driver)) throw new Error(`Unknown native Rust driver: ${driver}`);
+ return `agents/drivers/${driver}/Cargo.toml`;
+ })];
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
+ const plan = JSON.parse(process.env.CI_PLAN);
+ for (const manifest of lockfileManifests(plan)) {
+ console.log(`Checking locked dependency resolution: ${manifest}`);
+ const result = spawnSync("cargo", ["metadata", "--locked", "--manifest-path", manifest, "--format-version", "1"], {
+ stdio: ["ignore", "ignore", "inherit"],
+ });
+ if (result.error) throw result.error;
+ if (result.status !== 0) process.exit(result.status ?? 1);
+ }
+}
diff --git a/.github/scripts/ci-plan.mjs b/.github/scripts/ci-plan.mjs
new file mode 100644
index 0000000000..b97cfad679
--- /dev/null
+++ b/.github/scripts/ci-plan.mjs
@@ -0,0 +1,93 @@
+import { appendFileSync } from "node:fs";
+import { execFileSync } from "node:child_process";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import { goAgents, integrationCases, rustAgents, rustGroups } from "./ci-config.mjs";
+
+export function changedPaths(base, root) {
+ if (!base || /^0+$/.test(base)) return null;
+ execFileSync("git", ["rev-parse", "--verify", `${base}^{commit}`], { cwd: root, stdio: "pipe" });
+ return execFileSync("git", ["diff", "--no-renames", "--name-only", "-z", base, "HEAD", "--"], {
+ cwd: root, encoding: "utf8", maxBuffer: 16 * 1024 * 1024,
+ }).split("\0").filter(Boolean);
+}
+
+export function planCi({ files, metadata, root, eventName = "pull_request", rustChanged = false, agentsChanged = false }) {
+ const unknownDiff = files === null;
+ files ??= [];
+ const packages = metadata.packages.filter((pkg) => metadata.workspace_members.includes(pkg.id));
+ const members = new Map(packages.map((pkg) => [pkg.name, pkg]));
+ const packagePaths = packages.map((pkg) => [pkg.name, `${path.relative(root, path.dirname(pkg.manifest_path)).split(path.sep).join("/")}/`]);
+ const ciChanged = unknownDiff || files.some((file) => /^\.github\/(?:workflows\/ci(?:[.-]|\/)|scripts\/ci-|actions\/ci-)/.test(file));
+ const sharedRust = ciChanged || files.some((file) => /^(?:Cargo\.(?:toml|lock)$|rust-toolchain|\.cargo\/|vendor\/)/.test(file)
+ || /^(?:crates\/[^/]+|src-tauri)\/Cargo\.toml$/.test(file));
+ const affected = new Set();
+ let unknownRust = false;
+ for (const file of files) {
+ const owner = packagePaths.find(([, directory]) => file.startsWith(directory));
+ if (owner) affected.add(owner[0]);
+ else if (/^(?:crates|src-tauri)\//.test(file)) unknownRust = true;
+ if (file.startsWith("plugins/connection-types/")) affected.add("dbx-types");
+ if (file.startsWith("plugins/dialects/")) affected.add("dbx-sql");
+ }
+ const knownGroups = new Set(Object.values(rustGroups).flat());
+ const unknownMember = packages.some((pkg) => !knownGroups.has(pkg.name));
+ const rust = rustChanged || sharedRust || unknownRust || affected.size > 0 || files.includes("scripts/core-architecture.test.mjs");
+ const full = rust && (eventName !== "pull_request" || sharedRust || unknownRust || unknownMember || affected.size === 0);
+ if (full) for (const pkg of packages) affected.add(pkg.name);
+ let expanded;
+ do {
+ expanded = false;
+ for (const pkg of packages) {
+ if (!affected.has(pkg.name) && pkg.dependencies.some((dependency) => members.has(dependency.name) && affected.has(dependency.name))) {
+ affected.add(pkg.name);
+ expanded = true;
+ }
+ }
+ } while (expanded);
+ const rustMatrix = !rust ? [] : full ? [{ group: "workspace" }] : Object.entries(rustGroups)
+ .filter(([, names]) => names.some((name) => affected.has(name)))
+ .map(([group]) => ({ group }));
+ const nativeDrivers = [...goAgents.map((agent) => agent.driver), ...rustAgents];
+ const nativeChanges = new Set(nativeDrivers.filter((driver) => files.some((file) => file.startsWith(`agents/drivers/${driver}/`))));
+ const sharedAgents = ciChanged || sharedRust || files.some((file) => file.startsWith("agents/")
+ && !nativeDrivers.some((driver) => file.startsWith(`agents/drivers/${driver}/`)))
+ || files.some((file) => file.startsWith(".github/scripts/bump-agent-versions.") || file === ".github/workflows/agents-release.yml"
+ || file === "crates/dbx-drivers/assets/agent-protocol-v2.json");
+ const allAgents = sharedAgents || (agentsChanged && nativeChanges.size === 0);
+ if (allAgents) {
+ for (const driver of nativeDrivers) nativeChanges.add(driver);
+ }
+ if (affected.has("dbx-core")) nativeChanges.add("duckdb");
+ const goMatrix = goAgents.filter((agent) => nativeChanges.has(agent.driver));
+ const nativeRustMatrix = rustAgents.filter((driver) => nativeChanges.has(driver)).map((driver) => ({ driver }));
+ const liveMatrix = integrationCases.filter((entry) => nativeChanges.has(entry.driver));
+ const java = allAgents;
+ const agents = agentsChanged || sharedAgents || nativeChanges.size > 0;
+ return {
+ rust, rust_full: full, rust_matrix: { include: rustMatrix }, rust_groups_known: !unknownMember,
+ affected_packages: [...affected].sort(),
+ agents, agent_java: java, agent_go: { include: goMatrix }, agent_rust: { include: nativeRustMatrix },
+ agent_integration: { include: liveMatrix }, agent_go_changed: goMatrix.length > 0,
+ agent_rust_changed: nativeRustMatrix.length > 0, agent_integration_changed: liveMatrix.length > 0,
+ duckdb_changed: nativeChanges.has("duckdb"),
+ duckdb_windows: sharedRust || files.some((file) => file.startsWith("agents/drivers/duckdb/")),
+ fast: rust || agents || ciChanged,
+ };
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
+ const root = process.cwd();
+ const metadata = JSON.parse(execFileSync("cargo", ["metadata", "--locked", "--offline", "--no-deps", "--format-version", "1"], {
+ cwd: root, encoding: "utf8", maxBuffer: 16 * 1024 * 1024,
+ }));
+ const plan = planCi({
+ files: changedPaths(process.env.BASE_SHA, root), metadata, root, eventName: process.env.GITHUB_EVENT_NAME,
+ rustChanged: process.env.RUST_CHANGED === "true", agentsChanged: process.env.AGENTS_CHANGED === "true",
+ });
+ if (process.env.GITHUB_OUTPUT) {
+ const outputs = { ...plan, plan };
+ appendFileSync(process.env.GITHUB_OUTPUT, Object.entries(outputs).map(([name, value]) => `${name}=${JSON.stringify(value)}\n`).join(""));
+ }
+ console.log(JSON.stringify(plan, null, 2));
+}
diff --git a/.github/scripts/ci-plan.test.mjs b/.github/scripts/ci-plan.test.mjs
new file mode 100644
index 0000000000..6b6166d8de
--- /dev/null
+++ b/.github/scripts/ci-plan.test.mjs
@@ -0,0 +1,232 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { execFileSync } from "node:child_process";
+import { goAgents, integrationCases, rustGroups } from "./ci-config.mjs";
+import { changedPaths, planCi } from "./ci-plan.mjs";
+import { rustCommand } from "./ci-rust.mjs";
+import { gateFailures } from "./ci-gate.mjs";
+import { assertCoverage, parseCoverage } from "./ci-rust-coverage.mjs";
+
+const root = "/fixture";
+const graph = {
+ "dbx-types": [], "dbx-platform": [], "dbx-formats": [], "dbx-sql": ["dbx-types"],
+ "dbx-ai-provider": ["dbx-platform"], "dbx-plugin-runtime": ["dbx-platform", "dbx-types"],
+ "dbx-sqlite-worker": [], "dbx-drivers": ["dbx-platform", "dbx-sql", "dbx-types", "dbx-sqlite-worker"],
+ "dbx-core": ["dbx-drivers", "dbx-sql", "dbx-types", "dbx-platform", "dbx-ai-provider", "dbx-plugin-runtime", "dbx-formats"],
+ "dbx-mcp": ["dbx-core"], "dbx-cli": ["dbx-core", "dbx-mcp"], "dbx-web": ["dbx-core", "dbx-mcp"], "dbx": ["dbx-core", "dbx-mcp"],
+};
+const metadata = {
+ workspace_members: Object.keys(graph),
+ packages: Object.entries(graph).map(([name, dependencies]) => ({
+ id: name, name, manifest_path: `${root}/${name === "dbx" ? "src-tauri" : `crates/${name}`}/Cargo.toml`,
+ dependencies: dependencies.map((dependency) => ({ name: dependency })),
+ })),
+};
+const plan = (files, options = {}) => planCi({ files, metadata, root, ...options });
+const groups = (result) => result.rust_matrix.include.map((entry) => entry.group);
+
+test("foundation changes select transitive consumers and standalone DuckDB", () => {
+ const result = plan(["crates/dbx-types/src/lib.rs"]);
+ assert.deepEqual(groups(result), ["foundation", "drivers", "application"]);
+ for (const name of ["dbx-types", "dbx-sql", "dbx-drivers", "dbx-core", "dbx-web", "dbx-cli", "dbx-mcp", "dbx"]) {
+ assert.ok(result.affected_packages.includes(name), name);
+ }
+ assert.deepEqual(result.agent_rust.include, [{ driver: "duckdb" }]);
+ assert.equal(result.duckdb_changed, true);
+ assert.equal(result.duckdb_windows, false);
+ assert.equal(result.rust_groups_known, true);
+ assert.equal(result.agent_java, false);
+});
+
+test("core and consumer changes do not pull unrelated foundation test groups", () => {
+ assert.deepEqual(groups(plan(["crates/dbx-core/src/query/mod.rs"])), ["application"]);
+ const cli = plan(["crates/dbx-cli/src/main.rs"]);
+ assert.deepEqual(groups(cli), ["application"]);
+ assert.equal(cli.agents, false);
+ assert.deepEqual(groups(plan(["crates/dbx-drivers/src/db/postgres.rs"])), ["drivers", "application"]);
+});
+
+test("code generation and test-only dependencies participate in impact analysis", () => {
+ assert.deepEqual(groups(plan(["plugins/connection-types/postgres.yaml"])), ["foundation", "drivers", "application"]);
+ assert.deepEqual(groups(plan(["plugins/dialects/postgres.yaml"])), ["foundation", "drivers", "application"]);
+ const fixture = structuredClone(metadata);
+ fixture.packages.find((pkg) => pkg.name === "dbx-types").dependencies.push({ name: "dbx-formats", kind: "dev" });
+ assert.ok(plan(["crates/dbx-formats/src/lib.rs"], { metadata: fixture }).affected_packages.includes("dbx-types"));
+});
+
+for (const file of ["Cargo.toml", "Cargo.lock", ".cargo/config.toml", "rust-toolchain.toml", "vendor/wry/src/lib.rs",
+ "crates/dbx-sql/Cargo.toml", ".github/workflows/ci.yml", ".github/scripts/ci-plan.mjs", ".github/scripts/ci-gate.test.mjs"]) {
+ test(`shared input ${file} retains full workspace and Agent coverage`, () => {
+ const result = plan([file]);
+ assert.deepEqual(groups(result), ["workspace"]);
+ assert.equal(result.rust_full, true);
+ assert.equal(result.agent_go.include.length, 10);
+ assert.equal(result.agent_rust.include.length, 2);
+ assert.equal(result.agent_integration.include.length, 16);
+ assert.equal(result.agent_java, true);
+ assert.equal(result.duckdb_windows, true);
+ });
+}
+
+test("main and unknown graph changes fail open to full test coverage", () => {
+ assert.deepEqual(groups(plan(["crates/dbx-core/src/lib.rs"], { eventName: "push" })), ["workspace"]);
+ assert.deepEqual(groups(plan(["crates/new-engine/src/lib.rs"])), ["workspace"]);
+ assert.deepEqual(groups(plan(null)), ["workspace"]);
+ const fixture = structuredClone(metadata);
+ fixture.packages.push({ name: "new-member", id: "new-member", manifest_path: `${root}/crates/new-member/Cargo.toml`, dependencies: [] });
+ fixture.workspace_members.push("new-member");
+ assert.deepEqual(groups(plan(["crates/dbx-core/src/lib.rs"], { metadata: fixture })), ["workspace"]);
+ assert.equal(plan(["crates/dbx-core/src/lib.rs"], { metadata: fixture }).rust_groups_known, false);
+ assert.deepEqual(groups(plan(["scripts/core-architecture.test.mjs"])), ["workspace"]);
+});
+
+test("unrelated docs skip Rust and Agents without inventing a matrix row", () => {
+ const result = plan(["docs/README.md"]);
+ assert.equal(result.rust, false);
+ assert.equal(result.agents, false);
+ assert.equal(result.fast, false);
+ assert.deepEqual(result.rust_matrix, { include: [] });
+});
+
+for (const driver of ["duckdb", "tdengine", ...goAgents.map((entry) => entry.driver)]) {
+ test(`native ${driver} changes only select their Agent tests and live cases`, () => {
+ const result = plan([`agents/drivers/${driver}/source`], { agentsChanged: true });
+ assert.equal(result.rust, false);
+ assert.equal(result.agent_java, false);
+ assert.ok([...result.agent_go.include, ...result.agent_rust.include].every((entry) => entry.driver === driver));
+ assert.ok(result.agent_integration.include.every((entry) => entry.driver === driver));
+ });
+}
+
+test("shared Agent inputs and unknown native modules never silently lose coverage", () => {
+ for (const file of ["agents/common/src/main/java/Protocol.java", "agents/scripts/validate_agents.py", "agents/build.gradle",
+ "agents/drivers/new-driver/main.go", "crates/dbx-drivers/assets/agent-protocol-v2.json", ".github/workflows/agents-release.yml"]) {
+ const result = plan([file]);
+ assert.equal(result.agent_go.include.length, 10, file);
+ assert.equal(result.agent_integration.include.length, 16, file);
+ assert.equal(result.agent_java, true, file);
+ }
+ const fallback = plan(["future-agent-filter-input"], { agentsChanged: true });
+ assert.equal(fallback.agent_go.include.length, 10);
+ assert.equal(fallback.agent_integration.include.length, 16);
+ assert.equal(fallback.agent_java, true);
+});
+
+test("the independent live matrix preserves every old version and authentication case", () => {
+ assert.deepEqual(integrationCases.map((entry) => `${entry.scenario}:${entry.version}`), [
+ "zookeeper:3.4.14", "zookeeper:3.5.5", "zookeeper:3.7.0", "zookeeper:3.9.5", "zookeeper-sasl:3.7.0",
+ "rocketmq:4.9.8", "rocketmq:5.3.1", "tdengine:2.4.0.14", "tdengine:2.6.0.34", "tdengine:3.0.7.1",
+ "tdengine:3.3.6.13", "tdengine:3.4.2.2", "cassandra:3.11.19", "cassandra:5.0.6", "rabbitmq:3.13", "rabbitmq:4.3",
+ ]);
+ assert.equal(integrationCases.find((entry) => entry.version === "3.4.2.2").image, "tdengine/tsdb:3.4.2.2");
+ assert.deepEqual(goAgents.filter((entry) => entry.race).map((entry) => entry.driver), ["rocketmq", "zookeeper"]);
+});
+
+test("test groups partition every current workspace package exactly once", () => {
+ const selected = Object.values(rustGroups).flat();
+ assert.equal(new Set(selected).size, selected.length);
+ assert.deepEqual(selected.toSorted(), Object.keys(graph).toSorted());
+});
+
+test("workspace Cargo flags preserve full versus fast coverage and strict clippy", () => {
+ for (const mode of ["fast", "full"]) {
+ const command = rustCommand("test", "workspace", mode);
+ const features = command.at(-1).split(",");
+ assert.ok(command.includes("--workspace"));
+ assert.ok(command.includes("--locked"));
+ for (const pkg of ["dbx", "dbx-core", "dbx-web"]) {
+ for (const feature of ["duckdb-sidecar", "dynamodb", "mq-admin", "sqlite-sqlcipher"]) assert.ok(features.includes(`${pkg}/${feature}`));
+ assert.equal(features.includes(`${pkg}/system-fonts`), mode === "full");
+ }
+ assert.deepEqual(rustCommand("clippy", "workspace", mode).slice(-3), ["--", "-D", "warnings"]);
+ }
+ for (const group of Object.keys(rustGroups)) assert.ok(rustCommand("test", group, "fast").includes("--package"));
+ assert.throws(() => rustCommand("test", "unknown", "full"));
+ assert.throws(() => rustCommand("clippy", "foundation", "full"));
+});
+
+function results(files) {
+ const selected = plan(files);
+ return { changes: { result: "success", outputs: { plan: JSON.stringify(selected) } },
+ "fast-checks": { result: selected.fast ? "success" : "skipped" },
+ "rust-fmt-clippy": { result: selected.rust ? "success" : "skipped" }, "rust-test": { result: selected.rust ? "success" : "skipped" },
+ "agent-checks": { result: selected.agents ? "success" : "skipped" }, "agent-java": { result: selected.agent_java ? "success" : "skipped" },
+ "agent-go": { result: selected.agent_go_changed ? "success" : "skipped" },
+ "agent-rust": { result: selected.agent_rust_changed ? "success" : "skipped" },
+ "agent-integration": { result: selected.agent_integration_changed ? "success" : "skipped" } };
+}
+
+test("gates accept only successful selected jobs and intentionally skipped unselected jobs", () => {
+ for (const files of [["docs/README.md"], ["Cargo.lock"], ["agents/drivers/rabbitmq/main.go"]]) {
+ for (const mode of ["rust", "agents"]) assert.deepEqual(gateFailures(results(files), mode), []);
+ }
+ for (const result of ["skipped", "failure", "cancelled", undefined]) {
+ const needs = results(["Cargo.lock"]);
+ needs["rust-test"].result = result;
+ assert.ok(gateFailures(needs, "rust").length);
+ needs["agent-integration"].result = result;
+ assert.ok(gateFailures(needs, "agents").length);
+ }
+ const needs = results(["docs/README.md"]);
+ needs["agent-go"].result = "cancelled";
+ assert.ok(gateFailures(needs, "agents").length);
+ needs.changes.result = "failure";
+ assert.ok(gateFailures(needs, "rust").length);
+ assert.ok(gateFailures({ changes: { result: "success", outputs: { plan: "{}" } } }, "rust").length);
+ assert.ok(gateFailures({ changes: { result: "success", outputs: { plan: "null" } } }, "rust").length);
+});
+
+test("git diff routing includes both sides of renames, deleted files, and unusual filenames", () => {
+ const directory = mkdtempSync(path.join(tmpdir(), "dbx-ci-plan-"));
+ const git = (...args) => execFileSync("git", args, { cwd: directory, encoding: "utf8", stdio: "pipe" }).trim();
+ try {
+ git("init"); git("config", "user.email", "[email protected]"); git("config", "user.name", "CI Fixture");
+ writeFileSync(path.join(directory, "old.rs"), "unchanged source\n");
+ writeFileSync(path.join(directory, "deleted.rs"), "gone\n");
+ git("add", "."); git("-c", "core.hooksPath=/dev/null", "commit", "-m", "fixture baseline");
+ const base = git("rev-parse", "HEAD");
+ git("mv", "old.rs", "renamed\nfile.rs"); git("rm", "deleted.rs");
+ git("-c", "core.hooksPath=/dev/null", "commit", "-m", "fixture change");
+ assert.deepEqual(changedPaths(base, directory).toSorted(), ["old.rs", "deleted.rs", "renamed\nfile.rs"].toSorted());
+ assert.equal(changedPaths("0".repeat(40), directory), null);
+ assert.throws(() => changedPaths("not-a-ref", directory));
+ } finally { rmSync(directory, { recursive: true, force: true }); }
+});
+
+test("the final gate rejects absent routing outputs and skipped selected jobs", () => {
+ const needs = results(["docs/README.md"]);
+ const routed = { frontend: "frontend", packages: "packages", "github-scripts": "github_scripts",
+ "windows-win7-bundle": "windows_win7_bundle", "duckdb-windows-driver": "duckdb_windows", jdbc: "jdbc",
+ "offline-jdbc-release": "offline_jdbc", "nix-packaging": "nix" };
+ needs.rust = needs.agents = { result: "success" };
+ for (const [job, output] of Object.entries(routed)) {
+ needs[job] = { result: "skipped" };
+ needs.changes.outputs[output] = "false";
+ }
+ assert.deepEqual(gateFailures(needs, "all"), []);
+ needs.changes.outputs.frontend = "true";
+ assert.ok(gateFailures(needs, "all").length);
+ needs.frontend.result = "success";
+ assert.deepEqual(gateFailures(needs, "all"), []);
+ delete needs.changes.outputs.nix;
+ assert.ok(gateFailures(needs, "all").length);
+});
+
+test("coverage audit rejects lost features, omitted packages, and overlapping group selection", () => {
+ const coverage = { workspace: Object.fromEntries(Object.keys(graph).map((name) => [name, ["fixture"]])),
+ ...Object.fromEntries(Object.entries(rustGroups).map(([group, packages]) => [group, Object.fromEntries(packages.map((name) => [name, ["fixture"]]))])) };
+ assertCoverage(coverage);
+ const lost = structuredClone(coverage);
+ lost.foundation["dbx-types"] = [];
+ assert.throws(() => assertCoverage(lost), /lost workspace features/);
+ const missing = structuredClone(coverage);
+ delete missing.drivers["dbx-drivers"];
+ assert.throws(() => assertCoverage(missing), /wrong packages/);
+ const overlap = structuredClone(coverage);
+ overlap.foundation["dbx-core"] = ["fixture"];
+ assert.throws(() => assertCoverage(overlap), /wrong packages/);
+ assert.deepEqual(parseCoverage("dbx-core v0.1.0 (/fixture)|default,openapi\n"), { "dbx-core": ["default", "openapi"] });
+});
diff --git a/.github/scripts/ci-rust-coverage.mjs b/.github/scripts/ci-rust-coverage.mjs
new file mode 100644
index 0000000000..149c1e7366
--- /dev/null
+++ b/.github/scripts/ci-rust-coverage.mjs
@@ -0,0 +1,39 @@
+import assert from "node:assert/strict";
+import { execFileSync } from "node:child_process";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import { rustGroups } from "./ci-config.mjs";
+import { rustCommand } from "./ci-rust.mjs";
+
+export function parseCoverage(output) {
+ return Object.fromEntries(output.trim().split("\n").filter(Boolean).map((line) => {
+ const [name, features] = line.split("|");
+ assert.ok(features !== undefined, `Invalid Cargo tree output: ${line}`);
+ return [name.split(" ")[0], features.split(",").filter(Boolean)];
+ }));
+}
+
+export function assertCoverage(coverage) {
+ const names = Object.values(rustGroups).flat();
+ assert.equal(new Set(names).size, names.length, "Rust test groups overlap");
+ assert.deepEqual(names.toSorted(), Object.keys(coverage.workspace).toSorted(), "Rust test groups must cover every workspace package");
+ for (const [group, packages] of Object.entries(rustGroups)) {
+ assert.deepEqual(Object.keys(coverage[group]).toSorted(), packages.toSorted(), `${group} selected the wrong packages`);
+ for (const name of packages) {
+ const missing = coverage.workspace[name].filter((feature) => !coverage[group][name].includes(feature));
+ assert.deepEqual(missing, [], `${group}/${name} lost workspace features`);
+ }
+ }
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
+ const coverage = {};
+ for (const group of ["workspace", ...Object.keys(rustGroups)]) {
+ const args = rustCommand("test", group, "fast");
+ args[0] = "tree";
+ args.push("--depth", "0", "--prefix", "none", "--format", "{p}|{f}");
+ coverage[group] = parseCoverage(execFileSync("cargo", args, { encoding: "utf8", maxBuffer: 8 * 1024 * 1024, stdio: ["ignore", "pipe", "inherit"] }));
+ }
+ assertCoverage(coverage);
+ console.log(JSON.stringify(coverage, null, 2));
+}
diff --git a/.github/scripts/ci-rust.mjs b/.github/scripts/ci-rust.mjs
new file mode 100644
index 0000000000..9a190f79ca
--- /dev/null
+++ b/.github/scripts/ci-rust.mjs
@@ -0,0 +1,31 @@
+import { spawnSync } from "node:child_process";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import { rustGroups } from "./ci-config.mjs";
+
+export function rustCommand(action, group, mode) {
+ if (!["test", "clippy"].includes(action) || !["full", "fast"].includes(mode)
+ || (group !== "workspace" && !Object.hasOwn(rustGroups, group)) || (action === "clippy" && group !== "workspace")) {
+ throw new Error(`Unknown Rust CI configuration: ${action}/${group}/${mode}`);
+ }
+ const capabilityFeatures = ["duckdb-sidecar", "dynamodb", "mq-admin", "sqlite-sqlcipher"];
+ if (mode === "full") capabilityFeatures.push("system-fonts");
+ const appFeatures = ["dbx", "dbx-core", "dbx-web"].flatMap((name) => capabilityFeatures.map((feature) => `${name}/${feature}`));
+ const foundationFeatures = ["dbx-types/mq-admin", "dbx-types/openapi", "dbx-sql/duckdb-sidecar", "dbx-sql/openapi",
+ "dbx-platform/downloads", "dbx-platform/host-prompts", "dbx-platform/test-support", "dbx-plugin-runtime/default", "dbx-plugin-runtime/test-support"];
+ const features = group === "foundation" ? foundationFeatures : group === "drivers" ? [
+ ...["duckdb-sidecar", "dynamodb", "mq-admin", "sqlite-bundled", "sqlite-sqlcipher", "test-support"].map((feature) => `dbx-drivers/${feature}`),
+ "dbx-sqlite-worker/runtime", "dbx-types/openapi", "dbx-sql/openapi",
+ ] : appFeatures;
+ const packages = group === "workspace" ? ["--workspace"] : rustGroups[group].flatMap((name) => ["--package", name]);
+ return [action, ...packages, "--locked", ...(action === "clippy" ? ["--all-targets"] : []),
+ "--no-default-features", "--features", features.join(","), ...(action === "clippy" ? ["--", "-D", "warnings"] : [])];
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
+ const args = rustCommand(...process.argv.slice(2, 5));
+ console.log(`cargo ${args.join(" ")}`);
+ const result = spawnSync("cargo", args, { stdio: "inherit" });
+ if (result.error) throw result.error;
+ process.exit(result.status ?? 1);
+}
diff --git a/.github/scripts/ci-workflow.test.mjs b/.github/scripts/ci-workflow.test.mjs
new file mode 100644
index 0000000000..4ff8ebf2e8
--- /dev/null
+++ b/.github/scripts/ci-workflow.test.mjs
@@ -0,0 +1,73 @@
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import test from "node:test";
+
+const workflow = readFileSync(new URL("../workflows/ci.yml", import.meta.url), "utf8");
+function job(name) {
+ const jobs = workflow.slice(workflow.indexOf("\njobs:\n") + 7);
+ const pattern = /^ ([\w-]+):\s*$/gm;
+ const definitions = [...jobs.matchAll(pattern)];
+ const index = definitions.findIndex((match) => match[1] === name);
+ assert.ok(index >= 0, `missing job ${name}`);
+ return jobs.slice(definitions[index].index, definitions[index + 1]?.index ?? jobs.length);
+}
+
+test("fast checks run format and contracts before graph resolution or compilation", () => {
+ const fast = job("fast-checks");
+ for (const command of ["cargo fmt --check", "node --test scripts/core-architecture.test.mjs",
+ "node scripts/sync-connection-types.mjs --check", "node .github/scripts/ci-lockfiles.mjs", "node .github/scripts/ci-rust-coverage.mjs"]) assert.ok(fast.includes(command));
+ assert.ok(fast.indexOf("cargo fmt --check") < fast.indexOf("ci-lockfiles.mjs"));
+ assert.doesNotMatch(fast, /cargo (?:test|build|clippy)/);
+ assert.ok(fast.includes("needs.changes.outputs.rust_groups_known == 'true'"));
+});
+
+test("Agent and Rust matrices are bounded and do not cancel sibling failures", () => {
+ for (const [name, output, parallel] of [["rust-test", "rust_matrix", 3], ["agent-rust", "agent_rust", 2],
+ ["agent-go", "agent_go", 4], ["agent-integration", "agent_integration", 4]]) {
+ const content = job(name);
+ assert.match(content, /fail-fast: false/);
+ assert.ok(content.includes(`max-parallel: ${parallel}`));
+ assert.ok(content.includes(`fromJSON(needs.changes.outputs.${output})`));
+ assert.match(content, /if: needs\.changes\.outputs\.[\w_]+ == 'true'/);
+ if (name.startsWith("agent-")) assert.ok(content.includes("needs: [changes, fast-checks, agent-checks]"));
+ }
+});
+
+test("stable Rust, Agent and overall gates always inspect selected upstream results", () => {
+ for (const [name, mode, dependencies] of [["rust", "rust", ["fast-checks", "rust-fmt-clippy", "rust-test"]],
+ ["agents", "agents", ["fast-checks", "agent-checks", "agent-rust", "agent-go", "agent-integration", "agent-java"]],
+ ["ci", "all", ["rust", "agents", "frontend", "packages", "windows-win7-bundle", "duckdb-windows-driver", "nix-packaging"]]]) {
+ const content = job(name);
+ assert.match(content, /if: always\(\)/);
+ assert.ok(content.includes(`node .github/scripts/ci-gate.mjs ${mode}`));
+ assert.ok(content.includes("${{ toJSON(needs) }}"));
+ for (const dependency of dependencies) assert.match(content, new RegExp(`^ - ${dependency}$`, "m"));
+ }
+});
+
+test("every old Agent stage has an independent owner and Java packaging remains strict", () => {
+ assert.ok(job("agent-checks").includes("python3 -m unittest discover"));
+ assert.ok(job("agent-checks").includes("python3 scripts/validate_agents.py"));
+ assert.ok(job("agent-checks").includes("bump-agent-versions.test.mjs"));
+ assert.ok(job("agent-java").includes("./gradlew test shadowJar --continue"));
+ assert.ok(job("agent-java").includes("python3 scripts/validate_agent_jars.py"));
+ assert.ok(job("agent-rust").includes('cargo test --manifest-path "drivers/$DRIVER/Cargo.toml" --locked'));
+ assert.ok(job("agent-rust").includes("cargo build --manifest-path drivers/tdengine/Cargo.toml --locked --release --bin dbx-tdengine-driver"));
+ assert.ok(job("agent-go").includes("ci-agent-go.sh"));
+ assert.ok(job("agent-integration").includes("ci-agent-integration.sh"));
+ assert.ok(job("agent-integration").includes("Setup RocketMQ server Java"));
+ assert.ok(job("agent-integration").includes("if: matrix.scenario == 'rocketmq'"));
+ assert.ok(job("agent-integration").includes('java-version: "21"'));
+ for (const name of ["agent-rust", "agent-go", "agent-integration", "agent-java"]) assert.doesNotMatch(job(name), /continue-on-error: true/);
+});
+
+test("the planner uses the exact event base and preserves a single workflow cancellation scope", () => {
+ const changes = job("changes");
+ assert.ok(changes.includes("github.event.pull_request.base.sha || github.event.before"));
+ assert.ok(changes.includes("node .github/scripts/ci-plan.mjs"));
+ for (const flag of ["rust", "rust_full", "rust_matrix", "agents", "agent_go", "agent_rust", "agent_integration", "plan"]) {
+ assert.ok(changes.includes(`steps.plan.outputs.${flag}`));
+ }
+ assert.ok(workflow.includes("group: ${{ github.workflow }}-${{ github.ref }}"));
+ assert.ok(workflow.includes("cancel-in-progress: true"));
+});
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8bfaad5b65..d011a4827c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -2,9 +2,9 @@ name: CI
on:
push:
- branches: [main]
+ branches: [ main ]
pull_request:
- branches: [main]
+ branches: [ main ]
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@@ -255,7 +255,9 @@ jobs:
"$DRIVER" < /dev/null
rust-fmt-clippy:
- needs: changes
+ needs:
+ - changes
+ - fast-checks
if: needs.changes.outputs.rust == 'true'
runs-on: ubuntu-22.04
env:
@@ -264,8 +266,6 @@ jobs:
RUSTC_WRAPPER: sccache
# The fast lane skips only system font discovery while retaining the other default capabilities.
RUST_FEATURE_MODE: ${{ github.event_name == 'pull_request' && needs.changes.outputs.rust_full != 'true' && 'fast' || 'full' }}
- RUST_FAST_FEATURES: dbx/duckdb-sidecar,dbx/dynamodb,dbx/mq-admin,dbx/sqlite-sqlcipher,dbx-core/duckdb-sidecar,dbx-core/dynamodb,dbx-core/mq-admin,dbx-core/sqlite-sqlcipher,dbx-web/duckdb-sidecar,dbx-web/dynamodb,dbx-web/mq-admin,dbx-web/sqlite-sqlcipher
- RUST_FULL_FEATURES: dbx/duckdb-sidecar,dbx/dynamodb,dbx/mq-admin,dbx/sqlite-sqlcipher,dbx/system-fonts,dbx-core/duckdb-sidecar,dbx-core/dynamodb,dbx-core/mq-admin,dbx-core/sqlite-sqlcipher,dbx-core/system-fonts,dbx-web/duckdb-sidecar,dbx-web/dynamodb,dbx-web/mq-admin,dbx-web/sqlite-sqlcipher,dbx-web/system-fonts
# Fork PRs cannot read repository secrets, so retain the GHA backend for them.
SCCACHE_GHA_ENABLED: ${{ secrets.SCCACHE_S3_BUCKET == '' && 'true' || 'false' }}
steps:
@@ -324,19 +324,9 @@ jobs:
# PR caches are large and branch-scoped; restore them from main without saving per-PR copies.
save-if: ${{ github.ref == 'refs/heads/main' }}
- - name: Core architecture contracts
- run: node --test scripts/core-architecture.test.mjs
-
- - name: Cargo fmt check
- run: cargo fmt --check
-
- name: Cargo clippy
- run: |
- if [ "$RUST_FEATURE_MODE" = "fast" ]; then
- cargo clippy --workspace --locked --all-targets --no-default-features --features "$RUST_FAST_FEATURES" -- -D warnings
- else
- cargo clippy --workspace --locked --all-targets --no-default-features --features "$RUST_FULL_FEATURES" -- -D warnings
- fi
+ run: |-
+ node .github/scripts/ci-rust.mjs clippy workspace "$RUST_FEATURE_MODE"
- name: Show sccache stats
if: always()
@@ -344,7 +334,9 @@ jobs:
run: ${SCCACHE_PATH} --show-stats
rust-test:
- needs: changes
+ needs:
+ - changes
+ - fast-checks
if: needs.changes.outputs.rust == 'true'
runs-on: ubuntu-22.04
env:
@@ -355,10 +347,9 @@ jobs:
RUSTC_WRAPPER: sccache
# The fast lane skips only system font discovery while retaining the other default capabilities.
RUST_FEATURE_MODE: ${{ github.event_name == 'pull_request' && needs.changes.outputs.rust_full != 'true' && 'fast' || 'full' }}
- RUST_FAST_FEATURES: dbx/duckdb-sidecar,dbx/dynamodb,dbx/mq-admin,dbx/sqlite-sqlcipher,dbx-core/duckdb-sidecar,dbx-core/dynamodb,dbx-core/mq-admin,dbx-core/sqlite-sqlcipher,dbx-web/duckdb-sidecar,dbx-web/dynamodb,dbx-web/mq-admin,dbx-web/sqlite-sqlcipher
- RUST_FULL_FEATURES: dbx/duckdb-sidecar,dbx/dynamodb,dbx/mq-admin,dbx/sqlite-sqlcipher,dbx/system-fonts,dbx-core/duckdb-sidecar,dbx-core/dynamodb,dbx-core/mq-admin,dbx-core/sqlite-sqlcipher,dbx-core/system-fonts,dbx-web/duckdb-sidecar,dbx-web/dynamodb,dbx-web/mq-admin,dbx-web/sqlite-sqlcipher,dbx-web/system-fonts
# Fork PRs cannot read repository secrets, so retain the GHA backend for them.
SCCACHE_GHA_ENABLED: ${{ secrets.SCCACHE_S3_BUCKET == '' && 'true' || 'false' }}
+ RUST_TEST_GROUP: ${{ matrix.group }}
steps:
- uses: actions/checkout@v5
@@ -366,6 +357,7 @@ jobs:
run: |
sudo apt-get update
sudo apt-get install -y libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf libssl-dev libsecret-1-dev
+ if: matrix.group != 'foundation'
- name: Setup Rust
uses: dtolnay/[email protected]
@@ -416,33 +408,37 @@ jobs:
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Cargo test
- run: |
- if [ "$RUST_FEATURE_MODE" = "fast" ]; then
- cargo test --workspace --locked --no-default-features --features "$RUST_FAST_FEATURES"
- else
- cargo test --workspace --locked --no-default-features --features "$RUST_FULL_FEATURES"
- fi
+ run: |-
+ node .github/scripts/ci-rust.mjs test "$RUST_TEST_GROUP" "$RUST_FEATURE_MODE"
- name: Show sccache stats
if: always()
continue-on-error: true
run: ${SCCACHE_PATH} --show-stats
+ strategy:
+ fail-fast: false
+ max-parallel: 3
+ matrix: ${{ fromJSON(needs.changes.outputs.rust_matrix) }}
+ name: rust-test (${{ matrix.group }})
rust:
- needs: [changes, rust-fmt-clippy, rust-test]
- if: always() && needs.changes.outputs.rust == 'true'
+ needs:
+ - changes
+ - fast-checks
+ - rust-fmt-clippy
+ - rust-test
+ if: always()
runs-on: ubuntu-22.04
steps:
- - name: Check Rust jobs
- run: |
- if [ "${{ needs.rust-fmt-clippy.result }}" != "success" ]; then
- echo "rust-fmt-clippy result: ${{ needs.rust-fmt-clippy.result }}"
- exit 1
- fi
- if [ "${{ needs.rust-test.result }}" != "success" ]; then
- echo "rust-test result: ${{ needs.rust-test.result }}"
- exit 1
- fi
+ - uses: actions/checkout@v5
+ - name: Setup Node.js
+ uses: actions/setup-node@v6
+ with:
+ node-version: 22.13.0
+ - name: Check selected rust jobs
+ env:
+ NEEDS_JSON: ${{ toJSON(needs) }}
+ run: node .github/scripts/ci-gate.mjs rust
jdbc:
needs: changes
@@ -531,15 +527,26 @@ jobs:
outputs:
frontend: ${{ steps.filter.outputs.frontend }}
packages: ${{ steps.filter.outputs.packages }}
- rust: ${{ steps.filter.outputs.rust }}
- rust_full: ${{ steps.rust-mode.outputs.full }}
+ rust: ${{ steps.plan.outputs.rust }}
+ rust_full: ${{ steps.plan.outputs.rust_full }}
jdbc: ${{ steps.filter.outputs.jdbc }}
offline_jdbc: ${{ steps.filter.outputs.offline_jdbc }}
- agents: ${{ steps.filter.outputs.agents }}
- duckdb_windows: ${{ steps.filter.outputs.duckdb_windows }}
+ agents: ${{ steps.plan.outputs.agents }}
+ duckdb_windows: ${{ steps.filter.outputs.duckdb_windows == 'true' || steps.plan.outputs.duckdb_windows == 'true' }}
nix: ${{ steps.filter.outputs.nix }}
windows_win7_bundle: ${{ steps.filter.outputs.windows_win7_bundle }}
github_scripts: ${{ steps.filter.outputs.github_scripts }}
+ fast: ${{ steps.plan.outputs.fast }}
+ rust_matrix: ${{ steps.plan.outputs.rust_matrix }}
+ rust_groups_known: ${{ steps.plan.outputs.rust_groups_known }}
+ agent_java: ${{ steps.plan.outputs.agent_java }}
+ agent_go: ${{ steps.plan.outputs.agent_go }}
+ agent_rust: ${{ steps.plan.outputs.agent_rust }}
+ agent_integration: ${{ steps.plan.outputs.agent_integration }}
+ agent_go_changed: ${{ steps.plan.outputs.agent_go_changed }}
+ agent_rust_changed: ${{ steps.plan.outputs.agent_rust_changed }}
+ agent_integration_changed: ${{ steps.plan.outputs.agent_integration_changed }}
+ plan: ${{ steps.plan.outputs.plan }}
steps:
- uses: actions/checkout@v5
with:
@@ -659,37 +666,79 @@ jobs:
- '.github/workflows/ci.yml'
- 'apps/desktop/src/types/database.ts'
- 'crates/dbx-core/assets/database-drivers.manifest.json'
-
- - name: Select Rust feature coverage
- id: rust-mode
- shell: bash
+ - name: Setup Rust
+ uses: dtolnay/[email protected]
+ - name: Plan dependency-aware CI jobs
+ id: plan
env:
BASE_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}
RUST_CHANGED: ${{ steps.filter.outputs.rust }}
- run: |
- full=false
-
- if [ "$RUST_CHANGED" = "true" ] && [ "${{ github.event_name }}" = "push" ]; then
- full=true
- elif [ "$RUST_CHANGED" = "true" ]; then
- if [ -z "$BASE_SHA" ] || echo "$BASE_SHA" | grep -Eq '^0+$'; then
- BASE_SHA="HEAD~1"
- fi
-
- while IFS= read -r file; do
- case "$file" in
- Cargo.toml|Cargo.lock|rust-toolchain*|.github/workflows/ci.yml|*/Cargo.toml|vendor/*)
- full=true
- break
- ;;
- esac
-
- done < <(git diff --name-only "$BASE_SHA" HEAD -- Cargo.toml Cargo.lock 'rust-toolchain*' crates src-tauri vendor .github/workflows/ci.yml)
- fi
-
- echo "full=$full" >> "$GITHUB_OUTPUT"
+ AGENTS_CHANGED: ${{ steps.filter.outputs.agents }}
+ run: node .github/scripts/ci-plan.mjs
agents:
+ needs:
+ - changes
+ - fast-checks
+ - agent-checks
+ - agent-java
+ - agent-rust
+ - agent-go
+ - agent-integration
+ if: always()
+ runs-on: ubuntu-22.04
+ steps:
+ - uses: actions/checkout@v5
+ - name: Setup Node.js
+ uses: actions/setup-node@v6
+ with:
+ node-version: 22.13.0
+ - name: Check selected agents jobs
+ env:
+ NEEDS_JSON: ${{ toJSON(needs) }}
+ run: node .github/scripts/ci-gate.mjs agents
+ fast-checks:
+ needs: changes
+ if: needs.changes.outputs.fast == 'true'
+ runs-on: ubuntu-22.04
+ steps:
+ - uses: actions/checkout@v5
+ - name: Setup Rust
+ uses: dtolnay/[email protected]
+ with:
+ components: rustfmt
+ if: needs.changes.outputs.rust == 'true' || needs.changes.outputs.agent_rust_changed == 'true'
+ - name: Cargo fmt check
+ if: needs.changes.outputs.rust == 'true'
+ run: cargo fmt --check
+ - name: Setup Node.js
+ uses: actions/setup-node@v6
+ with:
+ node-version: 22.13.0
+ - name: CI planner and gate tests
+ run: node --test .github/scripts/ci-*.test.mjs
+ - name: Core architecture contracts
+ if: needs.changes.outputs.rust == 'true'
+ run: node --test scripts/core-architecture.test.mjs
+ - name: Setup pnpm
+ if: needs.changes.outputs.rust == 'true'
+ uses: pnpm/action-setup@v6
+ with:
+ version: 10.27.0
+ - name: Install generator dependencies
+ if: needs.changes.outputs.rust == 'true'
+ run: pnpm --filter dbx... install --frozen-lockfile --ignore-scripts
+ - name: Check generated connection types
+ if: needs.changes.outputs.rust == 'true'
+ run: node scripts/sync-connection-types.mjs --check
+ - name: Check workspace and standalone Cargo locks
+ env:
+ CI_PLAN: ${{ needs.changes.outputs.plan }}
+ run: node .github/scripts/ci-lockfiles.mjs
+ - name: Check grouped Rust feature coverage
+ if: needs.changes.outputs.rust == 'true' && needs.changes.outputs.rust_groups_known == 'true'
+ run: node .github/scripts/ci-rust-coverage.mjs
+ agent-checks:
needs: changes
if: needs.changes.outputs.agents == 'true'
runs-on: ubuntu-22.04
@@ -698,357 +747,163 @@ jobs:
working-directory: agents
steps:
- uses: actions/checkout@v5
-
- - name: Setup Java
- uses: actions/setup-java@v5
- with:
- distribution: temurin
- java-version: |
- 8
- 21
-
- - name: Setup Go
- uses: actions/setup-go@v5
+ - name: Setup Node.js
+ uses: actions/setup-node@v6
with:
- go-version: "1.25.x"
-
- - name: Setup Rust
- uses: dtolnay/[email protected]
-
- - name: Setup Gradle
- uses: gradle/actions/setup-gradle@v4
-
+ node-version: 22.13.0
- name: Install packaging tools
run: |
sudo apt-get update
sudo apt-get install -y zstd
-
- name: Agent script tests
run: |
python3 -m unittest discover -s scripts -p '*_test.py'
node --test ../.github/scripts/bump-agent-versions.test.mjs
-
- - name: DuckDB driver tests
- run: cargo test --manifest-path drivers/duckdb/Cargo.toml --locked
-
- - name: TDengine driver tests
- run: cargo test --manifest-path drivers/tdengine/Cargo.toml --locked
-
- name: Agent validation
run: python3 scripts/validate_agents.py
-
- - name: Oracle native agent tests
- run: go test ./...
- working-directory: agents/drivers/oracle-go
-
- - name: Xugu native agent tests
- run: GONOSUMDB=gitee.com/XuguDB/go-xugu-driver go test ./...
- working-directory: agents/drivers/xugu
-
- - name: RabbitMQ native agent tests
- run: go test ./...
- working-directory: agents/drivers/rabbitmq
-
- - name: RocketMQ native agent tests
- run: go test -race ./...
- working-directory: agents/drivers/rocketmq
-
- - name: ZooKeeper native agent tests
- run: go test -race ./...
- working-directory: agents/drivers/zookeeper
-
- - name: Cassandra native agent tests
- run: go test ./...
- working-directory: agents/drivers/cassandra-go
-
- - name: Hive native agent tests
- run: go test ./...
- working-directory: agents/drivers/hive-go
-
- - name: Vastbase native agent tests
- run: go test ./...
- working-directory: agents/drivers/vastbase-go
-
- - name: Neo4j native agent tests
- run: go test ./...
- working-directory: agents/drivers/neo4j-go
-
- - name: IoTDB native agent tests
- run: go test ./...
- working-directory: agents/drivers/iotdb
-
- - name: Oracle native agent build
- run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-oracle-linux-x64 .
- working-directory: agents/drivers/oracle-go
-
- - name: Xugu native agent build
- run: GONOSUMDB=gitee.com/XuguDB/go-xugu-driver CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-xugu-linux-x64 .
- working-directory: agents/drivers/xugu
-
- - name: RabbitMQ native agent build
- run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-rabbitmq-linux-x64 .
- working-directory: agents/drivers/rabbitmq
-
- - name: RocketMQ native agent build
- run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-rocketmq-linux-x64 .
- working-directory: agents/drivers/rocketmq
-
- - name: ZooKeeper native agent cross-builds
- shell: bash
- working-directory: agents/drivers/zookeeper
- run: |
- set -euo pipefail
- for target in darwin/arm64 darwin/amd64 linux/arm64 linux/amd64 windows/arm64 windows/amd64; do
- IFS=/ read -r goos goarch <<< "$target"
- output="/tmp/dbx-agent-zookeeper-${goos}-${goarch}"
- [ "$goos" = windows ] && output="${output}.exe"
- CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -trimpath -ldflags="-s -w" -o "$output" .
- done
-
- - name: ZooKeeper native agent integration tests
- shell: bash
- working-directory: agents/drivers/zookeeper
- run: |
- set -euo pipefail
- for version in 3.4.14 3.5.5 3.7.0 3.9.5; do
- name="dbx-zookeeper-${version//./-}"
- docker rm -fv "$name" >/dev/null 2>&1 || true
- docker run -d --name "$name" -p 2181:2181 "zookeeper:$version"
- cleanup() {
- docker rm -fv "$name" >/dev/null 2>&1 || true
- }
- trap cleanup EXIT
- ready=false
- for _ in $(seq 1 90); do
- if timeout 1 bash -c '</dev/tcp/127.0.0.1/2181' >/dev/null 2>&1; then
- ready=true
- break
- fi
- sleep 2
- done
- if [ "$ready" != true ]; then
- docker logs "$name"
- exit 1
- fi
- DBX_ZOOKEEPER_TEST_CONNECT_STRING=127.0.0.1:2181 \
- go test -run '^TestZooKeeperIntegration$' -count=1 ./...
- if [ "$version" = "3.7.0" ]; then
- DBX_ZOOKEEPER_TEST_CONNECT_STRING=127.0.0.1:2181 \
- go test -run '^TestZooKeeperLargeChildrenIntegration$' -count=1 ./...
- fi
- cleanup
- trap - EXIT
- done
-
- - name: ZooKeeper SASL DIGEST-MD5 integration test
- shell: bash
- working-directory: agents/drivers/zookeeper
- run: |
- set -euo pipefail
- name="dbx-zookeeper-sasl-3-7-0"
- jaas_file=$(mktemp)
- cat > "$jaas_file" <<'EOF'
- Server {
- org.apache.zookeeper.server.auth.DigestLoginModule required
- user_dbx="dbx-secret";
- };
- EOF
- chmod 0644 "$jaas_file"
- docker rm -fv "$name" >/dev/null 2>&1 || true
- docker run -d --name "$name" -p 2181:2181 \
- -v "$jaas_file:/conf/server-jaas.conf:ro" \
- -e ZOO_CFG_EXTRA="authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider" \
- -e JVMFLAGS="-Djava.security.auth.login.config=/conf/server-jaas.conf -Dzookeeper.sasl.serverconfig=Server -Dzookeeper.sessionRequireClientSASLAuth=true" \
- zookeeper:3.7.0
- cleanup() {
- docker rm -fv "$name" >/dev/null 2>&1 || true
- }
- trap cleanup EXIT
- ready=false
- for _ in $(seq 1 90); do
- if timeout 1 bash -c '</dev/tcp/127.0.0.1/2181' >/dev/null 2>&1; then
- ready=true
- break
- fi
- sleep 2
- done
- if [ "$ready" != true ]; then
- docker logs "$name"
- exit 1
- fi
- DBX_ZOOKEEPER_TEST_CONNECT_STRING=127.0.0.1:2181 \
- DBX_ZOOKEEPER_TEST_AUTH_SCHEME=sasl_digest \
- DBX_ZOOKEEPER_TEST_USERNAME=dbx \
- DBX_ZOOKEEPER_TEST_PASSWORD=dbx-secret \
- go test -run '^TestZooKeeperIntegration$' -count=1 ./...
-
- - name: RocketMQ native agent integration tests
- shell: bash
- working-directory: agents/drivers/rocketmq
- run: |
- set -euo pipefail
- for version in 4.9.8 5.3.1; do
- ./scripts/run-integration.sh "$version"
- done
-
- - name: Cassandra native agent build
- run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-cassandra-linux-x64 .
- working-directory: agents/drivers/cassandra-go
-
- - name: Hive native agent build
- run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-hive-linux-x64 .
- working-directory: agents/drivers/hive-go
-
- - name: Vastbase native agent build
- run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-vastbase-linux-x64 .
- working-directory: agents/drivers/vastbase-go
-
- - name: Neo4j native agent build
- run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-neo4j-linux-x64 .
- working-directory: agents/drivers/neo4j-go
-
- - name: IoTDB native agent build
- run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-iotdb-linux-x64 .
- working-directory: agents/drivers/iotdb
-
- - name: TDengine native agent build
- run: cargo build --manifest-path drivers/tdengine/Cargo.toml --locked --release --bin dbx-tdengine-driver
-
- - name: TDengine native agent integration tests
- shell: bash
- run: |
- set -euo pipefail
- while IFS='|' read -r version image; do
- name="dbx-tdengine-${version//./-}"
- docker rm -fv "$name" >/dev/null 2>&1 || true
- docker run -d --name "$name" \
- --memory 4g \
- --ulimit nofile=65535:65535 \
- -p 6030:6030 \
- -p 6041:6041 \
- "$image"
- cleanup() {
- docker rm -fv "$name" >/dev/null 2>&1 || true
- }
- trap cleanup EXIT
- ready=false
- dnodes=""
- for _ in $(seq 1 90); do
- dnodes="$(docker exec "$name" taos -s 'SHOW DNODES' 2>/dev/null || true)"
- if grep -Eq '\|[[:space:]]*ready[[:space:]]*\|' <<<"$dnodes"; then
- ready=true
- break
- fi
- sleep 2
- done
- if [ "$ready" != "true" ]; then
- printf '%s\n' "$dnodes"
- docker logs "$name"
- exit 1
- fi
- TDENGINE_INTEGRATION=1 \
- TDENGINE_TEST_HOST=127.0.0.1 \
- TDENGINE_TEST_PORT=6041 \
- cargo test --manifest-path drivers/tdengine/Cargo.toml --locked --test live -- --nocapture
- cleanup
- trap - EXIT
- done <<'EOF'
- 2.4.0.14|tdengine/tdengine:2.4.0.14
- 2.6.0.34|tdengine/tdengine:2.6.0.34
- 3.0.7.1|tdengine/tdengine:3.0.7.1
- 3.3.6.13|tdengine/tdengine:3.3.6.13
- 3.4.2.2|tdengine/tsdb:3.4.2.2
- EOF
-
- - name: Cassandra native agent integration tests
- shell: bash
- working-directory: agents/drivers/cassandra-go
- run: |
- set -euo pipefail
- for version in 3.11.19 5.0.6; do
- name="dbx-cassandra-${version//./-}"
- docker rm -fv "$name" >/dev/null 2>&1 || true
- docker run -d --name "$name" \
- -e CASSANDRA_CLUSTER_NAME="DBX Cassandra CI $version" \
- -e CASSANDRA_DC=dc1 \
- -e CASSANDRA_RACK=rack1 \
- -e CASSANDRA_ENDPOINT_SNITCH=GossipingPropertyFileSnitch \
- -e CASSANDRA_NUM_TOKENS=16 \
- -e MAX_HEAP_SIZE=512M \
- -e HEAP_NEWSIZE=100M \
- -p 9042:9042 \
- "cassandra:$version"
- cleanup() {
- docker rm -fv "$name" >/dev/null 2>&1 || true
- }
- trap cleanup EXIT
- ready=false
- for _ in $(seq 1 100); do
- if docker exec "$name" cqlsh -e 'SELECT release_version FROM system.local' >/dev/null 2>&1; then
- ready=true
- break
- fi
- sleep 3
- done
- if [ "$ready" != "true" ]; then
- docker logs "$name"
- exit 1
- fi
- CASSANDRA_TEST_HOST=127.0.0.1 \
- CASSANDRA_TEST_PORT=9042 \
- go test -run '^TestCassandraIntegration$' -count=1 ./...
- cleanup
- trap - EXIT
- done
-
- - name: RabbitMQ native agent integration tests
- shell: bash
- working-directory: agents/drivers/rabbitmq
+ agent-java:
+ needs: [changes, fast-checks, agent-checks]
+ if: needs.changes.outputs.agent_java == 'true'
+ runs-on: ubuntu-22.04
+ defaults:
+ run:
+ working-directory: agents
+ steps:
+ - uses: actions/checkout@v5
+ - name: Setup Java
+ uses: actions/setup-java@v5
+ with:
+ distribution: temurin
+ java-version: |-
+ 8
+ 21
+ - name: Setup Gradle
+ uses: gradle/actions/setup-gradle@v4
+ - name: Install packaging tools
run: |
- set -euo pipefail
- for version in 3.13 4.3; do
- name="dbx-rabbitmq-${version//./-}"
- cookie="dbx-ci-${GITHUB_RUN_ID:-local}-${version//./-}"
- # RabbitMQ is disposable in CI. Keep its data on a fresh tmpfs
- # owned by the image's rabbitmq user so .erlang.cookie is readable.
- docker rm -fv "$name" >/dev/null 2>&1 || true
- docker run -d --name "$name" \
- --user 999:999 \
- --tmpfs /var/lib/rabbitmq:rw,exec,uid=999,gid=999,mode=700 \
- -e RABBITMQ_DEFAULT_USER=dbx \
- -e RABBITMQ_DEFAULT_PASS=dbx-password \
- -e RABBITMQ_ERLANG_COOKIE="$cookie" \
- -p 5672:5672 -p 15672:15672 \
- "rabbitmq:${version}-management"
- cleanup() {
- docker rm -fv "$name" >/dev/null 2>&1 || true
- }
- trap cleanup EXIT
- ready=false
- for _ in $(seq 1 60); do
- if docker exec "$name" rabbitmq-diagnostics -q check_running >/dev/null 2>&1 \
- && curl --fail --silent --noproxy '*' --user dbx:dbx-password \
- http://127.0.0.1:15672/api/overview >/dev/null; then
- ready=true
- break
- fi
- sleep 2
- done
- if [ "$ready" != "true" ]; then
- docker inspect "$name" --format 'image={{.Config.Image}} user={{.Config.User}} status={{.State.Status}} exit={{.State.ExitCode}}' || true
- docker logs "$name"
- exit 1
- fi
- RABBITMQ_INTEGRATION=1 \
- RABBITMQ_USERNAME=dbx \
- RABBITMQ_PASSWORD=dbx-password \
- go test -run '^TestRabbitMQIntegration$' -count=1 ./...
- cleanup
- trap - EXIT
- done
-
+ sudo apt-get update
+ sudo apt-get install -y zstd
- name: Java agent tests and packages
run: ./gradlew test shadowJar --continue
-
- name: Agent jar validation
run: python3 scripts/validate_agent_jars.py
+ agent-rust:
+ needs: [changes, fast-checks, agent-checks]
+ if: needs.changes.outputs.agent_rust_changed == 'true'
+ runs-on: ubuntu-22.04
+ defaults:
+ run:
+ working-directory: agents
+ steps:
+ - uses: actions/checkout@v5
+ - name: Setup Rust
+ uses: dtolnay/[email protected]
+ - name: Rust cache
+ uses: swatinem/rust-cache@v2
+ with:
+ workspaces: agents/drivers/${{ matrix.driver }} -> target
+ cache-on-failure: true
+ save-if: ${{ github.ref == 'refs/heads/main' }}
+ - name: Native Rust driver tests
+ env:
+ DRIVER: ${{ matrix.driver }}
+ run: cargo test --manifest-path "drivers/$DRIVER/Cargo.toml" --locked
+ - name: TDengine native agent build
+ if: matrix.driver == 'tdengine'
+ run: cargo build --manifest-path drivers/tdengine/Cargo.toml --locked --release --bin dbx-tdengine-driver
+ strategy:
+ fail-fast: false
+ max-parallel: 2
+ matrix: ${{ fromJSON(needs.changes.outputs.agent_rust) }}
+ agent-go:
+ needs: [changes, fast-checks, agent-checks]
+ if: needs.changes.outputs.agent_go_changed == 'true'
+ runs-on: ubuntu-22.04
+ defaults:
+ run:
+ working-directory: agents
+ steps:
+ - uses: actions/checkout@v5
+ - name: Setup Go
+ uses: actions/setup-go@v5
+ with:
+ go-version: 1.25.x
+ cache-dependency-path: agents/drivers/${{ matrix.driver }}/go.sum
+ - name: Native Go tests and target builds
+ env:
+ DRIVER: ${{ matrix.driver }}
+ BINARY: ${{ matrix.binary }}
+ RACE: ${{ matrix.race }}
+ run: bash ../.github/scripts/ci-agent-go.sh "$DRIVER" "$BINARY" "$RACE"
+ strategy:
+ fail-fast: false
+ max-parallel: 4
+ matrix: ${{ fromJSON(needs.changes.outputs.agent_go) }}
+ agent-integration:
+ needs: [changes, fast-checks, agent-checks]
+ if: needs.changes.outputs.agent_integration_changed == 'true'
+ runs-on: ubuntu-22.04
+ defaults:
+ run:
+ working-directory: agents
+ steps:
+ - uses: actions/checkout@v5
+ - name: Setup RocketMQ server Java
+ if: matrix.scenario == 'rocketmq'
+ uses: actions/setup-java@v5
+ with:
+ distribution: temurin
+ java-version: "21"
+ - name: Setup Go
+ uses: actions/setup-go@v5
+ with:
+ go-version: 1.25.x
+ cache-dependency-path: agents/drivers/${{ matrix.driver }}/go.sum
+ if: matrix.driver != 'tdengine'
+ - name: Setup Rust
+ uses: dtolnay/[email protected]
+ if: matrix.driver == 'tdengine'
+ - name: TDengine Rust cache
+ if: matrix.driver == 'tdengine'
+ uses: swatinem/rust-cache@v2
+ with:
+ workspaces: agents/drivers/tdengine -> target
+ cache-on-failure: true
+ save-if: ${{ github.ref == 'refs/heads/main' && matrix.version == '3.4.2.2' }}
+ - name: Run one live engine and version
+ env:
+ SCENARIO: ${{ matrix.scenario }}
+ VERSION: ${{ matrix.version }}
+ IMAGE: ${{ matrix.image || '' }}
+ run: bash ../.github/scripts/ci-agent-integration.sh "$SCENARIO" "$VERSION" "$IMAGE"
+ name: agent-integration (${{ matrix.scenario }}, ${{ matrix.version }})
+ strategy:
+ fail-fast: false
+ max-parallel: 4
+ matrix: ${{ fromJSON(needs.changes.outputs.agent_integration) }}
+ ci:
+ needs:
+ - changes
+ - fast-checks
+ - rust
+ - agents
+ - frontend
+ - github-scripts
+ - packages
+ - windows-win7-bundle
+ - duckdb-windows-driver
+ - jdbc
+ - offline-jdbc-release
+ - nix-packaging
+ if: always()
+ runs-on: ubuntu-22.04
+ steps:
+ - uses: actions/checkout@v5
+ - name: Setup Node.js
+ uses: actions/setup-node@v6
+ with:
+ node-version: 22.13.0
+ - name: Check selected all jobs
+ env:
+ NEEDS_JSON: ${{ toJSON(needs) }}
+ run: node .github/scripts/ci-gate.mjs all