Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

feat(shell,ci): scripts and CI jobs SPAWNS the scripts they run

Martin Vogelb8a5050813df
diff --git a/Makefile.cbm b/Makefile.cbm
index fa664bd18..da481b1fa 100644
--- a/Makefile.cbm
+++ b/Makefile.cbm
@@ -521,6 +521,7 @@ PIPELINE_SRCS = \
src/pipeline/pass_operator.c \
src/pipeline/pass_shell.c \
src/pipeline/pass_terraform.c \
+ src/pipeline/pass_ci.c \
src/pipeline/pass_similarity.c \
src/pipeline/pass_semantic_edges.c \
src/pipeline/pass_complexity.c \
diff --git a/README.md b/README.md
index 8b439385a..c98c179e4 100644
--- a/README.md
+++ b/README.md
@@ -233,7 +233,7 @@ The install script placed beside the binary is **reported, not deleted** — uni
- `USAGE` — an identifier is used, but a unique callable target is not proven (including ambiguous or complex expressions)
- `IMPORTS`, `DEFINES`, `IMPLEMENTS`, `INHERITS`
- `HTTP_CALLS`, `ASYNC_CALLS` (cross-service)
-- `SPAWNS` — the caller starts another program (`subprocess.run`, `exec.Command`, `posix_spawn`, `Process.Start`, ...); the target is a `Process` node named after the program, or `<dynamic>` when the program is not a literal
+- `SPAWNS` — the caller starts another program (`subprocess.run`, `exec.Command`, `posix_spawn`, `Process.Start`, a shell command or Make recipe running a program by path, ...); the target is a `Process` node named after the program, or `<dynamic>` when the program is not a literal. When a shell script, Make recipe, GitHub Actions step or Prow job runs a script of the repository (`./hack/x.sh`, `bash hack/x.sh`), the target is that script's `Module` (else its `File`), and edges from CI carry `ci` and `job`
- `INFRA_MAPS` — a Kubernetes Service to each workload whose pod labels carry every `key: value` of the Service's selector, within one namespace
- `DEPENDS_ON` — a Helm chart to the charts it declares (the local chart for a resolvable `file://` repository, otherwise an external `Chart`; the edge keeps `repository`), and a `go.mod` / `requirements.txt` file to an external `Package`
- `EMITS`, `LISTENS_ON` (channels)
diff --git a/scripts/check-lsp-originality.sh b/scripts/check-lsp-originality.sh
index 3647ff50e..9ddb3944c 100755
--- a/scripts/check-lsp-originality.sh
+++ b/scripts/check-lsp-originality.sh
@@ -82,7 +82,8 @@ REFS=(
# Infrastructure hybrid LSP sources outside internal/cbm/lsp/ (scanned too).
INFRA_SRC=(internal/cbm/extract_k8s.c src/pipeline/pass_k8s.c src/pipeline/pass_operator.c
- src/pipeline/pass_shell.c src/pipeline/pass_terraform.c)
+ src/pipeline/pass_shell.c src/pipeline/pass_terraform.c
+ src/pipeline/pass_ci.c)
ONLY_LANG=""; REFRESH=0; LIST_ONLY=0
while [ $# -gt 0 ]; do
diff --git a/src/mcp/test_impact_engine_reach.c b/src/mcp/test_impact_engine_reach.c
index dd581a4f5..49a1142cc 100644
--- a/src/mcp/test_impact_engine_reach.c
+++ b/src/mcp/test_impact_engine_reach.c
@@ -22,8 +22,9 @@
* reached, never expanded (their caller is the runner's main). */
#define TE_RUNNER_ROLE_PREFIX "tf_maybe_run_"
-/* SPAWNS only carries reach from a Process node, which enters the walk
- * through te_spawn_lift alone. */
+/* SPAWNS carries reach from a Process node, which enters the walk through
+ * te_spawn_lift alone, and from a script of the repository (its Module /
+ * File, pass_shell.c) to whoever runs it. */
static const char *const te_walk_types[] = {"CALLS", "ASYNC_CALLS", "USAGE",
"CALL_REFERENCE", "READS", "SPAWNS"};
diff --git a/src/pipeline/pass_ci.c b/src/pipeline/pass_ci.c
new file mode 100644
index 000000000..2804e78d1
--- /dev/null
+++ b/src/pipeline/pass_ci.c
@@ -0,0 +1,727 @@
+/*
+ * pass_ci.c — the command lines CI configurations run, handed to the shell
+ * layer (pass_shell.c), which links what they deploy and start.
+ *
+ * PHASE 3 P3.5. Read with tree-sitter-yaml:
+ * - GitHub Actions: a workflow (.github/workflows/<name>.yml|yaml at the
+ * root -- GitHub reads no other) and a composite action (any
+ * action.yml|yaml with runs.using: composite). A step's run: is a shell
+ * text unless a shell other than bash / sh is named (step > job
+ * defaults.run > workflow defaults.run), or none is and the job's runs-on
+ * names a Windows runner (whose default is pwsh). It starts in the
+ * workspace -- the root -- or the working-directory (same precedence); env
+ * literals (workflow < job < step) are variables, a value holding `${{` an
+ * unknown one. In the text, `${{ github.workspace }}` is
+ * $GITHUB_WORKSPACE, `${{ github.action_path }}` $GITHUB_ACTION_PATH, any
+ * other `${{ }}` an unknown variable (its line breaks kept).
+ * - Prow: a file with a top-level presubmits / postsubmits / periodics key
+ * (in-repo job config). A job container's command + args is exec'd, no
+ * shell: the words are literal, a kubelet `$(VAR)` unknown; `bash -c TEXT`
+ * runs TEXT as shell. A decorated job starts in the repository's checkout
+ * (the root); undecorated, with an extra_ref taking the working directory,
+ * or with a container workingDir, the directory is unknown.
+ * Scalars: plain, quoted and literal block (|) values; a folded (>) block or
+ * one with an indentation indicator is skipped.
+ */
+#include "foundation/constants.h"
+#include "foundation/compat.h"
+#include "pipeline/pipeline_internal.h"
+#include "cbm.h"
+#include "lang_specs.h"
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+
+static const char CI_GHA[] = "github-actions";
+static const char CI_PROW[] = "prow";
+
+static const char CI_EXPR_OPEN[] = "${{";
+#define CI_EXPR_OPEN_LEN (sizeof(CI_EXPR_OPEN) - SKIP_ONE)
+static const char CI_QUOTED_QUOTE[] = "'\\''";
+#define CI_QUOTED_QUOTE_LEN (sizeof(CI_QUOTED_QUOTE) - SKIP_ONE)
+/* A rewritten run: text is at most this many times the original: the
+ * shortest expression `${{}}` (5 bytes) becomes `${__ci_expr}` (12). */
+enum { CI_TEXT_GROWTH = 3 };
+
+typedef struct {
+ CBMArena *a;
+ const char *src;
+ const char *rel;
+ cbm_ci_cmd_fn fn;
+ void *ud;
+} ci_t;
+
+/* name, value pairs; a value NULL: unknown. */
+typedef struct {
+ const char **kv;
+ int n;
+} ci_env_t;
+
+/* What a step inherits from its job and workflow. */
+typedef struct {
+ ci_env_t env;
+ const char *wd; /* working-directory; NULL: none named */
+ const char *shell; /* NULL: none named */
+ bool windows;
+ const char *action_dir; /* a composite action's directory */
+} ci_scope_t;
+
+typedef struct {
+ const char *key;
+ TSNode value; /* may be null: `key:` without a value */
+} ci_pair_t;
+
+static bool ci_type(TSNode n, const char *t) {
+ return !ts_node_is_null(n) && strcmp(ts_node_type(n), t) == 0;
+}
+
+/* The content of a node, through block_node / flow_node wrappers and past
+ * comments, anchors and tags; null for nothing (an alias stays an alias). */
+static TSNode ci_content(TSNode n) {
+ while (ci_type(n, "block_node") || ci_type(n, "flow_node")) {
+ TSNode next = {0};
+ uint32_t k = ts_node_named_child_count(n);
+ for (uint32_t i = 0; i < k; i++) {
+ TSNode c = ts_node_named_child(n, i);
+ if (!ci_type(c, "comment") && !ci_type(c, "anchor") && !ci_type(c, "tag")) {
+ next = c;
+ break;
+ }
+ }
+ n = next;
+ }
+ return n;
+}
+
+/* A flow scalar's value: line breaks folded (a single one to a space, n
+ * empty lines to n newlines), each line trimmed; `q` the quote (0: plain)
+ * whose escapes apply. */
+static const char *ci_fold(CBMArena *a, const char *s, size_t n, char q) {
+ char *out = cbm_arena_alloc(a, n + SKIP_ONE);
+ if (!out) {
+ return NULL;
+ }
+ size_t o = 0;
+ size_t i = 0;
+ while (i < n) {
+ char ch = s[i];
+ if (ch == '\n' || ch == '\r') {
+ while (o > 0 && (out[o - SKIP_ONE] == ' ' || out[o - SKIP_ONE] == '\t')) {
+ o--;
+ }
+ size_t j = i;
+ int breaks = 0;
+ while (j < n && (s[j] == ' ' || s[j] == '\t' || s[j] == '\r' || s[j] == '\n')) {
+ breaks += s[j] == '\n';
+ j++;
+ }
+ if (breaks > SKIP_ONE) {
+ for (int k = SKIP_ONE; k < breaks; k++) {
+ out[o++] = '\n';
+ }
+ } else {
+ out[o++] = ' ';
+ }
+ i = j;
+ continue;
+ }
+ if (q == '"' && ch == '\\' && i + SKIP_ONE < n) {
+ char x = s[i + SKIP_ONE];
+ i += PAIR_LEN;
+ if (x == '\n' || x == '\r') {
+ while (i < n && (s[i] == ' ' || s[i] == '\t' || s[i] == '\r' || s[i] == '\n')) {
+ i++;
+ }
+ continue;
+ }
+ out[o++] = x == 'n' ? '\n' : x == 't' ? '\t' : x;
+ continue;
+ }
+ if (q == '\'' && ch == '\'' && i + SKIP_ONE < n && s[i + SKIP_ONE] == '\'') {
+ out[o++] = '\'';
+ i += PAIR_LEN;
+ continue;
+ }
+ out[o++] = ch;
+ i++;
+ }
+ out[o] = '\0';
+ return out;
+}
+
+/* A literal block scalar's value (s at its `|`); NULL for a folded block or
+ * an indentation indicator. */
+static const char *ci_literal(CBMArena *a, const char *s, size_t n) {
+ if (n == 0 || s[0] != '|') {
+ return NULL;
+ }
+ size_t i = SKIP_ONE;
+ while (i < n && s[i] != '\n' && s[i] != '#') {
+ if (s[i] >= '1' && s[i] <= '9') {
+ return NULL;
+ }
+ i++;
+ }
+ while (i < n && s[i] != '\n') {
+ i++;
+ }
+ i += i < n ? SKIP_ONE : 0;
+ size_t indent = 0;
+ for (size_t j = i; j < n;) {
+ size_t k = j;
+ while (k < n && s[k] == ' ') {
+ k++;
+ }
+ if (k < n && s[k] != '\n' && s[k] != '\r') {
+ indent = k - j;
+ break;
+ }
+ while (k < n && s[k] != '\n') {
+ k++;
+ }
+ j = k + SKIP_ONE;
+ }
+ char *out = cbm_arena_alloc(a, n + SKIP_ONE);
+ if (!out) {
+ return NULL;
+ }
+ size_t o = 0;
+ while (i < n) {
+ size_t e = i;
+ while (e < n && s[e] != '\n') {
+ e++;
+ }
+ size_t k = i;
+ while (k < e && k - i < indent && s[k] == ' ') {
+ k++;
+ }
+ if (k - i == indent) {
+ memcpy(out + o, s + k, e - k);
+ o += e - k;
+ }
+ if (o > 0 && out[o - SKIP_ONE] == '\r') {
+ o--;
+ }
+ if (e < n) {
+ out[o++] = '\n';
+ }
+ i = e + SKIP_ONE;
+ }
+ out[o] = '\0';
+ return out;
+}
+
+/* A scalar's value and the 0-based row of its first character; NULL: no
+ * scalar this reader decodes. */
+static const char *ci_scalar(const ci_t *c, TSNode n, int *row) {
+ n = ci_content(n);
+ if (ts_node_is_null(n)) {
+ return NULL;
+ }
+ uint32_t b = ts_node_start_byte(n);
+ size_t len = ts_node_end_byte(n) - b;
+ const char *s = c->src + b;
+ if (row) {
+ *row = (int)ts_node_start_point(n).row;
+ }
+ if (ci_type(n, "plain_scalar")) {
+ return ci_fold(c->a, s, len, 0);
+ }
+ if (len >= PAIR_LEN && ci_type(n, "double_quote_scalar")) {
+ return ci_fold(c->a, s + SKIP_ONE, len - PAIR_LEN, '"');
+ }
+ if (len >= PAIR_LEN && ci_type(n, "single_quote_scalar")) {
+ return ci_fold(c->a, s + SKIP_ONE, len - PAIR_LEN, '\'');
+ }
+ if (ci_type(n, "block_scalar")) {
+ if (row) {
+ (*row)++;
+ }
+ return ci_literal(c->a, s, len);
+ }
+ return NULL;
+}
+
+/* The pairs of mapping `m` with a scalar key; *n their count. */
+static ci_pair_t *ci_pairs(const ci_t *c, TSNode m, int *n) {
+ *n = 0;
+ m = ci_content(m);
+ if (!ci_type(m, "block_mapping") && !ci_type(m, "flow_mapping")) {
+ return NULL;
+ }
+ uint32_t k = ts_node_named_child_count(m);
+ ci_pair_t *out = cbm_arena_alloc(c->a, (k + SKIP_ONE) * sizeof(*out));
+ if (!out) {
+ return NULL;
+ }
+ for (uint32_t i = 0; i < k; i++) {
+ TSNode p = ts_node_named_child(m, i);
+ if (!ci_type(p, "block_mapping_pair") && !ci_type(p, "flow_pair")) {
+ continue;
+ }
+ const char *key = ci_scalar(c, ts_node_child_by_field_name(p, TS_FIELD("key")), NULL);
+ if (key) {
+ out[(*n)++] = (ci_pair_t){key, ts_node_child_by_field_name(p, TS_FIELD("value"))};
+ }
+ }
+ return out;
+}
+
+/* The value of `key` in mapping `m` (the first such pair); null: none. */
+static TSNode ci_get(const ci_t *c, TSNode m, const char *key) {
+ int n = 0;
+ ci_pair_t *pairs = ci_pairs(c, m, &n);
+ for (int i = 0; i < n; i++) {
+ if (strcmp(pairs[i].key, key) == 0) {
+ return pairs[i].value;
+ }
+ }
+ TSNode none = {0};
+ return none;
+}
+
+static const char *ci_get_str(const ci_t *c, TSNode m, const char *key) {
+ return ci_scalar(c, ci_get(c, m, key), NULL);
+}
+
+/* The item contents of a sequence; *n their count. */
+static TSNode *ci_items(const ci_t *c, TSNode seq, int *n) {
+ *n = 0;
+ seq = ci_content(seq);
+ bool block = ci_type(seq, "block_sequence");
+ if (!block && !ci_type(seq, "flow_sequence")) {
+ return NULL;
+ }
+ uint32_t k = ts_node_named_child_count(seq);
+ TSNode *out = cbm_arena_alloc(c->a, (k + SKIP_ONE) * sizeof(*out));
+ if (!out) {
+ return NULL;
+ }
+ for (uint32_t i = 0; i < k; i++) {
+ TSNode it = ts_node_named_child(seq, i);
+ if (block && ci_type(it, "block_sequence_item")) {
+ TSNode v = {0};
+ for (uint32_t j = 0; j < ts_node_named_child_count(it); j++) {
+ TSNode x = ts_node_named_child(it, j);
+ if (!ci_type(x, "comment")) {
+ v = x;
+ break;
+ }
+ }
+ out[(*n)++] = ci_content(v);
+ } else if (!block && ci_type(it, "flow_node")) {
+ out[(*n)++] = ci_content(it);
+ }
+ }
+ return out;
+}
+
+/* `base` and then the pairs of env mapping `m`: a non-scalar or `${{` value
+ * is unknown, an absent one empty. */
+static ci_env_t ci_env_add(const ci_t *c, const ci_env_t *base, TSNode m) {
+ int n = 0;
+ ci_pair_t *pairs = ci_pairs(c, m, &n);
+ int bn = base ? base->n : 0;
+ ci_env_t out = {cbm_arena_alloc(c->a, (size_t)(bn + n + SKIP_ONE) * PAIR_LEN * sizeof(char *)),
+ 0};
+ if (!out.kv) {
+ return base ? *base : out;
+ }
+ for (int i = 0; i < bn; i++) {
+ out.kv[PAIR_LEN * out.n] = base->kv[PAIR_LEN * i];
+ out.kv[PAIR_LEN * out.n + SKIP_ONE] = base->kv[PAIR_LEN * i + SKIP_ONE];
+ out.n++;
+ }
+ for (int i = 0; i < n; i++) {
+ const char *v = ts_node_is_null(pairs[i].value) ? "" : ci_scalar(c, pairs[i].value, NULL);
+ out.kv[PAIR_LEN * out.n] = pairs[i].key;
+ out.kv[PAIR_LEN * out.n + SKIP_ONE] = v && !strstr(v, "${{") ? v : NULL;
+ out.n++;
+ }
+ return out;
+}
+
+/* ── GitHub Actions ────────────────────────────────────────────────── */
+
+/* The n bytes at b are exactly `word`. */
+static bool ci_span_is(const char *b, size_t n, const char *word) {
+ return strlen(word) == n && strncmp(b, word, n) == 0;
+}
+
+/* The run: text with its `${{ }}` expressions replaced (see the header),
+ * CBM_SZ_16 zero bytes past its end. */
+static const char *ci_gha_text(CBMArena *a, const char *t, size_t *len) {
+ size_t n = strlen(t);
+ char *out = cbm_arena_alloc(a, n * CI_TEXT_GROWTH + CBM_SZ_16 + CBM_SZ_16);
+ if (!out) {
+ return NULL;
+ }
+ size_t o = 0;
+ size_t i = 0;
+ while (i < n) {
+ const char *open = t + i + CI_EXPR_OPEN_LEN;
+ const char *end =
+ strncmp(t + i, CI_EXPR_OPEN, CI_EXPR_OPEN_LEN) == 0 ? strstr(open, "}}") : NULL;
+ if (!end) {
+ out[o++] = t[i++];
+ continue;
+ }
+ const char *b = open;
+ const char *e = end;
+ while (b < e && (*b == ' ' || *b == '\t')) {
+ b++;
+ }
+ while (e > b && (e[-SKIP_ONE] == ' ' || e[-SKIP_ONE] == '\t')) {
+ e--;
+ }
+ size_t il = (size_t)(e - b);
+ const char *rep = "${__ci_expr}";
+ if (ci_span_is(b, il, "github.workspace")) {
+ rep = "${GITHUB_WORKSPACE}";
+ } else if (ci_span_is(b, il, "github.action_path")) {
+ rep = "${GITHUB_ACTION_PATH}";
+ }
+ size_t rl = strlen(rep);
+ memcpy(out + o, rep, rl);
+ o += rl;
+ for (const char *p = open; p < end; p++) {
+ if (*p == '\n') {
+ out[o++] = '\n';
+ }
+ }
+ i = (size_t)(end - t) + PAIR_LEN;
+ }
+ memset(out + o, 0, CBM_SZ_16);
+ *len = o;
+ return out;
+}
+
+/* A shell GitHub runs a step with as bash: `bash ...` / `sh ...`. */
+static bool ci_bash_shell(const char *sh) {
+ size_t k = strncmp(sh, "bash", strlen("bash")) == 0 ? strlen("bash")
+ : strncmp(sh, "sh", strlen("sh")) == 0 ? strlen("sh")
+ : 0;
+ if (!k) {
+ return false;
+ }
+ char ch = sh[k];
+ return !(ch == '_' || (ch >= '0' && ch <= '9') || ((ch | ' ') >= 'a' && (ch | ' ') <= 'z'));
+}
+
+/* `s` contains `word` (lowercase ASCII), ignoring case. */
+static bool ci_has_word(const char *s, const char *word) {
+ size_t wl = strlen(word);
+ for (const char *p = s; p && *p; p++) {
+ size_t k = 0;
+ while (k < wl && p[k] && (char)(p[k] | ' ') == word[k]) {
+ k++;
+ }
+ if (k == wl) {
+ return true;
+ }
+ }
+ return false;
+}
+
+/* runs-on names a Windows runner: a label, or any label of a list. */
+static bool ci_windows(const ci_t *c, TSNode runs_on) {
+ int n = 0;
+ TSNode *items = ci_items(c, runs_on, &n);
+ if (!items) {
+ return ci_has_word(ci_scalar(c, runs_on, NULL), "windows");
+ }
+ for (int i = 0; i < n; i++) {
+ if (ci_has_word(ci_scalar(c, items[i], NULL), "windows")) {
+ return true;
+ }
+ }
+ return false;
+}
+
+static const char *ci_run_default(const ci_t *c, TSNode m, const char *key) {
+ return ci_get_str(c, ci_get(c, ci_get(c, m, "defaults"), "run"), key);
+}
+
+static void ci_gha_steps(const ci_t *c, TSNode steps, const char *job, const ci_scope_t *sc) {
+ int n = 0;
+ TSNode *items = ci_items(c, steps, &n);
+ for (int i = 0; i < n; i++) {
+ int row = 0;
+ const char *run = ci_scalar(c, ci_get(c, items[i], "run"), &row);
+ if (!run) {
+ continue;
+ }
+ const char *shell = ci_get_str(c, items[i], "shell");
+ shell = shell ? shell : sc->shell;
+ if (shell && shell[0] ? !ci_bash_shell(shell) : sc->windows) {
+ continue;
+ }
+ const char *wd = ci_get_str(c, items[i], "working-directory");
+ wd = wd ? wd : sc->wd;
+ if (!wd) {
+ wd = "";
+ } else if (strstr(wd, "${{") || wd[0] == '/' || wd[0] == '~') {
+ wd = NULL;
+ }
+ ci_env_t env = ci_env_add(c, &sc->env, ci_get(c, items[i], "env"));
+ size_t len = 0;
+ const char *text = ci_gha_text(c->a, run, &len);
+ if (!text) {
+ continue;
+ }
+ cbm_ci_cmd_t cmd = {.ci = CI_GHA,
+ .job = job,
+ .text = text,
+ .len = len,
+ .row = row,
+ .wd = wd,
+ .action_dir = sc->action_dir,
+ .env = env.kv,
+ .nenv = env.n,
+ .exec = false};
+ c->fn(&cmd, c->ud);
+ }
+}
+
+static void ci_workflow(const ci_t *c, TSNode root) {
+ ci_env_t wenv = ci_env_add(c, NULL, ci_get(c, root, "env"));
+ const char *wwd = ci_run_default(c, root, "working-directory");
+ const char *wsh = ci_run_default(c, root, "shell");
+ int n = 0;
+ ci_pair_t *jobs = ci_pairs(c, ci_get(c, root, "jobs"), &n);
+ for (int i = 0; i < n; i++) {
+ TSNode j = jobs[i].value;
+ const char *wd = ci_run_default(c, j, "working-directory");
+ const char *sh = ci_run_default(c, j, "shell");
+ ci_scope_t sc = {.env = ci_env_add(c, &wenv, ci_get(c, j, "env")),
+ .wd = wd ? wd : wwd,
+ .shell = sh ? sh : wsh,
+ .windows = ci_windows(c, ci_get(c, j, "runs-on")),
+ .action_dir = NULL};
+ ci_gha_steps(c, ci_get(c, j, "steps"), jobs[i].key, &sc);
+ }
+}
+
+static void ci_action(const ci_t *c, TSNode root) {
+ TSNode runs = ci_get(c, root, "runs");
+ const char *using = ci_get_str(c, runs, "using");
+ if (!using || strcmp(using, "composite") != 0) {
+ return;
+ }
+ const char *slash = strrchr(c->rel, '/');
+ ci_scope_t sc = {.action_dir =
+ slash ? cbm_arena_strndup(c->a, c->rel, (size_t)(slash - c->rel)) : ""};
+ ci_gha_steps(c, ci_get(c, runs, "steps"), NULL, &sc);
+}
+
+/* ── Prow ──────────────────────────────────────────────────────────── */
+
+static bool ci_is_shell(const char *w) {
+ static const char *const shells[] = {"bash", "sh", "zsh", "ksh", "dash", NULL};
+ const char *slash = strrchr(w, '/');
+ const char *base = slash ? slash + SKIP_ONE : w;
+ for (int i = 0; shells[i]; i++) {
+ if (strcmp(base, shells[i]) == 0) {
+ return true;
+ }
+ }
+ return false;
+}
+
+/* argv as one shell line of quoted words (a `$(VAR)` word unknown), with
+ * CBM_SZ_16 zero bytes past its end. */
+static const char *ci_exec_line(CBMArena *a, const char **w, int n, size_t *len) {
+ size_t cap = CBM_SZ_16 + CBM_SZ_16;
+ for (int i = 0; i < n; i++) {
+ cap += strlen(w[i]) * CI_QUOTED_QUOTE_LEN + CBM_SZ_16;
+ }
+ char *out = cbm_arena_alloc(a, cap);
+ if (!out) {
+ return NULL;
+ }
+ size_t o = 0;
+ for (int i = 0; i < n; i++) {
+ if (i) {
+ out[o++] = ' ';
+ }
+ if (strstr(w[i], "$(")) {
+ static const char unknown[] = "\"$__k8s_var\"";
+ memcpy(out + o, unknown, sizeof(unknown) - SKIP_ONE);
+ o += sizeof(unknown) - SKIP_ONE;
+ continue;
+ }
+ out[o++] = '\'';
+ for (const char *p = w[i]; *p; p++) {
+ if (*p == '\'') {
+ memcpy(out + o, CI_QUOTED_QUOTE, CI_QUOTED_QUOTE_LEN);
+ o += CI_QUOTED_QUOTE_LEN;
+ } else {
+ out[o++] = *p;
+ }
+ }
+ out[o++] = '\'';
+ }
+ memset(out + o, 0, CBM_SZ_16);
+ *len = o;
+ return out;
+}
+
+static void ci_prow_container(const ci_t *c, TSNode ct, const char *job, bool root_wd) {
+ int nc = 0;
+ int na = 0;
+ TSNode *cmd = ci_items(c, ci_get(c, ct, "command"), &nc);
+ TSNode *args = ci_items(c, ci_get(c, ct, "args"), &na);
+ int n = nc + na;
+ if (n == 0) {
+ return;
+ }
+ const char **w = cbm_arena_alloc(c->a, (size_t)n * sizeof(*w));
+ int *rows = cbm_arena_alloc(c->a, (size_t)n * sizeof(*rows));
+ if (!w || !rows) {
+ return;
+ }
+ for (int i = 0; i < n; i++) {
+ w[i] = ci_scalar(c, i < nc ? cmd[i] : args[i - nc], &rows[i]);
+ if (!w[i]) {
+ return;
+ }
+ }
+ const char *wdir = ci_get_str(c, ct, "workingDir");
+ cbm_ci_cmd_t out = {.ci = CI_PROW, .job = job, .wd = root_wd && !(wdir && wdir[0]) ? "" : NULL};
+ if (ci_is_shell(w[0])) {
+ int k = SKIP_ONE;
+ while (k < n && strcmp(w[k], "-c") != 0) {
+ k++;
+ }
+ if (k < n) {
+ if (k + SKIP_ONE >= n) {
+ return;
+ }
+ size_t tl = strlen(w[k + SKIP_ONE]);
+ char *text = cbm_arena_alloc(c->a, tl + CBM_SZ_16);
+ if (!text) {
+ return;
+ }
+ memcpy(text, w[k + SKIP_ONE], tl);
+ memset(text + tl, 0, CBM_SZ_16);
+ out.text = text;
+ out.len = tl;
+ out.row = rows[k + SKIP_ONE];
+ c->fn(&out, c->ud);
+ return;
+ }
+ }
+ out.text = ci_exec_line(c->a, w, n, &out.len);
+ out.row = rows[0];
+ out.exec = true;
+ if (out.text) {
+ c->fn(&out, c->ud);
+ }
+}
+
+static void ci_prow_job(const ci_t *c, TSNode j) {
+ const char *name = ci_get_str(c, j, "name");
+ const char *decorate = ci_get_str(c, j, "decorate");
+ bool root_wd = decorate && strcmp(decorate, "true") == 0;
+ int n = 0;
+ TSNode *refs = ci_items(c, ci_get(c, j, "extra_refs"), &n);
+ for (int i = 0; i < n; i++) {
+ const char *wd = ci_get_str(c, refs[i], "workdir");
+ root_wd = root_wd && !(wd && strcmp(wd, "true") == 0);
+ }
+ TSNode *cts = ci_items(c, ci_get(c, ci_get(c, j, "spec"), "containers"), &n);
+ for (int i = 0; i < n; i++) {
+ ci_prow_container(c, cts[i], name, root_wd);
+ }
+}
+
+static void ci_prow_jobs(const ci_t *c, TSNode seq) {
+ int n = 0;
+ TSNode *jobs = ci_items(c, seq, &n);
+ for (int i = 0; i < n; i++) {
+ ci_prow_job(c, jobs[i]);
+ }
+}
+
+static void ci_prow(const ci_t *c, TSNode root) {
+ static const char *const keys[] = {"presubmits", "postsubmits", "periodics", NULL};
+ for (int k = 0; keys[k]; k++) {
+ TSNode v = ci_content(ci_get(c, root, keys[k]));
+ int n = 0;
+ ci_pair_t *repos = ci_pairs(c, v, &n);
+ if (!repos) {
+ ci_prow_jobs(c, v);
+ continue;
+ }
+ for (int i = 0; i < n; i++) {
+ ci_prow_jobs(c, repos[i].value);
+ }
+ }
+}
+
+/* ── Entry ─────────────────────────────────────────────────────────── */
+
+static bool ci_yaml_ext(const char *p) {
+ const char *dot = strrchr(p, '.');
+ return dot && (strcmp(dot, ".yml") == 0 || strcmp(dot, ".yaml") == 0);
+}
+
+static bool ci_workflow_path(const char *rel) {
+ static const char dir[] = ".github/workflows/";
+ return strncmp(rel, dir, sizeof(dir) - SKIP_ONE) == 0 &&
+ !strchr(rel + sizeof(dir) - SKIP_ONE, '/') && ci_yaml_ext(rel);
+}
+
+static bool ci_action_path(const char *rel) {
+ const char *slash = strrchr(rel, '/');
+ const char *base = slash ? slash + SKIP_ONE : rel;
+ return strcmp(base, "action.yml") == 0 || strcmp(base, "action.yaml") == 0;
+}
+
+bool cbm_ci_path(const char *rel) {
+ return rel && (ci_workflow_path(rel) || ci_action_path(rel));
+}
+
+void cbm_ci_commands(CBMArena *a, const char *rel, const char *src, size_t len, cbm_ci_cmd_fn fn,
+ void *ud) {
+ if (!ci_yaml_ext(rel)) {
+ return;
+ }
+ TSParser *parser = ts_parser_new();
+ if (!parser) {
+ return;
+ }
+ TSTree *tree = NULL;
+ if (ts_parser_set_language(parser, cbm_ts_language(CBM_LANG_YAML))) {
+ tree = ts_parser_parse_string(parser, NULL, src, (uint32_t)len);
+ }
+ if (tree) {
+ ci_t c = {.a = a, .src = src, .rel = rel, .fn = fn, .ud = ud};
+ TSNode stream = ts_tree_root_node(tree);
+ for (uint32_t i = 0; i < ts_node_named_child_count(stream); i++) {
+ TSNode doc = ts_node_named_child(stream, i);
+ if (!ci_type(doc, "document")) {
+ continue;
+ }
+ TSNode root = {0};
+ for (uint32_t k = 0; k < ts_node_named_child_count(doc); k++) {
+ TSNode x = ts_node_named_child(doc, k);
+ if (ci_type(x, "block_node") || ci_type(x, "flow_node")) {
+ root = ci_content(x);
+ break;
+ }
+ }
+ if (ci_workflow_path(rel)) {
+ ci_workflow(&c, root);
+ } else if (ci_action_path(rel)) {
+ ci_action(&c, root);
+ } else {
+ ci_prow(&c, root);
+ }
+ }
+ ts_tree_delete(tree);
+ }
+ ts_parser_delete(parser);
+}
diff --git a/src/pipeline/pass_shell.c b/src/pipeline/pass_shell.c
index 02fc6fe2e..fca3b49fb 100644
--- a/src/pipeline/pass_shell.c
+++ b/src/pipeline/pass_shell.c
@@ -1,5 +1,5 @@
/*
- * pass_shell.c — the Bash (+ Make) infrastructure layer (PHASE 3 P3.2).
+ * pass_shell.c — the Bash (+ Make, CI) infrastructure layer (PHASE 3 P3.2, P3.5).
*
* A shell script or a Make recipe that puts manifests, a kustomization or a
* chart on a cluster DEPLOYS them: an edge from the function running the
@@ -18,8 +18,14 @@
*
* A program started by a path (./_build/kubermatic-installer, ${DIR}/x,
* $(LOCALBIN)/controller-gen) SPAWNS the Process of its name, as
- * pass_spawns.c draws a call that starts a program; a path naming a file of
- * the repository is a script of its own and not linked here.
+ * pass_spawns.c draws a call that starts a program. A program that is a file
+ * of the repository -- the path names one, or a shell runs one
+ * (bash|sh|zsh|ksh|dash [-opts] FILE) -- SPAWNS that script instead: its
+ * Module (the source of its own top-level edges), else its File (P3.5).
+ *
+ * CI (P3.5): the command lines GitHub Actions steps and Prow job containers
+ * run (pass_ci.c reads them) evaluate the same way, from the step's
+ * directory with its env literals; their edges carry the CI and the job.
*
* Words are evaluated by constant propagation over the script, nothing is
* executed: assignments (local / export / readonly / declare, arrays and
@@ -365,9 +371,11 @@ typedef struct {
const char *via; /* static */
int64_t tgt; /* the target node; 0: the external chart `ext` */
const char *ext; /* pass-arena */
- const char *path; /* pass-arena: the deployed path or chart */
+ const char *path; /* pass-arena: the deployed path or chart; a spawned script */
+ const char *ci; /* static: the CI running the command; NULL: none */
+ const char *job; /* pass-arena: its job; NULL: none */
bool chart;
- bool spawn; /* SPAWNS the Process `ext`, not DEPLOYS */
+ bool spawn; /* SPAWNS the script `tgt` / the Process `ext`, not DEPLOYS */
} sh_hit_t;
typedef struct {
@@ -438,7 +446,10 @@ typedef struct {
const char *file; /* the script / Makefile path */
const char *script; /* $0: anchored script path; NULL: none (Make) */
const char *sdir; /* the script's directory */
- int line_base; /* Make: the recipe line's row */
+ int line_base; /* Make / CI: the text's first row */
+ const char *ci; /* the CI running the text (static); NULL: none */
+ const char *job; /* its job; NULL: none */
+ bool exec; /* an argv exec'd without a shell (Prow) */
int in_func;
CBMHashTable *env; /* name -> const sh_val_t * */
sh_undo_t *undo;
@@ -1292,21 +1303,30 @@ static const char *sh_cd_target(sh_t *s, const char *v, sh_cwd_t *out) {
/* ── Deploy commands ───────────────────────────────────────────────── */
-static void sh_hit(sh_t *s, TSNode cmd, const char *via, const sh_target_t *t, const char *ext,
- bool chart) {
+/* Record hit `x` of command `cmd`: its file, line and CI context are the
+ * state's; ext / path are copied. */
+static void sh_hit_push(sh_t *s, TSNode cmd, sh_hit_t x) {
sh_hits_t *h = s->hits;
if (!sh_grow((void **)&h->items, &h->cap, h->count, sizeof(*h->items))) {
return;
}
- const char *path = t ? t->path : ext;
- h->items[h->count++] =
- (sh_hit_t){.file = s->file,
- .line = (int)ts_node_start_point(cmd).row + SKIP_ONE + s->line_base,
- .via = via,
- .tgt = t ? t->id : 0,
- .ext = ext ? cbm_arena_strdup(&h->arena, ext) : NULL,
- .path = path ? cbm_arena_strdup(&h->arena, path) : NULL,
- .chart = chart};
+ x.file = s->file;
+ x.line = (int)ts_node_start_point(cmd).row + SKIP_ONE + s->line_base;
+ x.ext = x.ext ? cbm_arena_strdup(&h->arena, x.ext) : NULL;
+ x.path = x.path ? cbm_arena_strdup(&h->arena, x.path) : NULL;
+ x.ci = s->ci;
+ x.job = s->job ? cbm_arena_strdup(&h->arena, s->job) : NULL;
+ h->items[h->count++] = x;
+}
+
+static void sh_hit(sh_t *s, TSNode cmd, const char *via, const sh_target_t *t, const char *ext,
+ bool chart) {
+ sh_hit_push(s, cmd,
+ (sh_hit_t){.via = via,
+ .tgt = t ? t->id : 0,
+ .ext = ext,
+ .path = t ? t->path : ext,
+ .chart = chart});
}
static void sh_hit_all(sh_t *s, TSNode cmd, const char *via, const sh_targets_t *ts, bool chart) {
@@ -1336,10 +1356,23 @@ static bool sh_numberish(const char *v) {
return true;
}
+/* Word i of argv is an option, assignment or count a wrapper takes before
+ * the program: by its value, or -- the value unknown -- by its text
+ * (env OUT="$TMP/x" prog: the assignment is no program). */
+static bool sh_wrapper_arg(const sh_t *s, const sh_argv_t *a, int i) {
+ const char *v = a->v[i];
+ if (v) {
+ return v[0] == '-' || strchr(v, '=') || sh_numberish(v);
+ }
+ uint32_t b = a->span[PAIR_LEN * i];
+ uint32_t e = a->span[PAIR_LEN * i + SKIP_ONE];
+ return e > b && (s->src[b] == '-' || memchr(s->src + b, '=', e - b));
+}
+
/* The index of the program argv runs, past wrappers (time, sudo, retry N,
* timeout D, env A=B, ...); -1 when argv has none. Its value may be unknown
* (NULL). */
-static int sh_program(const sh_argv_t *a) {
+static int sh_program(const sh_t *s, const sh_argv_t *a) {
static const char *const wrappers[] = {"time", "sudo", "exec", "nice", "nohup",
"command", "env", "timeout", "retry", NULL};
int i = 0;
@@ -1352,8 +1385,7 @@ static int sh_program(const sh_argv_t *a) {
return -1; /* command -v X: is there an X, not a run of it */
}
i++;
- while (i < a->n && a->v[i] &&
- (a->v[i][0] == '-' || strchr(a->v[i], '=') || sh_numberish(a->v[i]))) {
+ while (i < a->n && sh_wrapper_arg(s, a, i)) {
i++;
}
}
@@ -1679,7 +1711,7 @@ static void sh_find(sh_t *s, TSNode cmd, const sh_argv_t *a, int prog) {
sh_argv_push(&e, a->v[i], a->split[i], a->span[PAIR_LEN * i],
a->span[PAIR_LEN * i + SKIP_ONE]);
}
- int p = sh_program(&e);
+ int p = sh_program(s, &e);
if (p >= 0 && e.v[p] && strcmp(sh_basename(e.v[p]), "kubectl") == 0) {
sh_kubectl_t k = {0};
sh_kubectl_parse(&e, p + SKIP_ONE, &k);
@@ -1701,7 +1733,7 @@ static void sh_find(sh_t *s, TSNode cmd, const sh_argv_t *a, int prog) {
static void sh_deploy_command(sh_t *s, TSNode cmd, const sh_argv_t *a, const sh_src_t *in,
sh_src_t *out) {
*out = (sh_src_t){SH_SRC_NONE, NULL, NULL};
- int prog = sh_program(a);
+ int prog = sh_program(s, a);
if (prog < 0 || !a->v[prog]) {
return;
}
@@ -1728,39 +1760,74 @@ static void sh_deploy_command(sh_t *s, TSNode cmd, const sh_argv_t *a, const sh_
}
}
-/* A repository file: the path names a File node. */
-static int sh_t_repo_file(sh_t *s, const char *path, const sh_find_t *f, sh_targets_t *out) {
- (void)f;
- int64_t id = (int64_t)(intptr_t)cbm_ht_get(s->ix->file_ids, path);
- if (id <= 0) {
- return 0;
+/* The File node a repository path names; 0: none. */
+static int64_t sh_repo_file_id(const sh_t *s, const char *path) {
+ return (int64_t)(intptr_t)cbm_ht_get(s->ix->file_ids, path);
+}
+
+static const char *const SH_SHELLS[] = {"bash", "sh", "zsh", "ksh", "dash", NULL};
+
+/* The program is a script of the repository -- its path names a file of it,
+ * or a shell runs one (bash [-opts] FILE; -c TEXT is no file) -- resolved to
+ * exactly one file: that script is spawned. */
+static void sh_spawn_script(sh_t *s, TSNode cmd, const sh_argv_t *a, int p) {
+ const char *v = a->v[p];
+ const char *path = NULL;
+ if (sh_in(sh_basename(v), SH_SHELLS)) {
+ int j = p + SKIP_ONE;
+ while (j < a->n && a->v[j] && a->v[j][0] == '-' && !sh_eq(a->v[j], "-c")) {
+ j++;
+ }
+ path = j < a->n && !sh_eq(a->v[j], "-c") ? a->v[j] : NULL;
+ } else if (strchr(v, '/') && !strpbrk(v, "*?[")) {
+ path = v;
+ }
+ if (!path) {
+ return;
+ }
+ const char *c[PAIR_LEN] = {NULL, NULL};
+ int n = sh_candidates(s, path, c);
+ const char *hit = NULL;
+ int64_t id = 0;
+ int found = 0;
+ for (int i = 0; i < n; i++) {
+ int64_t f = sh_repo_file_id(s, c[i]);
+ if (f > 0) {
+ found++;
+ id = f;
+ hit = c[i];
+ }
+ }
+ if (found == SKIP_ONE) {
+ sh_hit_push(s, cmd,
+ (sh_hit_t){.via = s->exec ? "exec" : "shell",
+ .tgt = id,
+ .ext = sh_basename(hit),
+ .path = hit,
+ .spawn = true});
}
- sh_targets_push(out, id, path);
- return SKIP_ONE;
}
/* A program started by a path (./_build/x, ${DIR}/x, $(LOCALBIN)/x) SPAWNS
* the Process of its name -- the value's last segment, or, the value
* unknown, the word's literal last segment. A path naming a file of the
- * repository is a script of its own, not linked here. */
+ * repository is a script (sh_spawn_script), not a Process. */
static void sh_spawn(sh_t *s, TSNode cmd, const sh_argv_t *a) {
- int p = sh_program(a);
+ int p = sh_program(s, a);
if (p < 0) {
return;
}
const char *v = a->v[p];
const char *name = NULL;
if (v) {
+ sh_spawn_script(s, cmd, a, p);
if (!strchr(v, '/') || strpbrk(v, "*?[")) {
return;
}
const char *c[PAIR_LEN] = {NULL, NULL};
int n = sh_candidates(s, v, c);
for (int i = 0; i < n; i++) {
- sh_targets_t ts = {0};
- int hit = sh_t_repo_file(s, c[i], NULL, &ts);
- cbm_free(CBM_MEM_CLASS_OTHER, ts.items);
- if (hit) {
+ if (sh_repo_file_id(s, c[i]) > 0) {
return;
}
}
@@ -1786,15 +1853,7 @@ static void sh_spawn(sh_t *s, TSNode cmd, const sh_argv_t *a) {
if (!name || !name[0] || strcmp(name, ".") == 0 || strcmp(name, "..") == 0) {
return;
}
- sh_hits_t *h = s->hits;
- if (sh_grow((void **)&h->items, &h->cap, h->count, sizeof(*h->items))) {
- h->items[h->count++] =
- (sh_hit_t){.file = s->file,
- .line = (int)ts_node_start_point(cmd).row + SKIP_ONE + s->line_base,
- .via = "shell",
- .ext = cbm_arena_strdup(&h->arena, name),
- .spawn = true};
- }
+ sh_hit_push(s, cmd, (sh_hit_t){.via = s->exec ? "exec" : "shell", .ext = name, .spawn = true});
}
/* ── Statements ────────────────────────────────────────────────────── */
@@ -2633,8 +2692,9 @@ static void sh_makefile(sh_pass_t *p, const char *rel, const char *src, size_t l
uint32_t e = m.lines[r].e;
const char *raw = cbm_arena_strndup(&a, src + b, e - b);
const char *text = raw ? mk_expand(&m, raw) : NULL;
- if (!text || (!strchr(text, '/') && !sh_mentions_deploy_tool(text, strlen(text)))) {
- continue; /* neither a deploy nor a program started by a path */
+ if (!text || (!strchr(text, '/') && !strstr(text, "sh") &&
+ !sh_mentions_deploy_tool(text, strlen(text)))) {
+ continue; /* neither a deploy, a program started by a path, nor a shell */
}
size_t tl = strlen(text);
char *buf = cbm_arena_alloc(&a, tl + CBM_SZ_16);
@@ -2734,23 +2794,53 @@ static int64_t sh_external_chart(cbm_pipeline_ctx_t *ctx, const char *name, cons
"{\"source\":\"helm\",\"external\":true}");
}
-/* SPAWNS to the Process of `program`, as pass_spawns.c draws it. */
-static int sh_link_spawn(cbm_pipeline_ctx_t *ctx, int64_t src, const sh_hit_t *x) {
+/* A hit's CI context as JSON members (`,"ci":..,"job":..`); "" for none. */
+static void sh_ci_props(const sh_hit_t *x, char *out, size_t sz) {
+ out[0] = '\0';
+ if (!x->ci) {
+ return;
+ }
+ char job[CBM_SZ_512];
+ cbm_json_escape(job, (int)sizeof(job), x->job ? x->job : "");
+ if (x->job) {
+ snprintf(out, sz, ",\"ci\":\"%s\",\"job\":\"%s\"", x->ci, job);
+ } else {
+ snprintf(out, sz, ",\"ci\":\"%s\"", x->ci);
+ }
+}
+
+/* SPAWNS to the script `x->tgt` (its Module, else its File) or to the
+ * Process of `x->ext`, as pass_spawns.c draws it. */
+static int sh_link_spawn(cbm_pipeline_ctx_t *ctx, int64_t src, const sh_hit_t *x,
+ CBMHashTable *modules) {
char program[CBM_SZ_128];
cbm_json_escape(program, (int)sizeof(program), x->ext);
- char qn[CBM_SZ_256];
- snprintf(qn, sizeof(qn), "__process__%s", x->ext);
- char node_props[CBM_SZ_256];
- snprintf(node_props, sizeof(node_props), "{\"program\":\"%s\"}", program);
- int64_t process = cbm_gbuf_upsert_node(ctx->gbuf, "Process", x->ext, qn, "", 0, 0, node_props);
- if (process <= 0) {
+ char path[CBM_SZ_1K + CBM_SZ_16];
+ path[0] = '\0';
+ int64_t tgt = 0;
+ if (x->tgt > 0) {
+ int64_t mod = (int64_t)(intptr_t)cbm_ht_get(modules, x->path);
+ tgt = mod > 0 ? mod : x->tgt;
+ char esc[CBM_SZ_1K];
+ cbm_json_escape(esc, (int)sizeof(esc), x->path);
+ snprintf(path, sizeof(path), ",\"path\":\"%s\"", esc);
+ } else {
+ char qn[CBM_SZ_256];
+ snprintf(qn, sizeof(qn), "__process__%s", x->ext);
+ char node_props[CBM_SZ_256];
+ snprintf(node_props, sizeof(node_props), "{\"program\":\"%s\"}", program);
+ tgt = cbm_gbuf_upsert_node(ctx->gbuf, "Process", x->ext, qn, "", 0, 0, node_props);
+ }
+ if (tgt <= 0 || tgt == src) {
return 0;
}
- char props[CBM_SZ_512];
+ char ci[CBM_SZ_1K];
+ sh_ci_props(x, ci, sizeof(ci));
+ char props[CBM_SZ_4K];
snprintf(props, sizeof(props),
- "{\"api\":\"%s\",\"program\":\"%s\",\"line\":%d,\"confidence\":1.00}", x->via, program,
- x->line);
- cbm_gbuf_insert_edge(ctx->gbuf, src, process, "SPAWNS", props);
+ "{\"api\":\"%s\",\"program\":\"%s\"%s%s,\"line\":%d,\"confidence\":1.00}", x->via,
+ program, path, ci, x->line);
+ cbm_gbuf_insert_edge(ctx->gbuf, src, tgt, "SPAWNS", props);
return SKIP_ONE;
}
@@ -2767,7 +2857,11 @@ static int sh_link(cbm_pipeline_ctx_t *ctx, const sh_hits_t *h) {
return 0;
}
for (int i = 0; i < h->count; i++) {
- cbm_ht_set(files, h->items[i].file, (void *)h->items[i].file);
+ const sh_hit_t *x = &h->items[i];
+ cbm_ht_set(files, x->file, (void *)x->file);
+ if (x->spawn && x->tgt > 0) {
+ cbm_ht_set(files, x->path, (void *)x->path); /* a spawned script's Module */
+ }
}
sh_span_t *spans = NULL;
int n = 0;
@@ -2800,7 +2894,7 @@ static int sh_link(cbm_pipeline_ctx_t *ctx, const sh_hits_t *h) {
const sh_hit_t *x = &h->items[i];
int64_t src = sh_source(ctx, spans, n, modules, x->file, x->line);
if (x->spawn) {
- edges += src > 0 ? sh_link_spawn(ctx, src, x) : 0;
+ edges += src > 0 ? sh_link_spawn(ctx, src, x, modules) : 0;
continue;
}
int64_t tgt = x->tgt > 0 ? x->tgt : sh_external_chart(ctx, x->ext, x->file);
@@ -2808,10 +2902,13 @@ static int sh_link(cbm_pipeline_ctx_t *ctx, const sh_hits_t *h) {
continue;
}
char esc[CBM_SZ_1K];
- char props[CBM_SZ_2K];
+ char ci[CBM_SZ_1K];
+ char props[CBM_SZ_4K];
cbm_json_escape(esc, (int)sizeof(esc), x->path ? x->path : "");
- snprintf(props, sizeof(props), "{\"via\":\"%s\",\"%s\":\"%s\"%s,\"line\":%d}", x->via,
- x->chart ? "chart" : "path", esc, x->tgt > 0 ? "" : ",\"external\":true", x->line);
+ sh_ci_props(x, ci, sizeof(ci));
+ snprintf(props, sizeof(props), "{\"via\":\"%s\",\"%s\":\"%s\"%s%s,\"line\":%d}", x->via,
+ x->chart ? "chart" : "path", esc, x->tgt > 0 ? "" : ",\"external\":true", ci,
+ x->line);
cbm_gbuf_insert_edge(ctx->gbuf, src, tgt, "DEPLOYS", props);
edges++;
}
@@ -2821,15 +2918,93 @@ static int sh_link(cbm_pipeline_ctx_t *ctx, const sh_hits_t *h) {
return edges;
}
+/* ── CI configurations (pass_ci.c) ─────────────────────────────────── */
+
+typedef struct {
+ sh_pass_t *p;
+ const char *rel;
+ CBMArena *a;
+} sh_ci_t;
+
+static void sh_ci_var(sh_t *s, const char *name, const char *value) {
+ sh_val_t *v = value ? cbm_arena_alloc(s->a, sizeof(*v)) : NULL;
+ if (v) {
+ *v = (sh_val_t){value, NULL, 0};
+ }
+ sh_set(s, name, v ? v : &SH_UNKNOWN_VAL, false);
+}
+
+/* A command line CI runs: evaluated from its directory with its variables;
+ * $0 is unknown. */
+static void sh_ci_cmd(const cbm_ci_cmd_t *cmd, void *ud) {
+ sh_ci_t *x = ud;
+ sh_t s;
+ sh_state_init(&s, x->p, x->a, x->rel);
+ if (s.env) {
+ s.line_base = cmd->row;
+ s.ci = cmd->ci;
+ s.job = cmd->job;
+ s.exec = cmd->exec;
+ const char *wd = cmd->wd ? sh_join(x->a, "", cmd->wd) : NULL;
+ s.cwd = wd ? (sh_cwd_t){SH_AT, wd} : (sh_cwd_t){SH_LOST, NULL};
+ if (strcmp(cmd->ci, "github-actions") == 0) {
+ sh_ci_var(&s, "GITHUB_WORKSPACE", SH_ROOT);
+ if (cmd->action_dir) {
+ sh_ci_var(&s, "GITHUB_ACTION_PATH", sh_anchor(x->a, cmd->action_dir));
+ }
+ }
+ for (int i = 0; i < cmd->nenv; i++) {
+ sh_ci_var(&s, cmd->env[PAIR_LEN * i], cmd->env[PAIR_LEN * i + SKIP_ONE]);
+ }
+ sh_run(&s, x->p->parser, cmd->text, cmd->len);
+ }
+ sh_state_free(&s);
+}
+
+static void sh_ci_file(sh_pass_t *p, const char *rel, const char *src, size_t len) {
+ CBMArena a;
+ cbm_arena_init(&a);
+ sh_ci_t x = {.p = p, .rel = rel, .a = &a};
+ cbm_ci_commands(&a, rel, src, len, sh_ci_cmd, &x);
+ cbm_arena_destroy(&a);
+}
+
+/* Prow job configs: the files whose top-level keys (YAML Variables) include
+ * presubmits / postsubmits / periodics. */
+static CBMHashTable *sh_prow_files(cbm_pipeline_ctx_t *ctx) {
+ static const char *const keys[] = {"presubmits", "postsubmits", "periodics", NULL};
+ CBMHashTable *out = NULL;
+ for (int k = 0; keys[k]; k++) {
+ const cbm_gbuf_node_t **nodes = NULL;
+ int n = 0;
+ if (cbm_gbuf_find_by_name(ctx->gbuf, keys[k], &nodes, &n) != 0) {
+ continue;
+ }
+ for (int i = 0; i < n; i++) {
+ const char *fp = nodes[i]->file_path;
+ if (!fp || !sh_eq(nodes[i]->label, "Variable")) {
+ continue;
+ }
+ out = out ? out : cbm_ht_create(0);
+ if (out) {
+ cbm_ht_set(out, fp, (void *)fp);
+ }
+ }
+ }
+ return out;
+}
+
void cbm_pipeline_pass_shell(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
int file_count) {
sh_index_t ix = {0};
sh_hits_t hits = {0};
cbm_arena_init(&hits.arena);
sh_pass_t p = {.ix = &ix, .hits = &hits, .parser = NULL};
+ CBMHashTable *prow = sh_prow_files(ctx);
for (int i = 0; i < file_count; i++) {
CBMLanguage lang = files[i].language;
- if (lang != CBM_LANG_BASH && lang != CBM_LANG_MAKEFILE) {
+ bool ci = cbm_ci_path(files[i].rel_path) || (prow && cbm_ht_get(prow, files[i].rel_path));
+ if (lang != CBM_LANG_BASH && lang != CBM_LANG_MAKEFILE && !ci) {
continue;
}
size_t len = 0;
@@ -2843,7 +3018,9 @@ void cbm_pipeline_pass_shell(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *fil
p.parser = ts_parser_new();
}
if (p.ix_ok && p.parser) {
- if (lang == CBM_LANG_BASH) {
+ if (ci) {
+ sh_ci_file(&p, files[i].rel_path, src, len);
+ } else if (lang == CBM_LANG_BASH) {
sh_script(&p, files[i].rel_path, src, len);
} else {
sh_makefile(&p, files[i].rel_path, src, len);
@@ -2851,6 +3028,9 @@ void cbm_pipeline_pass_shell(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *fil
}
cbm_free(CBM_MEM_CLASS_OTHER, src);
}
+ if (prow) {
+ cbm_ht_free(prow);
+ }
int edges = hits.count > 0 ? sh_link(ctx, &hits) : 0;
if (p.parser) {
ts_parser_delete(p.parser);
diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h
index 311f92d2a..9ab34181c 100644
--- a/src/pipeline/pipeline_internal.h
+++ b/src/pipeline/pipeline_internal.h
@@ -1042,11 +1042,38 @@ int cbm_pipeline_pass_configlink(cbm_pipeline_ctx_t *ctx);
* Runs after USAGE resolution. */
int cbm_pipeline_pass_operator(cbm_pipeline_ctx_t *ctx);
-/* Shell scripts and Make recipes (pass_shell.c): the commands that put
- * manifests, kustomizations or charts on a cluster DEPLOYS them. Run at the
- * end of the K8s pass, over the same files. */
+/* Shell scripts, Make recipes and the commands CI runs (pass_shell.c): a
+ * command that puts manifests, kustomizations or charts on a cluster DEPLOYS
+ * them; one that starts a program by a path, or a script of the repository,
+ * SPAWNS it. Run at the end of the K8s pass, over the same files. */
void cbm_pipeline_pass_shell(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count);
+/* One command line a CI configuration runs (pass_ci.c). */
+typedef struct {
+ const char *ci; /* "github-actions" | "prow" */
+ const char *job; /* the job; NULL: none (a composite action's step) */
+ const char *text; /* shell text; CBM_SZ_16 zero bytes past len */
+ size_t len; /* bytes of text */
+ int row; /* 0-based source row of the text's first line */
+ const char *wd; /* start directory, repo-relative ("" = root); NULL: unknown */
+ const char *action_dir; /* a composite action's directory; NULL: none */
+ const char **env; /* name, value pairs (a value NULL: unknown) */
+ int nenv; /* pairs in env */
+ bool exec; /* an argv exec'd without a shell, quoted into text */
+} cbm_ci_cmd_t;
+
+typedef void (*cbm_ci_cmd_fn)(const cbm_ci_cmd_t *cmd, void *ud);
+
+/* `rel` is a CI configuration by its path: a GitHub workflow
+ * (.github/workflows/<name>.yml|yaml) or a composite action (action.yml|yaml). */
+bool cbm_ci_path(const char *rel);
+
+/* Hand each command line the CI configuration `rel` (YAML text src) runs to
+ * fn: GitHub Actions run: steps for a workflow or composite action, else
+ * Prow job containers. Strings live in arena `a`. */
+void cbm_ci_commands(CBMArena *a, const char *rel, const char *src, size_t len, cbm_ci_cmd_fn fn,
+ void *ud);
+
/* Terraform / OpenTofu modules (pass_terraform.c): a helm_release DEPLOYS its
* chart. Run at the end of the K8s pass, over the same files. */
void cbm_pipeline_pass_terraform(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c
index c6ac29041..83df57008 100644
--- a/tests/test_grammar_probe_f.c
+++ b/tests/test_grammar_probe_f.c
@@ -1776,6 +1776,50 @@ static char *gpf_spawns(const GpfFile *files, int nfiles) {
free(got_); \
} while (0)
+/* Every SPAWNS and DEPLOYS edge as "source -> target (TYPE api|via[
+ * ci[/job]])", sorted, one per line: top-level code as "@<file>"; a script
+ * as "script:<path>", a Process as "proc:<name>", a Chart as "chart:<name>". */
+static char *gpf_runs(const GpfFile *files, int nfiles) {
+ GpfProj lp;
+ cbm_store_t *store = gpf_index_files(&lp, files, nfiles);
+ char *out = NULL;
+ if (store) {
+ sqlite3_stmt *st = NULL;
+ const char *sql =
+ "SELECT coalesce(group_concat(x, char(10)), '') FROM (SELECT DISTINCT "
+ "CASE WHEN s.label IN ('Module','File') THEN '@' || s.file_path ELSE s.name END "
+ "|| ' -> ' || CASE WHEN t.label IN ('Module','File') THEN 'script:' || t.file_path "
+ "WHEN t.label='Process' THEN 'proc:' || t.name WHEN t.label='Chart' THEN "
+ "'chart:' || t.name ELSE t.label || ':' || t.file_path END "
+ "|| ' (' || e.type || ' ' || coalesce(json_extract(e.properties,'$.api'), "
+ "json_extract(e.properties,'$.via')) || coalesce(' ' || "
+ "json_extract(e.properties,'$.ci') || coalesce('/' || "
+ "json_extract(e.properties,'$.job'), ''), '') || ')' AS x "
+ "FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON t.id=e.target_id "
+ "WHERE e.project=?1 AND e.type IN ('SPAWNS','DEPLOYS') ORDER BY x)";
+ if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) {
+ sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC);
+ if (sqlite3_step(st) == SQLITE_ROW) {
+ out = strdup((const char *)sqlite3_column_text(st, 0));
+ }
+ }
+ sqlite3_finalize(st);
+ }
+ gpf_cleanup(&lp, store);
+ return out;
+}
+
+#define GPF_ASSERT_RUNS(files, n, want) \
+ do { \
+ char *got_ = gpf_runs(files, n); \
+ if (!got_ || strcmp(got_, want) != 0) { \
+ fprintf(stderr, "runs got:\n%s\nwant:\n%s\n", got_ ? got_ : "(null)", want); \
+ free(got_); \
+ FAIL("SPAWNS / DEPLOYS edge set"); \
+ } \
+ free(got_); \
+ } while (0)
+
#define GPF_CM(name) "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: " name "\n"
#define GPF_KUST "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n"
@@ -1974,6 +2018,105 @@ TEST(probe_make_programs_by_path_spawn) {
PASS();
}
+/* A script of the repository a command runs SPAWNS that script (its Module):
+ * by its path, through a shell (bash -e FILE), past `env A="$X/y"`; not
+ * itself. CI runs commands too: a GitHub workflow's run: steps from the
+ * workspace (`${{ github.workspace }}`, env literals, working-directory;
+ * an unknown `${{ }}` prefix leaves a Process; no pwsh step, no step of a
+ * Windows job), a composite action's from its directory
+ * (`${{ github.action_path }}`), a decorated Prow job's container command
+ * exec'd from the checkout (`bash -c TEXT` as shell; undecorated: the
+ * directory unknown, so a Process). Their edges carry the CI and the job. */
+TEST(probe_ci_and_scripts_run_scripts) {
+ static const GpfFile files[] = {
+ {"charts/web/Chart.yaml", "apiVersion: v2\nname: web\nversion: 0.1.0\n"},
+ {"hack/lib/helper.sh", "#!/bin/sh\necho helper\n"},
+ {"hack/lint.sh", "#!/bin/sh\necho lint\n"},
+ {"hack/ci/run-e2e.sh", "#!/bin/sh\necho e2e\n"},
+ {"hack/ci/lint.sh", "#!/bin/sh\necho lint\n"},
+ {"ops/apply.sh", "#!/bin/sh\necho apply\n"},
+ {"hack/test.sh", "#!/usr/bin/env bash\n"
+ "bash -e \"$(dirname \"$0\")/lib/helper.sh\"\n"
+ "env OUT=\"$TMPDIR/x.rpm\" ./hack/lint.sh\n"},
+ {"hack/e2e.sh", "#!/bin/sh\n"
+ "./hack/test.sh\n"
+ "[ -n \"$AGAIN\" ] || AGAIN=1 \"$0\"\n"},
+ {"Makefile", "test:\n"
+ "\t./hack/test.sh\n"
+ "\t/bin/bash hack/lint.sh\n"},
+ {".github/workflows/ci.yml",
+ "name: ci\n"
+ "on: [push]\n"
+ "env:\n"
+ " CHART_DIR: charts/web\n"
+ "jobs:\n"
+ " test:\n"
+ " runs-on: ubuntu-latest\n"
+ " steps:\n"
+ " - uses: actions/checkout@v4\n"
+ " - run: ./hack/test.sh\n"
+ " - name: multi\n"
+ " run: |\n"
+ " sh hack/lint.sh --fix\n"
+ " ${{ github.workspace }}/hack/e2e.sh\n"
+ " helm upgrade --install web \"$CHART_DIR\"\n"
+ " ./bin/tool --x\n"
+ " - working-directory: ops\n"
+ " run: ./apply.sh\n"
+ " - run: \"${{ matrix.dir }}/hack/other.sh\"\n"
+ " - shell: pwsh\n"
+ " run: ./hack/ci/lint.sh\n"
+ " win:\n"
+ " runs-on: windows-latest\n"
+ " steps:\n"
+ " - run: ./hack/ci/run-e2e.sh\n"},
+ {".github/actions/setup/setup.sh", "#!/bin/sh\necho setup\n"},
+ {".github/actions/setup/action.yml", "name: setup\n"
+ "runs:\n"
+ " using: composite\n"
+ " steps:\n"
+ " - shell: bash\n"
+ " run: ${{ github.action_path }}/setup.sh\n"},
+ {".prow/jobs.yaml", "presubmits:\n"
+ " - name: pre-e2e\n"
+ " decorate: true\n"
+ " spec:\n"
+ " containers:\n"
+ " - image: x\n"
+ " command:\n"
+ " - \"./hack/ci/run-e2e.sh\"\n"
+ " - name: pre-lint\n"
+ " decorate: true\n"
+ " spec:\n"
+ " containers:\n"
+ " - command: [\"bash\", \"-c\", \"./hack/ci/lint.sh\"]\n"
+ " - name: pre-raw\n"
+ " spec:\n"
+ " containers:\n"
+ " - command: [\"./hack/ci/run-e2e.sh\"]\n"}};
+ GPF_ASSERT_RUNS(
+ files, 13,
+ "@.github/actions/setup/action.yml -> script:.github/actions/setup/setup.sh "
+ "(SPAWNS shell github-actions)\n"
+ "@.github/workflows/ci.yml -> chart:web (DEPLOYS helm github-actions/test)\n"
+ "@.github/workflows/ci.yml -> proc:other.sh (SPAWNS shell github-actions/test)\n"
+ "@.github/workflows/ci.yml -> proc:tool (SPAWNS shell github-actions/test)\n"
+ "@.github/workflows/ci.yml -> script:hack/e2e.sh (SPAWNS shell github-actions/test)\n"
+ "@.github/workflows/ci.yml -> script:hack/lint.sh (SPAWNS shell github-actions/test)\n"
+ "@.github/workflows/ci.yml -> script:hack/test.sh (SPAWNS shell github-actions/test)\n"
+ "@.github/workflows/ci.yml -> script:ops/apply.sh (SPAWNS shell github-actions/test)\n"
+ "@.prow/jobs.yaml -> proc:run-e2e.sh (SPAWNS exec prow/pre-raw)\n"
+ "@.prow/jobs.yaml -> script:hack/ci/lint.sh (SPAWNS shell prow/pre-lint)\n"
+ "@.prow/jobs.yaml -> script:hack/ci/run-e2e.sh (SPAWNS exec prow/pre-e2e)\n"
+ "@hack/e2e.sh -> script:hack/test.sh (SPAWNS shell)\n"
+ "@hack/test.sh -> script:hack/lib/helper.sh (SPAWNS shell)\n"
+ "@hack/test.sh -> script:hack/lint.sh (SPAWNS shell)\n"
+ "test -> proc:bash (SPAWNS shell)\n"
+ "test -> script:hack/lint.sh (SPAWNS shell)\n"
+ "test -> script:hack/test.sh (SPAWNS shell)");
+ PASS();
+}
+
/* A Terraform helm_release DEPLOYS its chart: a chart dir of the module
* (./charts/x, ${path.module}/charts/x), else -- with a repository, or as
* oci://.../<name> -- the external chart of that name. A variable chart or a
@@ -3779,6 +3922,7 @@ SUITE(grammar_probe_f) {
RUN_TEST(probe_make_recipes_deploy);
RUN_TEST(probe_shell_programs_by_path_spawn);
RUN_TEST(probe_make_programs_by_path_spawn);
+ RUN_TEST(probe_ci_and_scripts_run_scripts);
RUN_TEST(probe_go_installer_deploys_charts);
RUN_TEST(probe_terraform_helm_release_deploys_its_chart);
RUN_TEST(probe_helm_templated_objects_are_resources);