Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

18.4 KB
Raw Blame
/*
 * pass_githistory.c — Analyze git log to find change coupling.
 *
 * Runs `git log --name-only --since=6 months ago` and computes
 * file pairs that change together frequently. Creates FILE_CHANGES_WITH
 * edges between File nodes with coupling_score properties.
 *
 * Skips commits with >20 files (refactoring/merge noise).
 * Requires minimum 3 co-changes for an edge.
 *
 * Depends on: pass_structure having created File nodes
 */
#include "foundation/constants.h"

enum { GH_RING = 4, GH_RING_MASK = 3, GH_INIT_CAP = 16, GH_MIN_COMMITS = 3, GH_MAX_FILES = 20 };

#define SLEN(s) (sizeof(s) - 1)
#include "pipeline/pipeline.h"
#include "pipeline/pipeline_internal.h"
#include "graph_buffer/graph_buffer.h"
#include "foundation/hash_table.h"
#include "foundation/log.h"
#include "foundation/platform.h"
#include "foundation/compat.h"
#include "foundation/compat_fs.h"
#include "foundation/str_util.h"

/* Minimum coupling score to create an edge */
#define MIN_COUPLING_SCORE 0.3

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

static const char *itoa_log(int val) {
    static CBM_TLS char bufs[GH_RING][CBM_SZ_32];
    static CBM_TLS int idx = 0;
    int i = idx;
    idx = (idx + SKIP_ONE) & GH_RING_MASK;
    snprintf(bufs[i], sizeof(bufs[i]), "%d", val);
    return bufs[i];
}

static bool ends_with(const char *s, size_t slen, const char *suffix) {
    size_t sflen = strlen(suffix);
    return slen >= sflen && strcmp(s + slen - sflen, suffix) == 0;
}

bool cbm_is_trackable_file(const char *path) {
    if (!path) {
        return false;
    }
    /* Skip directory prefixes */
#define LEN_NODE_MODULES_SLASH 13 /* strlen("node_modules/") */
    if (strncmp(path, ".git/", SLEN(".git/")) == 0 ||
        strncmp(path, "node_modules/", LEN_NODE_MODULES_SLASH) == 0 ||
        strncmp(path, "vendor/", SLEN("vendor/")) == 0 ||
        strncmp(path, "__pycache__/", SLEN("__pycache__/")) == 0 ||
        strncmp(path, ".cache/", SLEN(".cache/")) == 0) {
        return false;
    }
    /* Skip lock/generated file names */
    const char *base = strrchr(path, '/');
    base = base ? base + SKIP_ONE : path;
    if (strcmp(base, "package-lock.json") == 0 || strcmp(base, "yarn.lock") == 0 ||
        strcmp(base, "pnpm-lock.yaml") == 0 || strcmp(base, "Cargo.lock") == 0 ||
        strcmp(base, "poetry.lock") == 0 || strcmp(base, "composer.lock") == 0 ||
        strcmp(base, "Gemfile.lock") == 0 || strcmp(base, "Pipfile.lock") == 0) {
        return false;
    }
    /* Skip non-source file extensions */
    size_t len = strlen(path);
    if (ends_with(path, len, ".lock") || ends_with(path, len, ".sum") ||
        ends_with(path, len, ".min.js") || ends_with(path, len, ".min.css") ||
        ends_with(path, len, ".map") || ends_with(path, len, ".wasm") ||
        ends_with(path, len, ".png") || ends_with(path, len, ".jpg") ||
        ends_with(path, len, ".gif") || ends_with(path, len, ".ico") ||
        ends_with(path, len, ".svg")) {
        return false;
    }
    return true;
}

/* ── Commit parsing ───────────────────────────────────────────────── */

typedef struct {
    char **files;
    int count;
    int cap;
    long long timestamp; /* unix epoch of this commit; 0 when unknown */
} commit_t;

static void commit_add_file(commit_t *c, const char *file) {
    if (c->count >= c->cap) {
        c->cap = c->cap ? c->cap * PAIR_LEN : GH_INIT_CAP;
        c->files = safe_realloc(c->files, c->cap * sizeof(char *));
    }
    c->files[c->count++] = strdup(file);
}

static void commit_free(commit_t *c) {
    for (int i = 0; i < c->count; i++) {
        free(c->files[i]);
    }
    free(c->files);
}

/* ── git log parsing (popen "git log") ────────────────────────────── */

static int parse_git_log(const char *repo_path, commit_t **out, int *out_count) {
    *out = NULL;
    *out_count = 0;

    if (!cbm_validate_shell_path_arg(repo_path)) {
        return CBM_NOT_FOUND;
    }

    char cmd[CBM_SZ_1K];
#ifdef _WIN32
    /* cmd.exe does not recognize single quotes, and '/dev/null' is a POSIX path. */
    const char *null_dev = "NUL";
#else
    const char *null_dev = "/dev/null";
#endif
    /* git -C "<path>" works on both cmd.exe and POSIX shells. Double quotes are
     * safe here because cbm_validate_shell_arg (above) rejects ", $, `, \ and the
     * other shell metacharacters that would otherwise be active inside them. */
    snprintf(cmd, sizeof(cmd),
             "git -C \"%s\" log --name-only --pretty=format:COMMIT:%%H:%%ct "
             "--since=\"1 year ago\" --max-count=10000 2>%s",
             repo_path, null_dev);

    FILE *fp = cbm_popen_git(cmd);
    if (!fp) {
        return CBM_NOT_FOUND;
    }

    int cap = CBM_SZ_64;
    commit_t *commits = malloc(cap * sizeof(commit_t));
    int count = 0;
    commit_t current = {0};

    char line[CBM_SZ_1K];
    while (fgets(line, sizeof(line), fp)) {
        size_t len = strlen(line);
        while (len > 0 && (line[len - SKIP_ONE] == '\n' || line[len - SKIP_ONE] == '\r')) {
            line[--len] = '\0';
        }
        if (len == 0) {
            continue;
        }

        if (strncmp(line, "COMMIT:", SLEN("COMMIT:")) == 0) {
            if (current.count > 0) {
                if (count >= cap) {
                    cap *= PAIR_LEN;
                    commits = safe_realloc(commits, cap * sizeof(commit_t));
                }
                commits[count++] = current;
                memset(&current, 0, sizeof(current));
            }
            /* Parse the unix timestamp from "COMMIT:<hash>:<unix_epoch>".
             * Older callers / stripped-down git output without %ct land on 0. */
            const char *hash_end = strchr(line + SLEN("COMMIT:"), ':');
            if (hash_end) {
                current.timestamp = strtoll(hash_end + 1, NULL, 10);
            }
            continue;
        }

        if (cbm_is_trackable_file(line)) {
            commit_add_file(&current, line);
        }
    }
    if (current.count > 0) {
        if (count >= cap) {
            cap *= PAIR_LEN;
            commits = safe_realloc(commits, cap * sizeof(commit_t));
        }
        commits[count++] = current;
    } else {
        commit_free(&current);
    }

    cbm_pclose(fp);
    *out = commits;
    *out_count = count;
    return 0;
}

/* Callback to free hash table entries. */
static void free_counter(const char *key, void *val, void *ud) {
    (void)ud;
    safe_str_free(&key);
    free(val);
}

/* ── Standalone coupling computation (testable) ──────────────────── */

/* Context for collect_coupling_result callback. */
typedef struct {
    CBMHashTable *file_counts;
    CBMHashTable *pair_timestamps; /* pair_key → long long*: max commit ts */
    cbm_change_coupling_t *out;
    int out_count;
    int max_out;
} collect_coupling_ctx_t;

static void collect_coupling_cb(const char *pair_key, void *val, void *ud) {
    collect_coupling_ctx_t *cctx = ud;
    int co_count = *(int *)val;
    if (co_count < GH_MIN_COMMITS) {
        return;
    }
    if (cctx->out_count >= cctx->max_out) {
        return;
    }

    const char *sep = strchr(pair_key, '\x01');
    if (!sep) {
        return;
    }
    size_t la = sep - pair_key;
    const char *file_b = sep + SKIP_ONE;

    char file_a_buf[CBM_SZ_512];
    if (la >= sizeof(file_a_buf)) {
        return;
    }
    memcpy(file_a_buf, pair_key, la);
    file_a_buf[la] = '\0';

    int *count_a = cbm_ht_get(cctx->file_counts, file_a_buf);
    int *count_b = cbm_ht_get(cctx->file_counts, file_b);
    if (!count_a || !count_b) {
        return;
    }

    int min_total = *count_a < *count_b ? *count_a : *count_b;
    if (min_total == 0) {
        return;
    }

    double score = (double)co_count / (double)min_total;
    if (score < MIN_COUPLING_SCORE) {
        return;
    }

    cbm_change_coupling_t *cc = &cctx->out[cctx->out_count++];
    snprintf(cc->file_a, sizeof(cc->file_a), "%s", file_a_buf);
    snprintf(cc->file_b, sizeof(cc->file_b), "%s", file_b);
    cc->co_change_count = co_count;
    cc->coupling_score = score;
    long long *ts = cbm_ht_get(cctx->pair_timestamps, pair_key);
    cc->last_co_change = ts ? *ts : 0;
}

int cbm_compute_change_coupling(const cbm_commit_files_t *commits, int commit_count,
                                cbm_change_coupling_t *out, int max_out) {
    CBMHashTable *file_counts = cbm_ht_create(CBM_SZ_1K);
    CBMHashTable *pair_counts = cbm_ht_create(CBM_SZ_2K);
    /* Parallel table mapping pair_key → max commit timestamp seen for that
     * pair, so the resulting edge can carry last_co_change. The pair_counts
     * table consumes its key on insert; pair_timestamps gets its own copy. */
    CBMHashTable *pair_timestamps = cbm_ht_create(CBM_SZ_2K);

    for (int c = 0; c < commit_count; c++) {
        if (commits[c].count > GH_MAX_FILES) {
            continue;
        }

        for (int i = 0; i < commits[c].count; i++) {
            int *val = cbm_ht_get(file_counts, commits[c].files[i]);
            if (val) {
                (*val)++;
            } else {
                int *nv = malloc(sizeof(int));
                *nv = SKIP_ONE;
                cbm_ht_set(file_counts, strdup(commits[c].files[i]), nv);
            }
        }

        for (int i = 0; i < commits[c].count; i++) {
            for (int j = i + SKIP_ONE; j < commits[c].count; j++) {
                const char *a = commits[c].files[i];
                const char *b = commits[c].files[j];
                if (strcmp(a, b) > 0) {
                    const char *t = a;
                    a = b;
                    b = t;
                }
                size_t la = strlen(a);
                size_t lb = strlen(b);
                size_t pk_len = la + SKIP_ONE + lb + SKIP_ONE;
                char *pk = malloc(pk_len);
                memcpy(pk, a, la);
                pk[la] = '\x01';
                memcpy(pk + la + SKIP_ONE, b, lb + SKIP_ONE);

                int *val = cbm_ht_get(pair_counts, pk);
                if (val) {
                    (*val)++;
                    long long *ts = cbm_ht_get(pair_timestamps, pk);
                    if (ts && commits[c].timestamp > *ts) {
                        *ts = commits[c].timestamp;
                    }
                    free(pk);
                } else {
                    int *nv = malloc(sizeof(int));
                    *nv = SKIP_ONE;
                    /* pair_counts takes ownership of pk; pair_timestamps
                     * needs its own copy. */
                    char *pk2 = malloc(pk_len);
                    memcpy(pk2, pk, pk_len);
                    cbm_ht_set(pair_counts, pk, nv);
                    long long *nts = malloc(sizeof(long long));
                    *nts = commits[c].timestamp;
                    cbm_ht_set(pair_timestamps, pk2, nts);
                }
            }
        }
    }

    collect_coupling_ctx_t cctx = {
        .file_counts = file_counts,
        .pair_timestamps = pair_timestamps,
        .out = out,
        .out_count = 0,
        .max_out = max_out,
    };
    cbm_ht_foreach(pair_counts, collect_coupling_cb, &cctx);

    cbm_ht_foreach(pair_counts, free_counter, NULL);
    cbm_ht_free(pair_counts);
    cbm_ht_foreach(pair_timestamps, free_counter, NULL);
    cbm_ht_free(pair_timestamps);
    cbm_ht_foreach(file_counts, free_counter, NULL);
    cbm_ht_free(file_counts);

    return cctx.out_count;
}

/* ── Split pass: compute (I/O-bound) + apply (gbuf writes) ───────── */

/* Pre-computed coupling result buffer for fused post-pass parallelism. */
#define MAX_COUPLINGS 8192
#define MAX_FILE_TEMPORAL 16384

/* Compute change couplings without touching the graph buffer.
 * Can run on a separate thread while other passes use the gbuf. */
int cbm_pipeline_githistory_compute(const char *repo_path, cbm_githistory_result_t *result) {
    result->couplings = NULL;
    result->count = 0;
    result->commit_count = 0;
    result->file_temporal = NULL;
    result->file_temporal_count = 0;

    commit_t *commits = NULL;
    int commit_count = 0;
    int rc = parse_git_log(repo_path, &commits, &commit_count);
    if (rc != 0 || commit_count == 0) {
        free(commits);
        return 0;
    }

    result->commit_count = commit_count;

    /* Convert to testable format */
    cbm_commit_files_t *cf = calloc((size_t)commit_count, sizeof(cbm_commit_files_t));
    if (!cf) {
        for (int c = 0; c < commit_count; c++) {
            commit_free(&commits[c]);
        }
        free(commits);
        return 0;
    }
    for (int c = 0; c < commit_count; c++) {
        cf[c].files = commits[c].files;
        cf[c].count = commits[c].count;
        cf[c].timestamp = commits[c].timestamp;
    }

    cbm_change_coupling_t *couplings = malloc(MAX_COUPLINGS * sizeof(cbm_change_coupling_t));
    int coupling_count = cbm_compute_change_coupling(cf, commit_count, couplings, MAX_COUPLINGS);

    /* Per-file temporal aggregation: change_count + last_modified.
     * Single hash-table pass over the same commit set used for coupling so
     * we don't re-scan history. NULL on OOM is fine — the caller still
     * gets the couplings. */
    cbm_file_temporal_t *ft_arr =
        calloc(MAX_FILE_TEMPORAL, sizeof(cbm_file_temporal_t)); /* defined reads
        even under the analyzer's cross-iteration view of the index map */
    if (ft_arr) {
        int ft_count = 0;
        CBMHashTable *file_idx = cbm_ht_create(CBM_SZ_1K);
        for (int c = 0; c < commit_count; c++) {
            if (cf[c].count > GH_MAX_FILES) {
                continue;
            }
            for (int f = 0; f < cf[c].count; f++) {
                const char *fp = cf[c].files[f];
                int *idx = cbm_ht_get(file_idx, fp);
                if (idx) {
                    ft_arr[*idx].change_count++;
                    if (cf[c].timestamp > ft_arr[*idx].last_modified) {
                        ft_arr[*idx].last_modified = cf[c].timestamp;
                    }
                } else if (ft_count < MAX_FILE_TEMPORAL) {
                    /* Allocate the index cell and key BEFORE claiming a slot:
                     * the previous unchecked malloc dereferenced NULL on
                     * allocation failure, and a failed strdup would have
                     * leaked the cell. */
                    int *nidx = malloc(sizeof(int));
                    char *key = nidx ? strdup(fp) : NULL;
                    if (!nidx || !key) {
                        free(nidx);
                        continue;
                    }
                    int new_idx = ft_count++;
                    snprintf(ft_arr[new_idx].file_path, sizeof(ft_arr[new_idx].file_path), "%s",
                             fp);
                    ft_arr[new_idx].change_count = 1;
                    ft_arr[new_idx].last_modified = cf[c].timestamp;
                    *nidx = new_idx;
                    cbm_ht_set(file_idx, key, nidx);
                }
            }
        }
        cbm_ht_foreach(file_idx, free_counter, NULL);
        cbm_ht_free(file_idx);
        result->file_temporal = ft_arr;
        result->file_temporal_count = ft_count;
    }

    free(cf);
    for (int c = 0; c < commit_count; c++) {
        commit_free(&commits[c]);
    }
    free(commits);

    result->couplings = couplings;
    result->count = coupling_count;
    return 0;
}

/* Apply pre-computed couplings to the graph buffer (must be on main thread). */
int cbm_pipeline_githistory_apply(cbm_pipeline_ctx_t *ctx, const cbm_githistory_result_t *result) {
    int edge_count = 0;

    for (int i = 0; i < result->count; i++) {
        const cbm_change_coupling_t *cc = &result->couplings[i];

        char *qn_a = cbm_pipeline_fqn_compute(ctx->project_name, cc->file_a, "__file__");
        char *qn_b = cbm_pipeline_fqn_compute(ctx->project_name, cc->file_b, "__file__");

        const cbm_gbuf_node_t *node_a = cbm_gbuf_find_by_qn(ctx->gbuf, qn_a);
        const cbm_gbuf_node_t *node_b = cbm_gbuf_find_by_qn(ctx->gbuf, qn_b);

        free(qn_a);
        free(qn_b);

        if (!node_a || !node_b || node_a->id == node_b->id) {
            continue;
        }

        char props[CBM_SZ_128];
        snprintf(props, sizeof(props),
                 "{\"co_changes\":%d,\"coupling_score\":%.2f,\"last_co_change\":%lld}",
                 cc->co_change_count, cc->coupling_score, cc->last_co_change);

        cbm_gbuf_insert_edge(ctx->gbuf, node_a->id, node_b->id, "FILE_CHANGES_WITH", props);
        edge_count++;
    }

    /* Apply per-file temporal metadata to existing File nodes so callers
     * can query change_count / last_modified for hotspot analysis. The
     * extension is re-derived and JSON-escaped to keep the props blob
     * well-formed even for paths with quotes or backslashes. */
    for (int i = 0; i < result->file_temporal_count; i++) {
        const cbm_file_temporal_t *ft = &result->file_temporal[i];
        char *qn = cbm_pipeline_fqn_compute(ctx->project_name, ft->file_path, "__file__");
        const cbm_gbuf_node_t *node = cbm_gbuf_find_by_qn(ctx->gbuf, qn);
        free(qn);
        if (!node) {
            continue;
        }

        const char *base = strrchr(ft->file_path, '/');
        base = base ? base + SKIP_ONE : ft->file_path;
        const char *ext = strrchr(base, '.');
        char ext_escaped[CBM_SZ_64];
        cbm_json_escape(ext_escaped, (int)sizeof(ext_escaped), ext ? ext : "");

        char props[CBM_SZ_256];
        snprintf(props, sizeof(props),
                 "{\"extension\":\"%s\",\"last_modified\":%lld,\"change_count\":%d}", ext_escaped,
                 ft->last_modified, ft->change_count);

        cbm_gbuf_upsert_node(ctx->gbuf, node->label, node->name, node->qualified_name,
                             node->file_path, node->start_line, node->end_line, props);
    }

    return edge_count;
}

/* ── Main pass (original serial interface) ───────────────────────── */

int cbm_pipeline_pass_githistory(cbm_pipeline_ctx_t *ctx) {
    cbm_log_info("pass.start", "pass", "githistory");

    cbm_githistory_result_t result = {0};
    cbm_pipeline_githistory_compute(ctx->repo_path, &result);

    int edge_count = 0;
    if (result.count > 0 || result.file_temporal_count > 0) {
        edge_count = cbm_pipeline_githistory_apply(ctx, &result);
    }

    free(result.couplings);
    free(result.file_temporal);

    cbm_log_info("pass.done", "pass", "githistory", "commits", itoa_log(result.commit_count),
                 "edges", itoa_log(edge_count));
    return 0;
}