Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

13.7 KB
Raw Blame
/*
 * discover.h — File discovery, language detection, and gitignore matching.
 *
 * Provides:
 *   - Language detection from filename/extension (CBMLanguage registry)
 *   - .m file disambiguation (Objective-C vs Magma vs MATLAB)
 *   - Gitignore-style pattern parsing and matching
 *   - Recursive directory walk with hardcoded + gitignore filtering
 *
 * Depends on: foundation (platform.h for file ops), cbm.h (CBMLanguage enum)
 */
#ifndef CBM_DISCOVER_H
#define CBM_DISCOVER_H

#include <stdbool.h>
#include <stdint.h>

/* Use the existing CBMLanguage enum from extraction layer */
#include "cbm.h"
#include "foundation/index_policy.h"
#include "discover/userconfig.h"

/* ── Language detection ──────────────────────────────────────────── */

/* Detect language from a filename (basename only, not full path).
 * Checks special filenames first (Makefile, CMakeLists.txt, etc.),
 * then falls back to extension-based lookup.
 * Returns CBM_LANG_COUNT if unknown. */
CBMLanguage cbm_language_for_filename(const char *filename);

/* Detect language from a file extension (including the dot, e.g. ".go").
 * Returns CBM_LANG_COUNT if unknown. */
CBMLanguage cbm_language_for_extension(const char *ext);

/* Get the human-readable name for a language enum value.
 * Returns "Unknown" for CBM_LANG_COUNT or out-of-range values. */
const char *cbm_language_name(CBMLanguage lang);

/* Disambiguate .m files by reading first 4KB of content.
 * Returns CBM_LANG_OBJC, CBM_LANG_MAGMA, or CBM_LANG_MATLAB.
 * On read failure, defaults to CBM_LANG_MATLAB. */
CBMLanguage cbm_disambiguate_m(const char *path);

/* Disambiguate .cls files by reading first 4KB of content.
 * Returns CBM_LANG_OBJECTSCRIPT_UDL if a line starts with "Class <Uppercase>",
 * CBM_LANG_COUNT (unsupported) for a Visual Basic 6 class module (#721),
 * otherwise CBM_LANG_APEX. On read failure, defaults to CBM_LANG_APEX. */
CBMLanguage cbm_disambiguate_cls(const char *path);

/* Disambiguate .frm files by reading first 4KB of content (#721).
 * Returns CBM_LANG_COUNT (unsupported) for a Visual Basic 6 form, otherwise
 * CBM_LANG_FORM. On read failure, defaults to CBM_LANG_FORM. */
CBMLanguage cbm_disambiguate_frm(const char *path);

/* Disambiguate .res files by reading first 4KB of content (#2176).
 * Returns CBM_LANG_COUNT (not indexed) for binary content -- a Godot resource
 * or a Windows compiled resource file, recognised by a NUL byte -- otherwise
 * CBM_LANG_RESCRIPT. On read failure, defaults to CBM_LANG_RESCRIPT. */
CBMLanguage cbm_disambiguate_res(const char *path);

/* Disambiguate .d files by reading first 4KB of content.
 * Returns CBM_LANG_COUNT (not source) for a make-style dependency file written
 * by rustc/cargo, gcc/clang -MD or CMake ("target ...: prereq ..."), otherwise
 * CBM_LANG_DLANG. On read failure or any doubt, defaults to CBM_LANG_DLANG. */
CBMLanguage cbm_disambiguate_d(const char *path);

/* Disambiguate .inc files by reading first 4KB of content.
 * Returns CBM_LANG_CPP for explicit C-family code, or
 * CBM_LANG_OBJECTSCRIPT_ROUTINE for ObjectScript markers / a lone #define.
 * BitBake recipe syntax and unknown content retain CBM_LANG_BITBAKE.
 * On read failure, defaults to CBM_LANG_BITBAKE. */
CBMLanguage cbm_disambiguate_inc(const char *path);

/* Detect a supported script language from a file's shebang (#!...) first line.
 * Conservative fallback used only when filename/extension detection is unknown
 * (see detect_file_language); it never overrides extension or special-filename
 * matches. Opens the file read-only and reads only a bounded first line.
 * Recognizes the interpreter *basename* (not the parent path):
 *   python / python2 / python3 / dotted versions (python3.12) -> CBM_LANG_PYTHON
 *   sh / bash / dash / ksh / zsh                              -> CBM_LANG_BASH
 *   node / nodejs                                             -> CBM_LANG_JAVASCRIPT
 *   ruby -> RUBY, perl -> PERL, php -> PHP, lua -> LUA
 * Handles direct paths, "env <interp>", "env -S <interp> <args>", and CRLF.
 * Fails closed (returns CBM_LANG_COUNT) on read error, missing/malformed
 * shebang, an embedded NUL in the first line, or an unknown interpreter. */
CBMLanguage cbm_language_from_shebang(const char *path);

/* Disambiguate .cfc files by reading the head of the content.
 * Returns CBM_LANG_CFML if the component is tag-based (a "<cfcomponent" tag, or a
 * leading '<'), otherwise CBM_LANG_CFSCRIPT for script-dialect components.
 * On read failure, defaults to CBM_LANG_CFSCRIPT. */
CBMLanguage cbm_disambiguate_cfc(const char *path);

/* The language of a file from its base name and, where the name is not enough,
 * its first bytes: the one rule discovery and the pinned test-impact inventory
 * share. cbm_language_probe_bytes says how many leading bytes the name needs
 * (0: the name decides). cbm_language_classify takes those bytes raw, `more`
 * when the file has bytes beyond them, and `readable` false when they could
 * not be read (each probe then keeps the name's default, as a failed open
 * always did). The name lookup consults the installed user language config. */
size_t cbm_language_probe_bytes(const char *filename);
CBMLanguage cbm_language_classify(const char *filename, const unsigned char *head, size_t head_len,
                                  bool more, bool readable);

/* The same with the language config given explicitly (NULL: none) instead of
 * the installed process-wide one, for a caller that classifies a snapshot
 * under that snapshot's config while another index may have its own
 * installed. */
size_t cbm_language_probe_bytes_with(const cbm_userconfig_t *config, const char *filename);
CBMLanguage cbm_language_classify_with(const cbm_userconfig_t *config, const char *filename,
                                       const unsigned char *head, size_t head_len, bool more,
                                       bool readable);

/* The most leading bytes any name asks cbm_language_probe_bytes for. */
#define CBM_LANGUAGE_PROBE_MAX 16384

/* cbm_language_classify on the file at `path`, read raw. */
CBMLanguage cbm_language_for_file(const char *filename, const char *path);

/* ── Gitignore pattern matching ──────────────────────────────────── */

typedef struct cbm_gitignore cbm_gitignore_t;

/* Parse gitignore patterns from a file. Returns NULL on error (file not found, etc.).
 * Caller must call cbm_gitignore_free(). */
cbm_gitignore_t *cbm_gitignore_load(const char *path);

/* Parse gitignore patterns from a string (for testing).
 * Caller must call cbm_gitignore_free(). */
cbm_gitignore_t *cbm_gitignore_parse(const char *content);

/* Check if a relative path matches any gitignore pattern.
 * rel_path should use '/' separators. is_dir indicates if path is a directory. */
bool cbm_gitignore_matches(const cbm_gitignore_t *gi, const char *rel_path, bool is_dir);

/* Free a gitignore matcher. NULL-safe. */
void cbm_gitignore_free(cbm_gitignore_t *gi);

/* Append all patterns from src into dst. dst takes ownership of deep copies
 * of each src pattern; src is unchanged and must still be freed by the caller.
 * NULL-safe on either argument.
 * Returns true on success (or when there is nothing to merge). Returns false on
 * allocation failure, in which case dst is left exactly as it was (atomic) — no
 * partial merge — so a failed merge degrades to "as if src was absent". */
bool cbm_gitignore_merge(cbm_gitignore_t *dst, const cbm_gitignore_t *src);

/* ── Directory skip / suffix filters ─────────────────────────────── */

/* Index mode controls filtering aggressiveness.
 * IMPORTANT: these values MUST match pipeline.h exactly.  A previous
 * mismatch (this header had FAST=1, pipeline.h has FAST=2) caused
 * fast-mode filtering to silently no-op depending on include order —
 * the pipeline passed value 2, discover.c compared against 1, and no
 * files got filtered. */
#ifndef CBM_INDEX_MODE_T_DEFINED
#define CBM_INDEX_MODE_T_DEFINED
typedef enum {
    CBM_MODE_FULL = 0,     /* parse everything supported */
    CBM_MODE_MODERATE = 1, /* aggressive filtering + similarity/semantic edges */
    CBM_MODE_FAST = 2,     /* aggressive filtering + no similarity/semantic edges */
} cbm_index_mode_t;
#endif

/* Check if a directory name should always be skipped (e.g. .git, node_modules).
 * Only checks the basename, not the full path. */
bool cbm_should_skip_dir(const char *dirname, cbm_index_mode_t mode);

/* Check if a file has a suffix that should be skipped (e.g. .pyc, .png). */
bool cbm_has_ignored_suffix(const char *filename, cbm_index_mode_t mode);

/* Check if a specific filename should be skipped in fast mode (e.g. LICENSE, go.sum). */
bool cbm_should_skip_filename(const char *filename, cbm_index_mode_t mode);

/* Check if a path matches fast-mode substring patterns (e.g. .d.ts, .pb.go). */
bool cbm_matches_fast_pattern(const char *filename, cbm_index_mode_t mode);

/* ── File discovery ──────────────────────────────────────────────── */

typedef struct {
    char *path;           /* absolute path (heap-allocated) */
    char *rel_path;       /* relative to repo root (heap-allocated) */
    CBMLanguage language; /* detected language */
    int64_t size;         /* file size in bytes */
    int64_t mtime_ns;     /* modification time, ns (whole seconds on Windows) */
} cbm_file_info_t;

typedef struct {
    cbm_index_mode_t mode;                              /* discovery filtering mode */
    const char *ignore_file;                            /* .cbmignore path, or NULL */
    int64_t max_file_size;                              /* 0 = no per-file limit */
    const cbm_index_resource_policy_t *resource_policy; /* NULL = no resource limits */
    cbm_index_resource_violation_t *resource_violation; /* optional exact diagnostic */
} cbm_discover_opts_t;

typedef enum {
    CBM_DISCOVER_ERROR = -1,
    CBM_DISCOVER_OK = 0,
    CBM_DISCOVER_LIMIT_EXCEEDED = 1,
} cbm_discover_status_t;

/* Walk a repository directory tree and discover all source files.
 * Applies hardcoded filters, gitignore patterns, and language detection.
 * Returns 0 on success, -1 on error.
 * Caller must call cbm_discover_free() on the results. */
int cbm_discover(const char *repo_path, const cbm_discover_opts_t *opts, cbm_file_info_t **out,
                 int *count);

/* Apply the exact same full discovery/filter policy without retaining a file
 * array. Stops before counting more than max_files and performs no per-file
 * allocation. deadline_ms is an absolute cbm_now_ms() deadline; zero disables
 * it. Returns LIMIT_EXCEEDED when at least max_files + 1 indexable files exist,
 * ERROR on traversal/deadline/allocation failure, or OK with the exact count. */
cbm_discover_status_t cbm_discover_count_bounded(const char *repo_path,
                                                 const cbm_discover_opts_t *opts, int max_files,
                                                 uint64_t deadline_ms, int *count_out);

/* Like cbm_discover(), but also reports the directory subtrees that were
 * skipped during the walk (hardcoded ALWAYS_SKIP/FAST_SKIP dirs + gitignore
 * matches), so callers can surface which subtrees were dropped (#411).
 * On success, *excluded_out receives a heap-allocated array of strdup'd
 * relative directory paths and *excluded_count_out its length; the caller
 * owns it and must free via cbm_discover_free_excluded(). Pass NULL for
 * excluded_out (and/or excluded_count_out) to discard the list — the internal
 * accumulator is freed in that case (no leak).
 * Returns 0 on success, -1 on error. */
int cbm_discover_ex(const char *repo_path, const cbm_discover_opts_t *opts, cbm_file_info_t **out,
                    int *count, char ***excluded_out, int *excluded_count_out);

/* One deliberately-not-indexed file (#963): an individual file dropped by an
 * ignore mechanism during the walk (its parent directory was NOT excluded —
 * whole subtrees are reported separately as excluded dirs). BY DESIGN, not a
 * failure. */
typedef struct {
    char *rel_path; /* heap-allocated, relative to repo root */
    char *reason;   /* heap-allocated: "gitignore" | "cbmignore" |
                     * "skip-list" | "ignored-suffix" | "fast-pattern" |
                     * "size-cap" */
} cbm_ignored_file_t;

/* Stored per-file ignore entries are capped (the walk still counts ALL of
 * them in *ignored_total_out, so truncation is always explicit, never
 * silent). Whole excluded subtrees stay exhaustive via excluded_out. */
enum { CBM_DISCOVER_IGNORED_CAP = 2000 };

/* Like cbm_discover_ex(), but additionally reports the individual files that
 * ignore rules dropped (#963 "purposely not indexed"). *ignored_out receives
 * a heap array (caller frees via cbm_discover_free_ignored),
 * *ignored_count_out its stored length (<= CBM_DISCOVER_IGNORED_CAP), and
 * *ignored_total_out the TOTAL number of ignored files seen. Pass NULL to
 * skip the collection entirely. */
int cbm_discover_ex2(const char *repo_path, const cbm_discover_opts_t *opts, cbm_file_info_t **out,
                     int *count, char ***excluded_out, int *excluded_count_out,
                     cbm_ignored_file_t **ignored_out, int *ignored_count_out,
                     int *ignored_total_out);

/* Free an array of file info results. NULL-safe. */
void cbm_discover_free(cbm_file_info_t *files, int count);

/* Free the excluded-directory list returned by cbm_discover_ex(). NULL-safe. */
void cbm_discover_free_excluded(char **excluded, int count);

/* Free the ignored-file list returned by cbm_discover_ex2(). NULL-safe. */
void cbm_discover_free_ignored(cbm_ignored_file_t *ignored, int count);

#endif /* CBM_DISCOVER_H */