Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
/*
* discover.h — File discovery, language detection, and gitignore matching.
*
* Provides:
* - Language detection from filename/extension (CBMLanguage registry)
* - .m file disambiguation (Objective-C vs Magma vs MATLAB)
* - Gitignore-style pattern parsing and matching
* - Recursive directory walk with hardcoded + gitignore filtering
*
* Depends on: foundation (platform.h for file ops), cbm.h (CBMLanguage enum)
*/
#ifndef CBM_DISCOVER_H
#define CBM_DISCOVER_H
#include <stdbool.h>
#include <stdint.h>
/* Use the existing CBMLanguage enum from extraction layer */
#include "cbm.h"
#include "foundation/index_policy.h"
#include "discover/userconfig.h"
/* ── Language detection ──────────────────────────────────────────── */
/* Detect language from a filename (basename only, not full path).
* Checks special filenames first (Makefile, CMakeLists.txt, etc.),
* then falls back to extension-based lookup.
* Returns CBM_LANG_COUNT if unknown. */
CBMLanguage cbm_language_for_filename(const char *filename);
/* Detect language from a file extension (including the dot, e.g. ".go").
* Returns CBM_LANG_COUNT if unknown. */
CBMLanguage cbm_language_for_extension(const char *ext);
/* Get the human-readable name for a language enum value.
* Returns "Unknown" for CBM_LANG_COUNT or out-of-range values. */
const char *cbm_language_name(CBMLanguage lang);
/* Disambiguate .m files by reading first 4KB of content.
* Returns CBM_LANG_OBJC, CBM_LANG_MAGMA, or CBM_LANG_MATLAB.
* On read failure, defaults to CBM_LANG_MATLAB. */
CBMLanguage cbm_disambiguate_m(const char *path);
/* Disambiguate .cls files by reading first 4KB of content.
* Returns CBM_LANG_OBJECTSCRIPT_UDL if a line starts with "Class <Uppercase>",
* CBM_LANG_COUNT (unsupported) for a Visual Basic 6 class module (#721),
* otherwise CBM_LANG_APEX. On read failure, defaults to CBM_LANG_APEX. */
CBMLanguage cbm_disambiguate_cls(const char *path);
/* Disambiguate .frm files by reading first 4KB of content (#721).
* Returns CBM_LANG_COUNT (unsupported) for a Visual Basic 6 form, otherwise
* CBM_LANG_FORM. On read failure, defaults to CBM_LANG_FORM. */
CBMLanguage cbm_disambiguate_frm(const char *path);
/* Disambiguate .res files by reading first 4KB of content (#2176).
* Returns CBM_LANG_COUNT (not indexed) for binary content -- a Godot resource
* or a Windows compiled resource file, recognised by a NUL byte -- otherwise
* CBM_LANG_RESCRIPT. On read failure, defaults to CBM_LANG_RESCRIPT. */
CBMLanguage cbm_disambiguate_res(const char *path);
/* Disambiguate .d files by reading first 4KB of content.
* Returns CBM_LANG_COUNT (not source) for a make-style dependency file written
* by rustc/cargo, gcc/clang -MD or CMake ("target ...: prereq ..."), otherwise
* CBM_LANG_DLANG. On read failure or any doubt, defaults to CBM_LANG_DLANG. */
CBMLanguage cbm_disambiguate_d(const char *path);
/* Disambiguate .inc files by reading first 4KB of content.
* Returns CBM_LANG_CPP for explicit C-family code, or
* CBM_LANG_OBJECTSCRIPT_ROUTINE for ObjectScript markers / a lone #define.
* BitBake recipe syntax and unknown content retain CBM_LANG_BITBAKE.
* On read failure, defaults to CBM_LANG_BITBAKE. */
CBMLanguage cbm_disambiguate_inc(const char *path);
/* Detect a supported script language from a file's shebang (#!...) first line.
* Conservative fallback used only when filename/extension detection is unknown
* (see detect_file_language); it never overrides extension or special-filename
* matches. Opens the file read-only and reads only a bounded first line.
* Recognizes the interpreter *basename* (not the parent path):
* python / python2 / python3 / dotted versions (python3.12) -> CBM_LANG_PYTHON
* sh / bash / dash / ksh / zsh -> CBM_LANG_BASH
* node / nodejs -> CBM_LANG_JAVASCRIPT
* ruby -> RUBY, perl -> PERL, php -> PHP, lua -> LUA
* Handles direct paths, "env <interp>", "env -S <interp> <args>", and CRLF.
* Fails closed (returns CBM_LANG_COUNT) on read error, missing/malformed
* shebang, an embedded NUL in the first line, or an unknown interpreter. */
CBMLanguage cbm_language_from_shebang(const char *path);
/* Disambiguate .cfc files by reading the head of the content.
* Returns CBM_LANG_CFML if the component is tag-based (a "<cfcomponent" tag, or a
* leading '<'), otherwise CBM_LANG_CFSCRIPT for script-dialect components.
* On read failure, defaults to CBM_LANG_CFSCRIPT. */
CBMLanguage cbm_disambiguate_cfc(const char *path);
/* The language of a file from its base name and, where the name is not enough,
* its first bytes: the one rule discovery and the pinned test-impact inventory
* share. cbm_language_probe_bytes says how many leading bytes the name needs
* (0: the name decides). cbm_language_classify takes those bytes raw, `more`
* when the file has bytes beyond them, and `readable` false when they could
* not be read (each probe then keeps the name's default, as a failed open
* always did). The name lookup consults the installed user language config. */
size_t cbm_language_probe_bytes(const char *filename);
CBMLanguage cbm_language_classify(const char *filename, const unsigned char *head, size_t head_len,
bool more, bool readable);
/* The same with the language config given explicitly (NULL: none) instead of
* the installed process-wide one, for a caller that classifies a snapshot
* under that snapshot's config while another index may have its own
* installed. */
size_t cbm_language_probe_bytes_with(const cbm_userconfig_t *config, const char *filename);
CBMLanguage cbm_language_classify_with(const cbm_userconfig_t *config, const char *filename,
const unsigned char *head, size_t head_len, bool more,
bool readable);
/* The most leading bytes any name asks cbm_language_probe_bytes for. */
#define CBM_LANGUAGE_PROBE_MAX 16384
/* cbm_language_classify on the file at `path`, read raw. */
CBMLanguage cbm_language_for_file(const char *filename, const char *path);
/* ── Gitignore pattern matching ──────────────────────────────────── */
typedef struct cbm_gitignore cbm_gitignore_t;
/* Parse gitignore patterns from a file. Returns NULL on error (file not found, etc.).
* Caller must call cbm_gitignore_free(). */
cbm_gitignore_t *cbm_gitignore_load(const char *path);
/* Parse gitignore patterns from a string (for testing).
* Caller must call cbm_gitignore_free(). */
cbm_gitignore_t *cbm_gitignore_parse(const char *content);
/* Check if a relative path matches any gitignore pattern.
* rel_path should use '/' separators. is_dir indicates if path is a directory. */
bool cbm_gitignore_matches(const cbm_gitignore_t *gi, const char *rel_path, bool is_dir);
/* Free a gitignore matcher. NULL-safe. */
void cbm_gitignore_free(cbm_gitignore_t *gi);
/* Append all patterns from src into dst. dst takes ownership of deep copies
* of each src pattern; src is unchanged and must still be freed by the caller.
* NULL-safe on either argument.
* Returns true on success (or when there is nothing to merge). Returns false on
* allocation failure, in which case dst is left exactly as it was (atomic) — no
* partial merge — so a failed merge degrades to "as if src was absent". */
bool cbm_gitignore_merge(cbm_gitignore_t *dst, const cbm_gitignore_t *src);
/* ── Directory skip / suffix filters ─────────────────────────────── */
/* Index mode controls filtering aggressiveness.
* IMPORTANT: these values MUST match pipeline.h exactly. A previous
* mismatch (this header had FAST=1, pipeline.h has FAST=2) caused
* fast-mode filtering to silently no-op depending on include order —
* the pipeline passed value 2, discover.c compared against 1, and no
* files got filtered. */
#ifndef CBM_INDEX_MODE_T_DEFINED
#define CBM_INDEX_MODE_T_DEFINED
typedef enum {
CBM_MODE_FULL = 0, /* parse everything supported */
CBM_MODE_MODERATE = 1, /* aggressive filtering + similarity/semantic edges */
CBM_MODE_FAST = 2, /* aggressive filtering + no similarity/semantic edges */
} cbm_index_mode_t;
#endif
/* Check if a directory name should always be skipped (e.g. .git, node_modules).
* Only checks the basename, not the full path. */
bool cbm_should_skip_dir(const char *dirname, cbm_index_mode_t mode);
/* Check if a file has a suffix that should be skipped (e.g. .pyc, .png). */
bool cbm_has_ignored_suffix(const char *filename, cbm_index_mode_t mode);
/* Check if a specific filename should be skipped in fast mode (e.g. LICENSE, go.sum). */
bool cbm_should_skip_filename(const char *filename, cbm_index_mode_t mode);
/* Check if a path matches fast-mode substring patterns (e.g. .d.ts, .pb.go). */
bool cbm_matches_fast_pattern(const char *filename, cbm_index_mode_t mode);
/* ── File discovery ──────────────────────────────────────────────── */
typedef struct {
char *path; /* absolute path (heap-allocated) */
char *rel_path; /* relative to repo root (heap-allocated) */
CBMLanguage language; /* detected language */
int64_t size; /* file size in bytes */
int64_t mtime_ns; /* modification time, ns (whole seconds on Windows) */
} cbm_file_info_t;
typedef struct {
cbm_index_mode_t mode; /* discovery filtering mode */
const char *ignore_file; /* .cbmignore path, or NULL */
int64_t max_file_size; /* 0 = no per-file limit */
const cbm_index_resource_policy_t *resource_policy; /* NULL = no resource limits */
cbm_index_resource_violation_t *resource_violation; /* optional exact diagnostic */
} cbm_discover_opts_t;
typedef enum {
CBM_DISCOVER_ERROR = -1,
CBM_DISCOVER_OK = 0,
CBM_DISCOVER_LIMIT_EXCEEDED = 1,
} cbm_discover_status_t;
/* Walk a repository directory tree and discover all source files.
* Applies hardcoded filters, gitignore patterns, and language detection.
* Returns 0 on success, -1 on error.
* Caller must call cbm_discover_free() on the results. */
int cbm_discover(const char *repo_path, const cbm_discover_opts_t *opts, cbm_file_info_t **out,
int *count);
/* Apply the exact same full discovery/filter policy without retaining a file
* array. Stops before counting more than max_files and performs no per-file
* allocation. deadline_ms is an absolute cbm_now_ms() deadline; zero disables
* it. Returns LIMIT_EXCEEDED when at least max_files + 1 indexable files exist,
* ERROR on traversal/deadline/allocation failure, or OK with the exact count. */
cbm_discover_status_t cbm_discover_count_bounded(const char *repo_path,
const cbm_discover_opts_t *opts, int max_files,
uint64_t deadline_ms, int *count_out);
/* Like cbm_discover(), but also reports the directory subtrees that were
* skipped during the walk (hardcoded ALWAYS_SKIP/FAST_SKIP dirs + gitignore
* matches), so callers can surface which subtrees were dropped (#411).
* On success, *excluded_out receives a heap-allocated array of strdup'd
* relative directory paths and *excluded_count_out its length; the caller
* owns it and must free via cbm_discover_free_excluded(). Pass NULL for
* excluded_out (and/or excluded_count_out) to discard the list — the internal
* accumulator is freed in that case (no leak).
* Returns 0 on success, -1 on error. */
int cbm_discover_ex(const char *repo_path, const cbm_discover_opts_t *opts, cbm_file_info_t **out,
int *count, char ***excluded_out, int *excluded_count_out);
/* One deliberately-not-indexed file (#963): an individual file dropped by an
* ignore mechanism during the walk (its parent directory was NOT excluded —
* whole subtrees are reported separately as excluded dirs). BY DESIGN, not a
* failure. */
typedef struct {
char *rel_path; /* heap-allocated, relative to repo root */
char *reason; /* heap-allocated: "gitignore" | "cbmignore" |
* "skip-list" | "ignored-suffix" | "fast-pattern" |
* "size-cap" */
} cbm_ignored_file_t;
/* Stored per-file ignore entries are capped (the walk still counts ALL of
* them in *ignored_total_out, so truncation is always explicit, never
* silent). Whole excluded subtrees stay exhaustive via excluded_out. */
enum { CBM_DISCOVER_IGNORED_CAP = 2000 };
/* Like cbm_discover_ex(), but additionally reports the individual files that
* ignore rules dropped (#963 "purposely not indexed"). *ignored_out receives
* a heap array (caller frees via cbm_discover_free_ignored),
* *ignored_count_out its stored length (<= CBM_DISCOVER_IGNORED_CAP), and
* *ignored_total_out the TOTAL number of ignored files seen. Pass NULL to
* skip the collection entirely. */
int cbm_discover_ex2(const char *repo_path, const cbm_discover_opts_t *opts, cbm_file_info_t **out,
int *count, char ***excluded_out, int *excluded_count_out,
cbm_ignored_file_t **ignored_out, int *ignored_count_out,
int *ignored_total_out);
/* Free an array of file info results. NULL-safe. */
void cbm_discover_free(cbm_file_info_t *files, int count);
/* Free the excluded-directory list returned by cbm_discover_ex(). NULL-safe. */
void cbm_discover_free_excluded(char **excluded, int count);
/* Free the ignored-file list returned by cbm_discover_ex2(). NULL-safe. */
void cbm_discover_free_ignored(cbm_ignored_file_t *ignored, int count);
#endif /* CBM_DISCOVER_H */