Skip to content

"CLI-Anything: Making ALL Software Agent-Native" -- CLI-Hub: https://clianything.cc/

Python

Add automatic PR labeling

yuhao288dedd2a4ab
diff --git a/.github/labeler.yml b/.github/labeler.yml
new file mode 100644
index 000000000..0cbe6f0ed
--- /dev/null
+++ b/.github/labeler.yml
@@ -0,0 +1,25 @@
+cli-anything-hub:
+ - changed-files:
+ - any-glob-to-any-file:
+ - "cli-hub/**"
+ - "docs/hub/**"
+ - "registry.json"
+ - "public_registry.json"
+ - "matrix_registry.json"
+
+cli-anything-skill:
+ - changed-files:
+ - any-glob-to-any-file:
+ - "cli-anything-plugin/**"
+ - "codex-skill/**"
+ - "cli-hub-meta-skill/**"
+ - "openclaw-skill/**"
+ - "skills/**"
+ - "**/SKILL.md"
+
+github-actions:
+ - changed-files:
+ - any-glob-to-any-file:
+ - ".github/workflows/**"
+ - ".github/labeler.yml"
+ - ".github/scripts/pr-labeler.js"
diff --git a/.github/scripts/pr-labeler.js b/.github/scripts/pr-labeler.js
new file mode 100644
index 000000000..25aa62950
--- /dev/null
+++ b/.github/scripts/pr-labeler.js
@@ -0,0 +1,173 @@
+const LABELS = {
+ "new-cli": {
+ color: "0E8A16",
+ description: "Adds a new CLI or generated harness",
+ },
+ "existing-cli-fix": {
+ color: "1D76DB",
+ description: "Fixes or improves an existing CLI harness",
+ },
+ "cli-anything-skill": {
+ color: "5319E7",
+ description: "Changes CLI-Anything plugin or skill files",
+ },
+ "cli-anything-hub": {
+ color: "FBCA04",
+ description: "Changes CLI-Hub, registries, or hub docs",
+ },
+ "documentation": {
+ color: "0075CA",
+ description: "Documentation issue or improvement",
+ },
+ "github-actions": {
+ color: "6F42C1",
+ description: "Changes GitHub Actions or automation",
+ },
+};
+
+const SCRIPT_MANAGED_LABELS = ["new-cli", "existing-cli-fix", "documentation"];
+const REGISTRY_FILES = new Set([
+ "registry.json",
+ "public_registry.json",
+ "matrix_registry.json",
+]);
+
+function isHarnessFile(path) {
+ return /^[^/]+\/agent-harness\//.test(path);
+}
+
+function isNewHarnessManifest(file) {
+ return (
+ file.status === "added" &&
+ /^[^/]+\/agent-harness\/(setup\.py|pyproject\.toml)$/.test(file.filename)
+ );
+}
+
+function isDocumentationFile(path) {
+ if (/(^|\/)SKILL\.md$/i.test(path)) {
+ return false;
+ }
+
+ return (
+ /^README(?:_[A-Z]+)?\.md$/.test(path) ||
+ /^(CONTRIBUTING|SECURITY)\.md$/.test(path) ||
+ /^docs\//.test(path) ||
+ /^[^/]+\.md$/i.test(path)
+ );
+}
+
+function titleLooksLikeRegistryCli(title) {
+ return /\b(add|introduce|new)\b/i.test(title) && /\b(cli|harness|registry)\b/i.test(title);
+}
+
+function computeScriptLabels(files, title) {
+ const paths = files.map((file) => file.filename);
+ const labelsToApply = new Set();
+
+ const hasHarnessChange = paths.some(isHarnessFile);
+ const hasNewHarness = files.some(isNewHarnessManifest);
+ const registryOnly = paths.length > 0 && paths.every((path) => REGISTRY_FILES.has(path));
+ const registryNewCli = registryOnly && titleLooksLikeRegistryCli(title || "");
+ const documentationOnly = paths.length > 0 && paths.every(isDocumentationFile);
+
+ if (hasNewHarness || registryNewCli) {
+ labelsToApply.add("new-cli");
+ } else if (hasHarnessChange) {
+ labelsToApply.add("existing-cli-fix");
+ }
+
+ if (documentationOnly) {
+ labelsToApply.add("documentation");
+ }
+
+ return labelsToApply;
+}
+
+async function ensureLabels(github, owner, repo, core) {
+ const existing = await github.paginate(github.rest.issues.listLabelsForRepo, {
+ owner,
+ repo,
+ per_page: 100,
+ });
+ const existingNames = new Set(existing.map((label) => label.name));
+
+ for (const [name, definition] of Object.entries(LABELS)) {
+ if (existingNames.has(name)) {
+ continue;
+ }
+
+ core.info(`Creating missing label: ${name}`);
+ await github.rest.issues.createLabel({
+ owner,
+ repo,
+ name,
+ color: definition.color,
+ description: definition.description,
+ });
+ }
+}
+
+async function syncScriptLabels(github, owner, repo, pullNumber, currentLabels, labelsToApply, core) {
+ for (const label of labelsToApply) {
+ if (currentLabels.has(label)) {
+ continue;
+ }
+
+ core.info(`Adding label: ${label}`);
+ await github.rest.issues.addLabels({
+ owner,
+ repo,
+ issue_number: pullNumber,
+ labels: [label],
+ });
+ }
+
+ for (const label of SCRIPT_MANAGED_LABELS) {
+ if (!currentLabels.has(label) || labelsToApply.has(label)) {
+ continue;
+ }
+
+ core.info(`Removing label: ${label}`);
+ try {
+ await github.rest.issues.removeLabel({
+ owner,
+ repo,
+ issue_number: pullNumber,
+ name: label,
+ });
+ } catch (error) {
+ if (error.status !== 404) {
+ throw error;
+ }
+ }
+ }
+}
+
+module.exports = async ({ github, context, core }) => {
+ const pullRequest = context.payload.pull_request;
+ if (!pullRequest) {
+ core.info("No pull_request payload found; skipping PR labeling.");
+ return;
+ }
+
+ const { owner, repo } = context.repo;
+ const pullNumber = pullRequest.number;
+
+ await ensureLabels(github, owner, repo, core);
+
+ const files = await github.paginate(github.rest.pulls.listFiles, {
+ owner,
+ repo,
+ pull_number: pullNumber,
+ per_page: 100,
+ });
+
+ const labelsToApply = computeScriptLabels(files, pullRequest.title);
+
+ const currentLabels = new Set((pullRequest.labels || []).map((label) => label.name));
+ await syncScriptLabels(github, owner, repo, pullNumber, currentLabels, labelsToApply, core);
+};
+
+module.exports.computeScriptLabels = computeScriptLabels;
+module.exports.LABELS = LABELS;
+module.exports.SCRIPT_MANAGED_LABELS = SCRIPT_MANAGED_LABELS;
diff --git a/.github/scripts/tests/pr-labeler-fixtures.json b/.github/scripts/tests/pr-labeler-fixtures.json
new file mode 100644
index 000000000..475624694
--- /dev/null
+++ b/.github/scripts/tests/pr-labeler-fixtures.json
@@ -0,0 +1,270 @@
+[
+ {
+ "number": 262,
+ "title": "Add MseeP.ai badge",
+ "files": [
+ {"filename": "README.md", "status": "modified"}
+ ],
+ "expected": ["documentation"]
+ },
+ {
+ "number": 260,
+ "title": "feat: add hacker-feeds-cli to registry",
+ "files": [
+ {"filename": "registry.json", "status": "modified"}
+ ],
+ "expected": ["cli-anything-hub", "new-cli"]
+ },
+ {
+ "number": 259,
+ "title": "feat: add ve-twini to registry",
+ "files": [
+ {"filename": "registry.json", "status": "modified"}
+ ],
+ "expected": ["cli-anything-hub", "new-cli"]
+ },
+ {
+ "number": 258,
+ "title": "feat: add sliver to registry",
+ "files": [
+ {"filename": "registry.json", "status": "modified"}
+ ],
+ "expected": ["cli-anything-hub", "new-cli"]
+ },
+ {
+ "number": 256,
+ "title": "feat(Calibre): add a CLI-Anything harness for Calibre Desktop",
+ "files": [
+ {"filename": ".gitignore", "status": "modified"},
+ {"filename": "README.md", "status": "modified"},
+ {"filename": "calibre/agent-harness/CALIBRE.md", "status": "added"},
+ {"filename": "calibre/agent-harness/cli_anything/calibre/calibre_cli.py", "status": "added"},
+ {"filename": "calibre/agent-harness/cli_anything/calibre/core/session.py", "status": "added"},
+ {"filename": "calibre/agent-harness/cli_anything/calibre/skills/SKILL.md", "status": "added"},
+ {"filename": "calibre/agent-harness/setup.py", "status": "added"},
+ {"filename": "cli-anything-plugin/scripts/setup-cli-anything.sh", "status": "modified"},
+ {"filename": "codex-skill/scripts/install.sh", "status": "modified"},
+ {"filename": "registry.json", "status": "modified"},
+ {"filename": "skills/cli-anything-calibre/SKILL.md", "status": "added"}
+ ],
+ "expected": ["cli-anything-hub", "cli-anything-skill", "new-cli"]
+ },
+ {
+ "number": 254,
+ "title": "feat: add quietshrink harness - Apple Silicon screen recording compressor",
+ "files": [
+ {"filename": ".gitignore", "status": "modified"},
+ {"filename": "quietshrink/agent-harness/QUIETSHRINK.md", "status": "added"},
+ {"filename": "quietshrink/agent-harness/cli_anything/quietshrink/quietshrink_cli.py", "status": "added"},
+ {"filename": "quietshrink/agent-harness/cli_anything/quietshrink/skills/SKILL.md", "status": "added"},
+ {"filename": "quietshrink/agent-harness/setup.py", "status": "added"},
+ {"filename": "registry.json", "status": "modified"},
+ {"filename": "skills/cli-anything-quietshrink/SKILL.md", "status": "added"}
+ ],
+ "expected": ["cli-anything-hub", "cli-anything-skill", "new-cli"]
+ },
+ {
+ "number": 252,
+ "title": "Add cli-anything-rekordbox: Pioneer Rekordbox 6/7 harness",
+ "files": [
+ {"filename": ".gitignore", "status": "modified"},
+ {"filename": "ableton/agent-harness/setup.py", "status": "added"},
+ {"filename": "ableton/agent-harness/cli_anything/ableton/ableton_cli.py", "status": "added"},
+ {"filename": "rekordbox/agent-harness/setup.py", "status": "added"},
+ {"filename": "rekordbox/agent-harness/cli_anything/rekordbox/rekordbox_cli.py", "status": "added"},
+ {"filename": "rekordbox/agent-harness/cli_anything/rekordbox/skills/SKILL.md", "status": "added"},
+ {"filename": "serum/agent-harness/setup.py", "status": "added"},
+ {"filename": "vital/agent-harness/setup.py", "status": "added"},
+ {"filename": "skills/cli-anything-rekordbox/SKILL.md", "status": "added"}
+ ],
+ "expected": ["cli-anything-skill", "new-cli"]
+ },
+ {
+ "number": 251,
+ "title": "feat: add s&box CLI harness",
+ "files": [
+ {"filename": ".gitignore", "status": "modified"},
+ {"filename": "README.md", "status": "modified"},
+ {"filename": "registry.json", "status": "modified"},
+ {"filename": "sbox/agent-harness/SBOX.md", "status": "added"},
+ {"filename": "sbox/agent-harness/cli_anything/sbox/sbox_cli.py", "status": "added"},
+ {"filename": "sbox/agent-harness/cli_anything/sbox/skills/SKILL.md", "status": "added"},
+ {"filename": "sbox/agent-harness/setup.py", "status": "added"},
+ {"filename": "skills/cli-anything-sbox/SKILL.md", "status": "added"}
+ ],
+ "expected": ["cli-anything-hub", "cli-anything-skill", "new-cli"]
+ },
+ {
+ "number": 245,
+ "title": "feat: mature lldb agent harness",
+ "files": [
+ {"filename": "lldb/agent-harness/LLDB.md", "status": "modified"},
+ {"filename": "lldb/agent-harness/cli_anything/lldb/lldb_cli.py", "status": "modified"},
+ {"filename": "lldb/agent-harness/cli_anything/lldb/skills/SKILL.md", "status": "modified"},
+ {"filename": "lldb/agent-harness/setup.py", "status": "modified"},
+ {"filename": "registry.json", "status": "modified"},
+ {"filename": "skills/cli-anything-lldb/SKILL.md", "status": "modified"}
+ ],
+ "expected": ["cli-anything-hub", "cli-anything-skill", "existing-cli-fix"]
+ },
+ {
+ "number": 244,
+ "title": "feat: improve Unreal Insights live analysis",
+ "files": [
+ {"filename": "skills/cli-anything-unrealinsights/SKILL.md", "status": "modified"},
+ {"filename": "unrealinsights/agent-harness/UNREALINSIGHTS.md", "status": "modified"},
+ {"filename": "unrealinsights/agent-harness/cli_anything/unrealinsights/core/analyze.py", "status": "added"},
+ {"filename": "unrealinsights/agent-harness/cli_anything/unrealinsights/skills/SKILL.md", "status": "modified"},
+ {"filename": "unrealinsights/agent-harness/cli_anything/unrealinsights/unrealinsights_cli.py", "status": "modified"}
+ ],
+ "expected": ["cli-anything-skill", "existing-cli-fix"]
+ },
+ {
+ "number": 241,
+ "title": "Feat/add firefly iii cli",
+ "files": [
+ {"filename": ".gitignore", "status": "modified"},
+ {"filename": "firefly-iii/agent-harness/README.md", "status": "added"},
+ {"filename": "firefly-iii/agent-harness/cli_anything/firefly_iii/firefly_iii_cli.py", "status": "added"},
+ {"filename": "firefly-iii/agent-harness/cli_anything/firefly_iii/skills/SKILL.md", "status": "added"},
+ {"filename": "firefly-iii/agent-harness/setup.py", "status": "added"},
+ {"filename": "firefly-iii/agent-harness/skills/cli-anything-firefly-iii/SKILL.md", "status": "added"}
+ ],
+ "expected": ["cli-anything-skill", "new-cli"]
+ },
+ {
+ "number": 240,
+ "title": "fix(blender): align docs and render execute with real behavior",
+ "files": [
+ {"filename": "README.md", "status": "modified"},
+ {"filename": "README_CN.md", "status": "modified"},
+ {"filename": "blender/agent-harness/cli_anything/blender/README.md", "status": "modified"},
+ {"filename": "blender/agent-harness/cli_anything/blender/blender_cli.py", "status": "modified"},
+ {"filename": "blender/agent-harness/cli_anything/blender/core/render.py", "status": "modified"},
+ {"filename": "blender/agent-harness/cli_anything/blender/utils/blender_backend.py", "status": "modified"}
+ ],
+ "expected": ["existing-cli-fix"]
+ },
+ {
+ "number": 238,
+ "title": "feat: add NSLogger CLI harness",
+ "files": [
+ {"filename": ".gitignore", "status": "modified"},
+ {"filename": "README.md", "status": "modified"},
+ {"filename": "nslogger/agent-harness/NSLOGGER.md", "status": "added"},
+ {"filename": "nslogger/agent-harness/cli_anything/nslogger/nslogger_cli.py", "status": "added"},
+ {"filename": "nslogger/agent-harness/cli_anything/nslogger/skills/SKILL.md", "status": "added"},
+ {"filename": "nslogger/agent-harness/setup.py", "status": "added"},
+ {"filename": "registry.json", "status": "modified"},
+ {"filename": "skills/cli-anything-nslogger/SKILL.md", "status": "added"}
+ ],
+ "expected": ["cli-anything-hub", "cli-anything-skill", "new-cli"]
+ },
+ {
+ "number": 237,
+ "title": "refactor(openclaw-skill->macrocli): rename, complete backends, add recorder and visual anchor",
+ "files": [
+ {"filename": "macrocli/SKILL.md", "status": "added"},
+ {"filename": "macrocli/agent-harness/MACROCLI.md", "status": "added"},
+ {"filename": "macrocli/agent-harness/cli_anything/macrocli/macrocli_cli.py", "status": "added"},
+ {"filename": "macrocli/agent-harness/cli_anything/macrocli/skills/SKILL.md", "status": "added"},
+ {"filename": "macrocli/agent-harness/setup.py", "status": "added"},
+ {"filename": "openclaw-skill/agent-harness/cli_anything/openclaw/openclaw_cli.py", "status": "modified"},
+ {"filename": "registry.json", "status": "modified"},
+ {"filename": "skills/cli-anything-macrocli/SKILL.md", "status": "added"}
+ ],
+ "expected": ["cli-anything-hub", "cli-anything-skill", "new-cli"]
+ },
+ {
+ "number": 233,
+ "title": "fix(browser): add MCP timeout guard to prevent fs hangs",
+ "files": [
+ {"filename": "browser/agent-harness/cli_anything/browser/browser_cli.py", "status": "modified"},
+ {"filename": "browser/agent-harness/cli_anything/browser/skills/SKILL.md", "status": "modified"},
+ {"filename": "browser/agent-harness/cli_anything/browser/utils/domshell_backend.py", "status": "modified"},
+ {"filename": "browser/agent-harness/setup.py", "status": "modified"},
+ {"filename": "registry.json", "status": "modified"}
+ ],
+ "expected": ["cli-anything-hub", "cli-anything-skill", "existing-cli-fix"]
+ },
+ {
+ "number": 220,
+ "title": "fix(browser): harden bridge and sync REPL template",
+ "files": [
+ {"filename": ".github/workflows/pr-ci.yml", "status": "added"},
+ {"filename": "browser/agent-harness/cli_anything/browser/browser_cli.py", "status": "modified"},
+ {"filename": "browser/agent-harness/cli_anything/browser/utils/repl_skin.py", "status": "modified"},
+ {"filename": "cli-anything-plugin/repl_skin.py", "status": "modified"},
+ {"filename": "gimp/agent-harness/cli_anything/gimp/utils/repl_skin.py", "status": "modified"}
+ ],
+ "expected": ["cli-anything-skill", "existing-cli-fix", "github-actions"]
+ },
+ {
+ "number": 218,
+ "title": "fix: Executing cli-anything-browser --json fs ls yields no return result",
+ "files": [
+ {"filename": "browser/agent-harness/cli_anything/browser/tests/test_core.py", "status": "modified"},
+ {"filename": "browser/agent-harness/cli_anything/browser/utils/domshell_backend.py", "status": "modified"}
+ ],
+ "expected": ["existing-cli-fix"]
+ },
+ {
+ "number": 202,
+ "title": "security: add user confirmation guard to meta-skill installation flow",
+ "files": [
+ {"filename": "cli-hub-meta-skill/SKILL.md", "status": "modified"},
+ {"filename": "docs/hub/SKILL.md", "status": "modified"}
+ ],
+ "expected": ["cli-anything-hub", "cli-anything-skill"]
+ },
+ {
+ "number": 189,
+ "title": "feat: add MiniMax CLI harness (chat + TTS)",
+ "files": [
+ {"filename": ".gitignore", "status": "modified"},
+ {"filename": "minimax/agent-harness/cli_anything/minimax/minimax_cli.py", "status": "added"},
+ {"filename": "minimax/agent-harness/cli_anything/minimax/skills/SKILL.md", "status": "added"},
+ {"filename": "minimax/agent-harness/setup.py", "status": "added"},
+ {"filename": "registry.json", "status": "modified"}
+ ],
+ "expected": ["cli-anything-hub", "cli-anything-skill", "new-cli"]
+ },
+ {
+ "number": 187,
+ "title": "Refactor AnyGen CLI for better state management",
+ "files": [
+ {"filename": "anygen/agent-harness/cli_anything/anygen/anygen_cli.py", "status": "modified"}
+ ],
+ "expected": ["existing-cli-fix"]
+ },
+ {
+ "number": 167,
+ "title": "feat(freecad): add FreeCAD 1.0.2 backward compatibility",
+ "files": [
+ {"filename": "freecad/agent-harness/FREECAD.md", "status": "modified"},
+ {"filename": "freecad/agent-harness/cli_anything/freecad/core/generate.py", "status": "modified"},
+ {"filename": "freecad/agent-harness/cli_anything/freecad/freecad_cli.py", "status": "modified"},
+ {"filename": "freecad/agent-harness/cli_anything/freecad/skills/SKILL.md", "status": "modified"},
+ {"filename": "registry.json", "status": "modified"}
+ ],
+ "expected": ["cli-anything-hub", "cli-anything-skill", "existing-cli-fix"]
+ },
+ {
+ "number": 106,
+ "title": "fix: split token-aware into default exclusions + optional --economic mode",
+ "files": [
+ {"filename": "cli-anything-plugin/HARNESS.md", "status": "modified"},
+ {"filename": "cli-anything-plugin/commands/cli-anything.md", "status": "modified"},
+ {"filename": "cli-anything-plugin/commands/refine.md", "status": "modified"}
+ ],
+ "expected": ["cli-anything-skill"]
+ },
+ {
+ "number": "registry-maintenance",
+ "title": "Update registry dates",
+ "files": [
+ {"filename": "registry.json", "status": "modified"}
+ ],
+ "expected": ["cli-anything-hub"]
+ }
+]
diff --git a/.github/scripts/tests/pr-labeler.test.js b/.github/scripts/tests/pr-labeler.test.js
new file mode 100644
index 000000000..887e7d863
--- /dev/null
+++ b/.github/scripts/tests/pr-labeler.test.js
@@ -0,0 +1,199 @@
+const assert = require("node:assert/strict");
+const fs = require("node:fs");
+const path = require("node:path");
+const test = require("node:test");
+
+const labeler = require("../pr-labeler.js");
+const fixtures = require("./pr-labeler-fixtures.json");
+
+const REPO_ROOT = path.resolve(__dirname, "../../..");
+const LABELER_CONFIG_PATH = path.join(REPO_ROOT, ".github/labeler.yml");
+const EXPECTED_LABELS = [
+ "cli-anything-hub",
+ "cli-anything-skill",
+ "documentation",
+ "existing-cli-fix",
+ "github-actions",
+ "new-cli",
+];
+
+function normalizeLabels(labels) {
+ return [...labels].sort();
+}
+
+function sameLabels(left, right) {
+ return JSON.stringify(left) === JSON.stringify(right);
+}
+
+function loadLabelerPatterns() {
+ const patternsByLabel = new Map();
+ let currentLabel = null;
+
+ for (const line of fs.readFileSync(LABELER_CONFIG_PATH, "utf8").split(/\r?\n/)) {
+ const labelMatch = line.match(/^([a-z0-9-]+):$/);
+ if (labelMatch) {
+ currentLabel = labelMatch[1];
+ patternsByLabel.set(currentLabel, []);
+ continue;
+ }
+
+ const patternMatch = line.match(/-\s+"([^"]+)"/);
+ if (currentLabel && patternMatch) {
+ patternsByLabel.get(currentLabel).push(patternMatch[1]);
+ }
+ }
+
+ return patternsByLabel;
+}
+
+function globToRegex(pattern) {
+ let regex = "^";
+
+ for (let index = 0; index < pattern.length; index += 1) {
+ const char = pattern[index];
+ const next = pattern[index + 1];
+ const afterNext = pattern[index + 2];
+
+ if (char === "*" && next === "*" && afterNext === "/") {
+ regex += "(?:.*/)?";
+ index += 2;
+ continue;
+ }
+
+ if (char === "*" && next === "*") {
+ regex += ".*";
+ index += 1;
+ continue;
+ }
+
+ if (char === "*") {
+ regex += "[^/]*";
+ continue;
+ }
+
+ regex += char.replace(/[|\\{}()[\]^$+?.]/g, "\\$&");
+ }
+
+ regex += "$";
+ return new RegExp(regex);
+}
+
+function computePathLabels(files) {
+ const labels = new Set();
+ const patternsByLabel = loadLabelerPatterns();
+
+ for (const [label, patterns] of patternsByLabel.entries()) {
+ const regexes = patterns.map(globToRegex);
+ if (files.some((file) => regexes.some((regex) => regex.test(file.filename)))) {
+ labels.add(label);
+ }
+ }
+
+ return labels;
+}
+
+function computeAllLabels(sample) {
+ const labels = new Set(labeler.computeScriptLabels(sample.files, sample.title));
+ for (const pathLabel of computePathLabels(sample.files)) {
+ labels.add(pathLabel);
+ }
+ return normalizeLabels(labels);
+}
+
+function summarizeMetrics(results) {
+ const labels = new Set(EXPECTED_LABELS);
+ for (const result of results) {
+ for (const label of result.expected) labels.add(label);
+ for (const label of result.predicted) labels.add(label);
+ }
+
+ const perLabel = {};
+ for (const label of labels) {
+ let truePositive = 0;
+ let falsePositive = 0;
+ let falseNegative = 0;
+
+ for (const result of results) {
+ const expected = new Set(result.expected);
+ const predicted = new Set(result.predicted);
+
+ if (expected.has(label) && predicted.has(label)) truePositive += 1;
+ if (!expected.has(label) && predicted.has(label)) falsePositive += 1;
+ if (expected.has(label) && !predicted.has(label)) falseNegative += 1;
+ }
+
+ const precisionDenominator = truePositive + falsePositive;
+ const recallDenominator = truePositive + falseNegative;
+
+ perLabel[label] = {
+ truePositive,
+ falsePositive,
+ falseNegative,
+ precision: precisionDenominator === 0 ? 1 : truePositive / precisionDenominator,
+ recall: recallDenominator === 0 ? 1 : truePositive / recallDenominator,
+ };
+ }
+
+ const exactMatches = results.filter((result) => sameLabels(result.predicted, result.expected)).length;
+
+ return {
+ exactAccuracy: exactMatches / results.length,
+ perLabel,
+ };
+}
+
+test("real PR fixture label accuracy and recall", () => {
+ const results = fixtures.map((sample) => ({
+ number: sample.number,
+ title: sample.title,
+ expected: normalizeLabels(sample.expected),
+ predicted: computeAllLabels(sample),
+ }));
+
+ const mismatches = results.filter((result) => {
+ return !sameLabels(result.predicted, result.expected);
+ });
+
+ const metrics = summarizeMetrics(results);
+
+ assert.deepStrictEqual(mismatches, [], JSON.stringify({mismatches, metrics}, null, 2));
+ assert.equal(metrics.exactAccuracy, 1);
+
+ for (const [label, metric] of Object.entries(metrics.perLabel)) {
+ assert.equal(metric.precision, 1, `${label} precision: ${JSON.stringify(metric)}`);
+ assert.equal(metric.recall, 1, `${label} recall: ${JSON.stringify(metric)}`);
+ }
+});
+
+test("labeler config and script agree on the supported label set", () => {
+ const configLabels = new Set(loadLabelerPatterns().keys());
+ const scriptLabels = new Set(Object.keys(labeler.LABELS));
+ for (const label of EXPECTED_LABELS) {
+ assert(scriptLabels.has(label), `${label} must be creatable by pr-labeler.js`);
+ }
+
+ assert(configLabels.has("cli-anything-hub"));
+ assert(configLabels.has("cli-anything-skill"));
+ assert(configLabels.has("github-actions"));
+});
+
+test("registry-only maintenance is not treated as a new CLI", () => {
+ const labels = computeAllLabels({
+ title: "Update registry dates",
+ files: [{filename: "registry.json", status: "modified"}],
+ });
+
+ assert.deepStrictEqual(labels, ["cli-anything-hub"]);
+});
+
+test("mixed README and harness changes are not documentation-only", () => {
+ const labels = computeAllLabels({
+ title: "fix(blender): update docs and render behavior",
+ files: [
+ {filename: "README.md", status: "modified"},
+ {filename: "blender/agent-harness/cli_anything/blender/core/render.py", status: "modified"},
+ ],
+ });
+
+ assert.deepStrictEqual(labels, ["existing-cli-fix"]);
+});
diff --git a/.github/workflows/pr-labeler-tests.yml b/.github/workflows/pr-labeler-tests.yml
new file mode 100644
index 000000000..c3e41f423
--- /dev/null
+++ b/.github/workflows/pr-labeler-tests.yml
@@ -0,0 +1,43 @@
+name: PR Labeler Tests
+
+on:
+ pull_request:
+ paths:
+ - ".github/labeler.yml"
+ - ".github/scripts/pr-labeler.js"
+ - ".github/scripts/tests/pr-labeler-fixtures.json"
+ - ".github/scripts/tests/pr-labeler.test.js"
+ - ".github/workflows/pr-labeler.yml"
+ - ".github/workflows/pr-labeler-tests.yml"
+ push:
+ branches:
+ - main
+ paths:
+ - ".github/labeler.yml"
+ - ".github/scripts/pr-labeler.js"
+ - ".github/scripts/tests/pr-labeler-fixtures.json"
+ - ".github/scripts/tests/pr-labeler.test.js"
+ - ".github/workflows/pr-labeler.yml"
+ - ".github/workflows/pr-labeler-tests.yml"
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+jobs:
+ test:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Check out repository
+ uses: actions/checkout@v4
+
+ - name: Set up Node
+ uses: actions/setup-node@v4
+ with:
+ node-version: "20"
+
+ - name: Validate PR labeler script
+ run: node --check .github/scripts/pr-labeler.js
+
+ - name: Run PR labeler tests
+ run: node .github/scripts/tests/pr-labeler.test.js
diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml
new file mode 100644
index 000000000..0214c63e8
--- /dev/null
+++ b/.github/workflows/pr-labeler.yml
@@ -0,0 +1,33 @@
+name: Pull Request Labeler
+
+on:
+ pull_request_target:
+ types: [opened, synchronize, reopened, ready_for_review]
+
+permissions:
+ contents: read
+ issues: write
+ pull-requests: write
+
+jobs:
+ label:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Check out base branch
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ github.event.pull_request.base.ref }}
+ persist-credentials: false
+
+ - name: Apply computed labels
+ uses: actions/github-script@v7
+ with:
+ script: |
+ const labelPullRequest = require("./.github/scripts/pr-labeler.js");
+ await labelPullRequest({ github, context, core });
+
+ - name: Apply path labels
+ uses: actions/labeler@v6
+ with:
+ configuration-path: .github/labeler.yml
+ sync-labels: true
diff --git a/.gitignore b/.gitignore
index 5ea93cc6f..d87b6ce41 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,6 +24,7 @@
!/.github/workflows/
!/.github/scripts/
!/.github/CODEOWNERS
+!/.github/labeler.yml
!/.github/PULL_REQUEST_TEMPLATE.md
!/.github/ISSUE_TEMPLATE/