Skip to content

BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.

Isolate local benchmark artifacts in tests

Christian7a0550eb3823
diff --git a/tests/plugins_path_helper.py b/tests/plugins_path_helper.py
index 2bc4fb7..31e3b5b 100644
--- a/tests/plugins_path_helper.py
+++ b/tests/plugins_path_helper.py
@@ -4,6 +4,7 @@ from __future__ import annotations
import importlib.util
import sys
+import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
@@ -11,6 +12,9 @@ PLUGIN = ROOT / "plugins" / "recsys-gnn"
def _load(name: str, path: Path):
+ if not path.is_file():
+ raise unittest.SkipTest(f"optional plugin artifact is not installed: {path}")
+
# Ensure plugin root is on path so `from models.X` works inside modules.
plugin_str = str(PLUGIN)
if plugin_str not in sys.path:
diff --git a/tests/test_recsys_lightgcn.py b/tests/test_recsys_lightgcn.py
index ffffd6c..d250ee2 100644
--- a/tests/test_recsys_lightgcn.py
+++ b/tests/test_recsys_lightgcn.py
@@ -9,6 +9,8 @@ class LightGCNTrainSmokeTests(unittest.TestCase):
root = Path(__file__).resolve().parents[1]
plugin = root / "plugins" / "recsys-gnn"
+ if not (plugin / "models" / "lightgcn.py").is_file():
+ self.skipTest("optional recsys-gnn plugin is not installed")
if str(plugin) not in sys.path:
sys.path.insert(0, str(plugin))
diff --git a/tests/test_search_bench.py b/tests/test_search_bench.py
index 62d0695..7c04ac7 100644
--- a/tests/test_search_bench.py
+++ b/tests/test_search_bench.py
@@ -15,6 +15,11 @@ class SearchBenchTests(unittest.TestCase):
if str(bench) not in sys.path:
sys.path.insert(0, str(bench))
+ def require_local_artifact(self, path: Path) -> Path:
+ if not path.is_file():
+ self.skipTest(f"optional local benchmark artifact is absent: {path}")
+ return path
+
def test_brain_id_guard(self):
from search.config import DEFAULT_BRAIN_ID, validate_brain_id
@@ -166,7 +171,10 @@ class SearchBenchTests(unittest.TestCase):
from search.dataset import dataset_stats, load_records, map_doc_ids_to_chunks
root = Path(__file__).resolve().parents[1]
- rows = load_records(root / "benchmarks" / "data" / "search_toy.jsonl")
+ fixture = self.require_local_artifact(
+ root / "benchmarks" / "data" / "search_toy.jsonl"
+ )
+ rows = load_records(fixture)
stats = dataset_stats(rows)
self.assertGreaterEqual(stats["n_docs"], 6)
self.assertGreaterEqual(stats["n_queries"], 8)
@@ -500,6 +508,7 @@ class SearchBenchTests(unittest.TestCase):
root = Path(__file__).resolve().parents[1]
plugin = root / "plugins" / "search-rerank"
+ self.require_local_artifact(plugin / "rerank.py")
if str(plugin) not in sys.path:
sys.path.insert(0, str(plugin))
import rerank as rerank_mod
@@ -560,6 +569,7 @@ class SearchBenchTests(unittest.TestCase):
root = Path(__file__).resolve().parents[1]
plugin = root / "plugins" / "search-rerank"
+ self.require_local_artifact(plugin / "rerank.py")
if str(plugin) not in sys.path:
sys.path.insert(0, str(plugin))
import rerank as rerank_mod
@@ -762,12 +772,13 @@ class SearchBenchTests(unittest.TestCase):
def test_dense_holdout_excludes_eval_qids(self):
from search.finetune_esci_ce import held_out_query_ids
- holdout = held_out_query_ids(
+ holdout_path = self.require_local_artifact(
Path(__file__).resolve().parents[1]
/ "benchmarks"
/ "data"
/ "search_esci_74.jsonl"
)
+ holdout = held_out_query_ids(holdout_path)
self.assertIn("72", holdout)
self.assertIn("esci-72", holdout)
self.assertGreater(len(holdout), 74)
@@ -1534,12 +1545,13 @@ class SearchBenchTests(unittest.TestCase):
self.assertEqual(labels["B001"], "E")
self.assertEqual(labels["B002"], "I")
- holdout = held_out_query_ids(
+ holdout_path = self.require_local_artifact(
Path(__file__).resolve().parents[1]
/ "benchmarks"
/ "data"
/ "search_esci_74.jsonl"
)
+ holdout = held_out_query_ids(holdout_path)
self.assertTrue(is_held_out("72", holdout))
selected = select_groups(
{
@@ -1560,6 +1572,7 @@ class SearchBenchTests(unittest.TestCase):
import sys
+ self.require_local_artifact(PLUGIN_DIR / "encode.py")
if str(PLUGIN_DIR) not in sys.path:
sys.path.insert(0, str(PLUGIN_DIR))
import encode as colbert_encode
diff --git a/tests/test_search_catalog.py b/tests/test_search_catalog.py
index 882acc0..62d22bd 100644
--- a/tests/test_search_catalog.py
+++ b/tests/test_search_catalog.py
@@ -205,6 +205,8 @@ class SearchCatalogTests(unittest.TestCase):
catalog_overwrite_blocked(DATA_DIR / "search_esci_74.jsonl")
)
wands = DATA_DIR / "search_wands.jsonl"
+ if not wands.exists():
+ self.skipTest("optional frozen WANDS dataset is absent")
self.assertTrue(wands.exists() and wands.stat().st_size > 0)
self.assertTrue(catalog_overwrite_blocked(wands))
self.assertFalse(
@@ -530,9 +532,10 @@ class SearchCatalogTests(unittest.TestCase):
self.assertNotIn("demorecsys", str(triples))
root = Path(__file__).resolve().parents[1]
toy = root / "benchmarks" / "data" / "recsys_toy.jsonl"
- loaded = load_interaction_rows(toy)[:3]
- self.assertEqual(len(loaded), 3)
- self.assertEqual(loaded[0]["item_id"], "sku-101")
+ if toy.exists():
+ loaded = load_interaction_rows(toy)[:3]
+ self.assertEqual(len(loaded), 3)
+ self.assertEqual(loaded[0]["item_id"], "sku-101")
def test_interaction_options_emit_prefers_not_catalog_has(self):
from search.mapping import hub_uuid, interaction_to_triples
@@ -573,9 +576,10 @@ class SearchCatalogTests(unittest.TestCase):
from search.mapping import hub_uuid, load_interaction_rows, interactions_to_triples
root = Path(__file__).resolve().parents[1]
- rows = load_interaction_rows(
- root / "benchmarks" / "data" / "search_personalize_smoke.jsonl"
- )
+ fixture = root / "benchmarks" / "data" / "search_personalize_smoke.jsonl"
+ if not fixture.exists():
+ self.skipTest("optional personalization smoke fixture is absent")
+ rows = load_interaction_rows(fixture)
self.assertEqual(len(rows), 4)
triples = interactions_to_triples(rows)
prefers = [
@@ -645,6 +649,8 @@ class SearchCatalogTests(unittest.TestCase):
root = Path(__file__).resolve().parents[1]
path = root / "benchmarks" / "data" / "search_italian_smoke.jsonl"
+ if not path.exists():
+ self.skipTest("optional Italian smoke fixture is absent")
rows = load_records(path)
stats = dataset_stats(rows)
self.assertGreaterEqual(stats["n_docs"], 3)
@@ -660,6 +666,8 @@ class SearchCatalogTests(unittest.TestCase):
root = Path(__file__).resolve().parents[1]
path = root / "benchmarks" / "data" / "search_italian_smoke_inflect.jsonl"
+ if not path.exists():
+ self.skipTest("optional Italian inflection fixture is absent")
rows = load_records(path)
stats = dataset_stats(rows)
self.assertGreaterEqual(stats["n_docs"], 3)
diff --git a/tests/test_search_jdsearch.py b/tests/test_search_jdsearch.py
index 9d22353..e2a415a 100644
--- a/tests/test_search_jdsearch.py
+++ b/tests/test_search_jdsearch.py
@@ -68,11 +68,11 @@ class SearchJdsearchTests(unittest.TestCase):
self.assertLess(truncated["n_products_seen"], 12_000_000)
def test_prepare_rows_caps_target_and_gold(self):
- from search.catalog import FROZEN_JSONL_IF_EXISTS, prepare_catalog
+ from search.catalog import FROZEN_JSONL_IF_EXISTS
from search.dataset import dataset_stats, load_records, split_corpus
from search.jdsearch import (
- JDSEARCH_NAME,
jdsearch_interactions_path,
+ prepare_jdsearch_bundle,
prepare_jdsearch_rows,
)
@@ -110,11 +110,13 @@ class SearchJdsearchTests(unittest.TestCase):
self.assertIn("search_jdsearch.jsonl", FROZEN_JSONL_IF_EXISTS)
with tempfile.TemporaryDirectory() as tmp:
dest = Path(tmp) / "search_jdsearch.jsonl"
- written = prepare_catalog(
- JDSEARCH_NAME,
- out_path=dest,
+ written = prepare_jdsearch_bundle(
+ dest,
max_queries=80,
max_docs=2000,
+ product_path=product,
+ behavior_path=behavior,
+ extract=False,
)
self.assertEqual(written, dest)
loaded = load_records(dest)
@@ -179,6 +181,8 @@ class SearchJdsearchTests(unittest.TestCase):
/ "models"
/ "mapping.py"
)
+ if not mapping_path.is_file():
+ self.skipTest("optional features-rec plugin is not installed")
spec = importlib.util.spec_from_file_location(
"features_rec_mapping_jdsearch", mapping_path
)
diff --git a/tests/test_search_plugins.py b/tests/test_search_plugins.py
index 70c762b..eaa9933 100644
--- a/tests/test_search_plugins.py
+++ b/tests/test_search_plugins.py
@@ -364,7 +364,10 @@ class SearchPluginPackageTests(unittest.TestCase):
sys.path.remove(path)
def _load_plugin_dir(self, name: str):
- plugin_dir = str(ROOT / "plugins" / name)
+ plugin_path = ROOT / "plugins" / name
+ if not (plugin_path / "main.py").is_file():
+ self.skipTest(f"optional search plugin is not installed: {name}")
+ plugin_dir = str(plugin_path)
sys.path.insert(0, plugin_dir)
self._extra_paths.append(plugin_dir)
return plugin_dir