BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.
Python ai-agentsai-memoryknowledge-graphknowledge-managementknowledge-representationreccomendation-system
from __future__ import annotations
import json
import re
from pathlib import Path
from typing import Any, Sequence
from src.core.search.recommend import behavior_weight
HAS_EVENT_NAME = "HAS"
ENTITY_TYPE = "ENTITY"
CLASS_LABEL = "CLASS"
TYPE_LABEL = "TYPE"
ATTR_LABEL = "ATTR"
EVENT_LABEL = "EVENT"
USER_TYPE = "USER"
OPTION_ALLOWLIST = frozenset(
{"color", "material", "category", "brand", "size", "style"}
)
_BEHAVIOR_EVENT = {
"view": "View",
"viewed": "View",
"click": "View",
"clicked": "View",
"cart": "AddToCart",
"add_to_cart": "AddToCart",
"added_to_cart": "AddToCart",
"addtocart": "AddToCart",
"purchase": "Purchase",
"purchased": "Purchase",
"buy": "Purchase",
"bought": "Purchase",
"favorite": "Favorite",
"favourite": "Favorite",
"favorites": "Favorite",
"favourites": "Favorite",
"wishlist": "Wishlist",
"add_to_favorite": "Favorite",
"addtofavorite": "Favorite",
"add_to_favourite": "Favorite",
"addtofavourite": "Favorite",
"add_to_wishlist": "Wishlist",
"addtowishlist": "Wishlist",
"follow": "Follow",
"flw": "Follow",
}
FEATURE_KEY_CAP = 16
FEATURE_KEY_PRIORITY = (
"style",
"mood",
"color",
"colour",
"material",
"shape",
"finish",
"fabric",
"design",
"woodtone",
)
FEATURE_SKIP_TOKENS = (
"width",
"height",
"weight",
"length",
"depth",
"capacity",
"temperature",
"rating",
"price",
"count",
"warranty",
"origin",
"assembly",
"thickness",
"diameter",
"clearance",
"sku",
)
CONTINUOUS_PROPERTY_KEYS = (
"price",
"rating",
"average_rating",
"rating_count",
"review_count",
"width",
"height",
"depth",
"weight",
"length",
)
BOOLEAN_NOISE = {
"",
"yes",
"no",
"true",
"false",
"y",
"n",
"0",
"1",
"none",
"n/a",
"na",
"unknown",
"null",
}
_LABEL_RE = re.compile(r"[^A-Za-z0-9]+")
_TEXT_FIELD_RE = re.compile(
r"^(Title|Brand|Color|Class|Category|Features|Locale|Hierarchy|Description|Price|Rating)\s*:\s*(.+)$",
re.IGNORECASE,
)
_HIER_SPLIT = re.compile(r"\s*[>/|]+\s*")
def entity_uuid(doc_id: str) -> str:
return str(doc_id)
def hub_uuid(kind: str, value: str) -> str:
return f"hub:{kind}:{value.strip().lower()}"
def event_uuid(owner: str, kind: str, seq: int) -> str:
return f"evt:{owner}:{kind}:{seq}"
def rel_uuid(owner: str, kind: str, seq: int) -> str:
return f"rel:{owner}:{kind}:{seq}"
def sanitize_label(raw: str) -> str:
cleaned = _LABEL_RE.sub("_", (raw or "").strip().upper()).strip("_")
if not cleaned or cleaned[0].isdigit():
return ATTR_LABEL
return cleaned[:40]
_DOC_ID_RE = re.compile(r"DOCID\s+(\S+)")
def compose_search_text(*parts: str | None) -> str:
chunks = [str(part).strip() for part in parts if str(part or "").strip()]
return " ".join(chunks)
def doc_id_from_text(text: str | None) -> str | None:
match = _DOC_ID_RE.search(str(text or ""))
if not match:
return None
doc_id = match.group(1).strip().rstrip(".")
return doc_id or None
def node_id_from_passage_text(text: str | None) -> str | None:
doc_id = doc_id_from_text(text)
if not doc_id:
return None
return entity_uuid(doc_id)
def entity_search_text(doc: dict[str, Any], fields: dict[str, Any] | None = None) -> str:
raw = str(doc.get("text") or "").strip()
if raw:
return raw
resolved = fields if fields is not None else doc_fields(doc)
return compose_search_text(
resolved.get("title"),
resolved.get("class"),
resolved.get("hierarchy"),
resolved.get("brand"),
resolved.get("color"),
resolved.get("features"),
resolved.get("description"),
)
def node_embed_text(node_data: Any) -> tuple[str, str]:
props = getattr(node_data, "properties", None) or {}
if not isinstance(props, dict):
props = {}
search_text = str(props.get("search_text") or "").strip()
name = str(getattr(node_data, "name", "") or "").strip()
if search_text:
uuid = str(getattr(node_data, "uuid", "") or "").strip()
return search_text, f"uuid:{uuid}" if uuid else search_text
return name, name
def split_hierarchy(raw: str | None) -> list[str]:
text = str(raw or "").strip()
if not text:
return []
parts = [item.strip() for item in _HIER_SPLIT.split(text) if item.strip()]
seen: set[str] = set()
out: list[str] = []
for part in parts:
key = part.lower()
if key in seen:
continue
seen.add(key)
out.append(part)
return out
def _feature_key_rank(key: str) -> int:
lowered = key.lower().replace(" ", "")
for index, token in enumerate(FEATURE_KEY_PRIORITY):
if token in lowered:
return index
return len(FEATURE_KEY_PRIORITY)
def _skip_feature_key(key: str) -> bool:
lowered = key.lower().replace(" ", "")
if not lowered:
return True
return any(token in lowered for token in FEATURE_SKIP_TOKENS)
def parse_feature_string(
raw: str | None, *, cap: int = FEATURE_KEY_CAP
) -> list[tuple[str, str]]:
text = str(raw or "").strip()
if not text:
return []
pairs: list[tuple[str, str]] = []
seen: set[tuple[str, str]] = set()
parts = [part.strip() for part in text.split("|") if part.strip()]
if ":" in text and not parts:
parts = [text]
for part in parts:
if ":" not in part:
continue
key, value = part.split(":", 1)
key = key.strip()
value = value.strip()
if not key or not value:
continue
if _skip_feature_key(key):
continue
if value.lower() in BOOLEAN_NOISE:
continue
item = (key, value)
if item in seen:
continue
seen.add(item)
pairs.append(item)
pairs.sort(key=lambda item: (_feature_key_rank(item[0]), item[0].lower()))
if cap <= 0:
return pairs
kept: list[tuple[str, str]] = []
keys: set[str] = set()
for key, value in pairs:
key_id = key.lower()
if key_id not in keys and len(keys) >= cap:
continue
keys.add(key_id)
kept.append((key, value))
return kept
def _fields_from_text(text: str) -> dict[str, str]:
out: dict[str, str] = {}
for line in (text or "").splitlines():
match = _TEXT_FIELD_RE.match(line.strip())
if not match:
continue
key = match.group(1).strip().lower()
value = match.group(2).strip()
if value:
out[key] = value
return out
def doc_fields(doc: dict[str, Any]) -> dict[str, Any]:
text_fields = _fields_from_text(str(doc.get("text") or ""))
title = str(doc.get("title") or text_fields.get("title") or "").strip()
class_name = str(
doc.get("class")
or doc.get("product_class")
or text_fields.get("class")
or ""
).strip()
hierarchy = str(
doc.get("hierarchy")
or doc.get("category_hierarchy")
or text_fields.get("hierarchy")
or text_fields.get("category")
or ""
).strip()
if not class_name:
class_name = str(text_fields.get("category") or "").strip()
brand = str(doc.get("brand") or text_fields.get("brand") or "").strip()
color = str(doc.get("color") or text_fields.get("color") or "").strip()
locale = str(doc.get("locale") or text_fields.get("locale") or "").strip()
features = str(doc.get("features") or text_fields.get("features") or "").strip()
description = str(
doc.get("description") or text_fields.get("description") or ""
).strip()
dataset = str(doc.get("dataset") or "").strip()
properties: dict[str, str] = {}
for key in CONTINUOUS_PROPERTY_KEYS:
value = str(doc.get(key) or text_fields.get(key) or "").strip()
if value:
properties[key] = value
return {
"title": title,
"class": class_name,
"hierarchy": hierarchy,
"brand": brand,
"color": color,
"locale": locale,
"features": features,
"description": description,
"dataset": dataset,
"properties": properties,
}
def _triple_pred_name(value: Any) -> str:
if value is None:
return ""
if isinstance(value, dict):
return str(value.get("name") or "").strip().upper()
return str(getattr(value, "name", "") or "").strip().upper()
def is_static_has_triple(triple: Any) -> bool:
if isinstance(triple, dict):
event = triple.get("event")
subj_event = triple.get("subj_event")
event_obj = triple.get("event_obj")
else:
event = getattr(triple, "event", None)
subj_event = getattr(triple, "subj_event", None)
event_obj = getattr(triple, "event_obj", None)
happened = None
event_name = ""
if isinstance(event, dict):
happened = event.get("happened_at")
event_name = str(event.get("name") or "").strip().upper()
elif event is not None:
happened = getattr(event, "happened_at", None)
event_name = str(getattr(event, "name", "") or "").strip().upper()
if happened:
return False
pred = _triple_pred_name(subj_event) or HAS_EVENT_NAME
if event is None:
return pred == HAS_EVENT_NAME
other = _triple_pred_name(event_obj) or HAS_EVENT_NAME
return event_name == HAS_EVENT_NAME and pred == HAS_EVENT_NAME and other == HAS_EVENT_NAME
def has_triple(
*,
subject: dict[str, Any],
object_node: dict[str, Any],
seq: int,
kind: str,
) -> dict[str, Any]:
owner = str(subject.get("uuid") or subject.get("name") or "node")
return {
"subject": subject,
"subj_event": {
"name": HAS_EVENT_NAME,
"uuid": rel_uuid(owner, f"{kind}_has", seq),
},
"object": object_node,
}
def _entity_node(
doc_id: str,
name: str,
*,
description: str | None = None,
properties: dict[str, Any] | None = None,
search_text: str | None = None,
) -> dict[str, Any]:
props = dict(properties or {})
blob = str(search_text or "").strip() or compose_search_text(name, description)
if blob:
props["search_text"] = blob
node: dict[str, Any] = {
"name": name or doc_id,
"type": ENTITY_TYPE,
"uuid": entity_uuid(doc_id),
"labels": [ENTITY_TYPE],
"properties": props,
}
if description:
node["description"] = description
return node
def _hub_node(kind: str, value: str, extra_label: str | None = None) -> dict[str, Any]:
labels = [kind]
if extra_label:
sanitized = sanitize_label(extra_label)
if sanitized and sanitized not in labels:
labels.append(sanitized)
search_text = compose_search_text(extra_label or kind, value)
return {
"name": value,
"type": kind,
"uuid": hub_uuid(kind.lower(), value),
"labels": labels,
"description": search_text,
"properties": {
"search_text": search_text,
"catalog_labels": labels,
},
}
def doc_to_triples(doc: dict[str, Any]) -> list[dict[str, Any]]:
doc_id = str(doc.get("doc_id") or "").strip()
if not doc_id:
return []
fields = doc_fields(doc)
name = fields["title"] or doc_id
subject = _entity_node(
doc_id,
name,
description=fields["description"] or None,
properties=dict(fields["properties"] or {}),
search_text=entity_search_text(doc, fields),
)
triples: list[dict[str, Any]] = []
seq = 0
class_name = fields["class"]
if class_name:
triples.append(
has_triple(
subject=subject,
object_node=_hub_node(CLASS_LABEL, class_name),
seq=seq,
kind="class",
)
)
seq += 1
class_key = class_name.strip().lower()
for part in split_hierarchy(fields["hierarchy"]):
if part.strip().lower() == class_key:
continue
triples.append(
has_triple(
subject=subject,
object_node=_hub_node(TYPE_LABEL, part, extra_label="CATEGORY"),
seq=seq,
kind="hierarchy",
)
)
seq += 1
if fields["brand"]:
triples.append(
has_triple(
subject=subject,
object_node=_hub_node(ATTR_LABEL, fields["brand"], extra_label="BRAND"),
seq=seq,
kind="brand",
)
)
seq += 1
if fields["color"]:
triples.append(
has_triple(
subject=subject,
object_node=_hub_node(ATTR_LABEL, fields["color"], extra_label="COLOR"),
seq=seq,
kind="color",
)
)
seq += 1
if fields["locale"]:
triples.append(
has_triple(
subject=subject,
object_node=_hub_node(TYPE_LABEL, fields["locale"], extra_label="LOCALE"),
seq=seq,
kind="locale",
)
)
seq += 1
for key, value in parse_feature_string(fields["features"]):
triples.append(
has_triple(
subject=subject,
object_node=_hub_node(ATTR_LABEL, value, extra_label=key),
seq=seq,
kind=f"feat:{sanitize_label(key).lower()}",
)
)
seq += 1
if not triples:
dataset = fields["dataset"] or "ITEM"
triples.append(
has_triple(
subject=subject,
object_node=_hub_node(TYPE_LABEL, dataset),
seq=seq,
kind="type",
)
)
return triples
def docs_to_triples(docs: Sequence[dict[str, Any]]) -> list[dict[str, Any]]:
triples: list[dict[str, Any]] = []
for doc in docs:
triples.extend(doc_to_triples(doc))
return triples
def catalog_entity_backfill_rows(docs: Sequence[dict[str, Any]]) -> list[dict[str, str]]:
rows: list[dict[str, str]] = []
for doc in docs:
doc_id = str(doc.get("doc_id") or "").strip()
if not doc_id:
continue
fields = doc_fields(doc)
rows.append(
{
"uuid": entity_uuid(doc_id),
"name": str(fields.get("title") or doc_id),
"search_text": entity_search_text(doc, fields),
}
)
return rows
def format_happened_at(timestamp: str | None) -> str | None:
if not timestamp:
return None
raw = str(timestamp).strip()
if not raw:
return None
from datetime import datetime
if len(raw) == 10 and raw[2] == "/" and raw[5] == "/":
return raw
for fmt in (
"%Y-%m-%dT%H:%M:%S%z",
"%Y-%m-%dT%H:%M:%S.%f%z",
"%Y-%m-%dT%H:%M:%SZ",
"%Y-%m-%dT%H:%M:%S",
"%Y-%m-%d %H:%M:%S",
"%Y-%m-%d",
"%m/%d/%Y",
"%d/%m/%Y",
):
try:
parsed = datetime.strptime(
raw.replace("Z", "+0000") if fmt.endswith("%z") and raw.endswith("Z") else raw,
fmt,
)
return parsed.strftime("%m/%d/%Y")
except ValueError:
continue
try:
parsed = datetime.fromisoformat(raw.replace("Z", "+00:00"))
return parsed.strftime("%m/%d/%Y")
except ValueError:
return raw
def _interaction_event_name(behavior: str) -> str:
key = (behavior or "").strip().lower().replace("-", "_").replace(" ", "_")
if key in _BEHAVIOR_EVENT:
return _BEHAVIOR_EVENT[key]
compact = key.replace("_", "")
for alias, name in _BEHAVIOR_EVENT.items():
if alias.replace("_", "") == compact:
return name
label = (behavior or "Interaction").strip() or "Interaction"
return label[:1].upper() + label[1:]
def _option_pairs(row: dict[str, Any]) -> list[tuple[str, str]]:
pairs: list[tuple[str, str]] = []
seen: set[tuple[str, str]] = set()
for src in (row.get("options"), row.get("attributes")):
if not isinstance(src, dict):
continue
for key, raw in src.items():
facet = str(key).strip().lower()
if facet not in OPTION_ALLOWLIST or raw is None:
continue
value = str(raw).strip()
if not value:
continue
pair = (facet, value)
if pair in seen:
continue
seen.add(pair)
pairs.append(pair)
return pairs
def prefers_triple(
*,
user_id: str,
facet: str,
value: str,
seq: int,
amount: float,
) -> dict[str, Any]:
extra = "CATEGORY" if facet == "category" else facet.upper()
kind = CLASS_LABEL if facet == "category" else ATTR_LABEL
return {
"subject": {
"name": user_id,
"type": USER_TYPE,
"uuid": f"user:{user_id}",
"labels": [USER_TYPE],
},
"subj_event": {
"name": "PREFERS",
"uuid": rel_uuid(user_id, f"prefers:{facet}:{value}", seq),
"amount": amount,
"properties": {"facet": facet, "value": value, "weight": amount},
},
"object": _hub_node(kind, value, extra_label=extra),
}
def interaction_to_triples(
row: dict[str, Any] | Sequence[dict[str, Any]], *, seq: int = 1
) -> list[dict[str, Any]]:
if not isinstance(row, dict):
return interactions_to_triples(row, seq_start=seq)
user_id = str(row.get("user_id") or "").strip()
item_id = str(row.get("item_id") or "").strip()
if not user_id or not item_id:
return []
behavior = str(row.get("behavior") or "interaction").strip() or "interaction"
event_name = _interaction_event_name(behavior)
happened_at = format_happened_at(row.get("timestamp") or row.get("ts"))
item_name = str(row.get("title") or row.get("item_name") or item_id)
event_node: dict[str, Any] = {
"name": event_name,
"type": EVENT_LABEL,
"uuid": event_uuid(user_id, f"{item_id}:{behavior}", seq),
"labels": [EVENT_LABEL],
}
if happened_at:
event_node["happened_at"] = happened_at
user_node = {
"name": user_id,
"type": USER_TYPE,
"uuid": f"user:{user_id}",
"labels": [USER_TYPE],
}
triples = [
{
"subject": user_node,
"subj_event": {
"name": "MADE",
"uuid": rel_uuid(user_id, f"made:{item_id}", seq),
},
"event": event_node,
"event_obj": {
"name": "TARGETED",
"uuid": rel_uuid(user_id, f"tgt:{item_id}", seq),
},
"object": _entity_node(item_id, item_name),
}
]
weight = behavior_weight(event_name)
for facet, value in _option_pairs(row):
triples.append(
prefers_triple(
user_id=user_id,
facet=facet,
value=value,
seq=seq,
amount=weight,
)
)
catalog_doc = {
"doc_id": item_id,
"title": item_name,
"class": row.get("category") or row.get("class"),
"brand": row.get("brand"),
"color": row.get("color"),
"features": row.get("features"),
"dataset": "interaction",
}
existing = {
str((t.get("event") or {}).get("uuid") or "")
for t in triples
if (t.get("event") or {}).get("uuid")
}
for triple in doc_to_triples(catalog_doc):
event_id = str((triple.get("event") or {}).get("uuid") or "")
if event_id and event_id in existing:
continue
triples.append(triple)
return triples
def interactions_to_triples(
rows: Sequence[dict[str, Any]],
*,
seq_start: int = 1,
) -> list[dict[str, Any]]:
triples: list[dict[str, Any]] = []
for i, row in enumerate(rows):
triples.extend(interaction_to_triples(row, seq=seq_start + i))
return triples
def load_interaction_rows(path: Path) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
for line in path.read_text(encoding="utf-8").splitlines():
text = line.strip()
if not text:
continue
obj = json.loads(text)
if not isinstance(obj, dict):
continue
if obj.get("user_id") and obj.get("item_id"):
rows.append(obj)
return rows
def structured_ingest_body(
triples: Sequence[dict[str, Any]],
*,
brain_id: str,
mode: str = "deterministic",
) -> dict[str, Any]:
return {
"mode": mode,
"brain_id": brain_id,
"data": list(triples),
}
__all__ = [
"HAS_EVENT_NAME",
"ENTITY_TYPE",
"CLASS_LABEL",
"TYPE_LABEL",
"ATTR_LABEL",
"EVENT_LABEL",
"USER_TYPE",
"FEATURE_KEY_CAP",
"compose_search_text",
"doc_fields",
"doc_id_from_text",
"doc_to_triples",
"entity_search_text",
"node_id_from_passage_text",
"docs_to_triples",
"catalog_entity_backfill_rows",
"entity_uuid",
"format_happened_at",
"has_triple",
"is_static_has_triple",
"hub_uuid",
"interaction_to_triples",
"prefers_triple",
"interactions_to_triples",
"load_interaction_rows",
"node_embed_text",
"parse_feature_string",
"sanitize_label",
"split_hierarchy",
"structured_ingest_body",
]