Skip to content

BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.

feat(files): docparser taskid passby

Christianaca15a1a95b8
diff --git a/README.md b/README.md
index 511132f..466ff5e 100644
--- a/README.md
+++ b/README.md
@@ -7,7 +7,7 @@
<h1 align="center">🧠 BrainAPI</h1>
<p align="center">
- <strong>A cognitive memory layer for AI applications</strong>
+ <strong>A cognitive reasoning memory layer for modern applications</strong>
<br/>
<em>Transform unstructured data into actionable knowledge with human-inspired retrieval</em>
</p>
diff --git a/src/config.py b/src/config.py
index efb66da..85517d9 100644
--- a/src/config.py
+++ b/src/config.py
@@ -3,7 +3,7 @@ File: /config.py
Created Date: Sunday October 19th 2025
Author: Christian Nonis <[email protected]>
-----
-Last Modified: Tuesday December 23rd 2025 9:24:20 pm
+Last Modified: Thursday January 29th 2026 8:43:59 pm
Modified By: Christian Nonis <[email protected]>
-----
"""
@@ -164,9 +164,9 @@ class CeleryConfig:
def __init__(self):
"""
Initialize CeleryConfig by loading the worker concurrency setting from the environment.
-
+
Sets `self.worker_concurrency` from the `CELERY_WORKER_CONCURRENCY` environment variable and validates its presence.
-
+
Raises:
ValueError: If `CELERY_WORKER_CONCURRENCY` is not set.
"""
@@ -183,7 +183,7 @@ class PricingConfig:
def __init__(self):
"""
Initialize pricing configuration from environment variables.
-
+
Attributes:
input_token_price (float): Price per input token from INPUT_TOKEN_PRICE, defaults to 0.0 if unset.
output_token_price (float): Price per output token from OUTPUT_TOKEN_PRICE, defaults to 0.0 if unset.
@@ -200,9 +200,9 @@ class Config:
def __init__(self):
"""
Initialize the application's central configuration by composing environment-backed sub-configurations and loading runtime flags.
-
+
This constructor instantiates Azure, Redis, Neo4j, Milvus, Embeddings, Mongo, GCP, Celery, and Pricing configuration objects, reads the RUN_GRAPH_CONSOLIDATOR flag into `run_graph_consolidator`, and loads the `BRAINPAT_TOKEN` into `brainpat_token`.
-
+
Raises:
ValueError: If `BRAINPAT_TOKEN` is not set in the environment.
"""
@@ -219,10 +219,14 @@ class Config:
self.run_graph_consolidator = (
os.getenv("RUN_GRAPH_CONSOLIDATOR", "true") == "true"
)
+ self.docparser_endpoint = os.getenv("DOCPARSER_ENDPOINT")
+ self.docparser_token = os.getenv("DOCPARSER_TOKEN")
+ self.app_host = os.getenv("APP_HOST")
+
self.brainpat_token = os.getenv("BRAINPAT_TOKEN")
if not self.brainpat_token:
raise ValueError("BrainPAT token is not set")
-config = Config()
\ No newline at end of file
+config = Config()
diff --git a/src/lib/embeddings/client.py b/src/lib/embeddings/client.py
index 97b6e7c..05e9ea6 100644
--- a/src/lib/embeddings/client.py
+++ b/src/lib/embeddings/client.py
@@ -4,7 +4,7 @@ Created Date: Friday October 24th 2025
Author: Christian Nonis <[email protected]>
-----
Last Modified: Friday October 24th 2025 7:11:17 pm
-Modified By: the developer formerly known as Christian Nonis at <[email protected]>
+Modified By: Christian Nonis <[email protected]>
-----
"""
@@ -119,7 +119,9 @@ class EmbeddingsClient(EmbeddingsClient):
raise EmbeddingError(error_msg) from e
except RequestException as e:
error_msg = f"Request error during embedding: {e}"
- print(f"Embedding encoding failed: {error_msg}")
+ print(
+ f"Embedding encoding failed: {error_msg} -- input was: {type(text)} - {len(text) if isinstance(text, str) else "not a string"}"
+ )
raise EmbeddingError(error_msg) from e
except Exception as e:
error_msg = f"Unexpected error during embedding: {e}"
diff --git a/src/services/api/routes/ingest.py b/src/services/api/routes/ingest.py
index e0f72cb..44f13bf 100644
--- a/src/services/api/routes/ingest.py
+++ b/src/services/api/routes/ingest.py
@@ -3,20 +3,23 @@ File: /ingest.py
Created Date: Sunday October 19th 2025
Author: Christian Nonis <[email protected]>
-----
-Last Modified: Sunday October 19th 2025 12:41:30 pm
-Modified By: the developer formerly known as Christian Nonis at <[email protected]>
+Last Modified: Monday January 5th 2026 7:57:27 pm
+Modified By: Christian Nonis <[email protected]>
-----
"""
import json
import asyncio
-from fastapi import APIRouter, HTTPException
+import requests
+from uuid import uuid4
+from fastapi import APIRouter, HTTPException, Request
from typing import Literal
from typing_extensions import Annotated
-from fastapi import Form, UploadFile
+from fastapi import File, Form, UploadFile
from starlette.responses import JSONResponse
from celery.exceptions import OperationalError
+from src.config import config
from src.services.api.constants.requests import (
IngestionRequestBody,
IngestionStructuredRequestBody,
@@ -34,14 +37,24 @@ ingest_router = APIRouter(prefix="/ingest", tags=["ingest"])
@ingest_router.post(path="/")
-async def ingest_data(data: IngestionRequestBody):
+async def ingest_data(data: IngestionRequestBody, request: Request):
"""
Ingest data to the processing pipeline and save to the memory.
"""
+
+ flow_task_identifier = None
+
+ if request.headers.get("Task-Identifier"):
+ flow_task_identifier = request.headers.get("Task-Identifier")
+
task = None
for attempt in range(MAX_TASK_RETRIES):
try:
- task = ingest_data_task.delay(data.model_dump())
+ kwargs = {"args": [data.model_dump()]}
+ if flow_task_identifier:
+ kwargs["task_id"] = flow_task_identifier
+
+ task = ingest_data_task.apply_async(**kwargs)
break
except OperationalError:
if attempt == MAX_TASK_RETRIES - 1:
@@ -89,14 +102,44 @@ async def ingest_structured_data(data: IngestionStructuredRequestBody):
@ingest_router.post(path="/file")
async def ingest_file(
- file: Annotated[UploadFile, Form()],
+ file: Annotated[UploadFile, File()],
brain_id: str = Form(default="default"),
- understanding_level: str = Annotated[
- Literal["basic", "deep"], Form(default="deep")
- ],
):
"""
Ingest a file into the processing pipeline and save to the memory.
"""
- return JSONResponse(content={"message": "File ingested successfully"})
+ task = str(uuid4())
+
+ cache_adapter.set(
+ key=f"task:{task}",
+ value=json.dumps({"status": "queued", "task_id": task}),
+ brain_id=brain_id,
+ expires_in=3600 * 24 * 7,
+ )
+
+ try:
+ file.file.seek(0)
+
+ # We are forwarding to the DocParser API to convert files into markdown text
+ # and by forwarding the webhook (our app host) and the task identifier,
+ # we can be called back from the DocParser API to start the ingestion process
+ # with the created markdown text.
+ response = requests.post(
+ f"{config.docparser_endpoint}/ingest",
+ files={"file": (file.filename or "file", file.file, file.content_type)},
+ data={
+ "brain_id": brain_id,
+ "webhook_callback": config.app_host,
+ "identifier": task,
+ },
+ headers={"Authorization": f"Bearer {config.docparser_token}"},
+ )
+ if response.status_code != 200:
+ raise HTTPException(status_code=500, detail=response.text)
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=str(e))
+
+ return JSONResponse(
+ content={"message": "File ingested successfully", "task_id": task}
+ )