OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
remove log points
Edwin Jose689b40651ae0
diff --git a/.cursor/debug.log b/.cursor/debug.lognew file mode 100644index 000000000..6855bbaed--- /dev/null+++ b/.cursor/debug.log@@ -0,0 +1,11 @@+{"hypothesisId":"H3","location":"chunks/page.tsx:AccessControl","message":"chunks page fileData ACL","data":{"filename":"sample_doc.docx.docx","owner":"107549107036797017082","allowed_users_len":3,"allowed_groups_len":0,"showAccessControl":true},"timestamp":1770799353369}+{"hypothesisId":"H3","location":"chunks/page.tsx:AccessControl","message":"chunks page fileData ACL","data":{"filename":"sample_doc.docx.docx","owner":"107549107036797017082","allowed_users_len":3,"allowed_groups_len":0,"showAccessControl":true},"timestamp":1770799353371}+{"hypothesisId":"H3","location":"chunks/page.tsx:AccessControl","message":"chunks page fileData ACL","data":{"filename":"sample_doc.docx.docx","owner":"107549107036797017082","allowed_users_len":3,"allowed_groups_len":0,"showAccessControl":true},"timestamp":1770799353373}+{"hypothesisId":"H3","location":"chunks/page.tsx:AccessControl","message":"chunks page fileData ACL","data":{"filename":"sample_doc.docx.docx","owner":"107549107036797017082","allowed_users_len":3,"allowed_groups_len":0,"showAccessControl":true},"timestamp":1770799353374}+{"hypothesisId":"H3","location":"chunks/page.tsx:AccessControl","message":"chunks page fileData ACL","data":{"filename":"sample_doc.docx.docx","owner":"107549107036797017082","allowed_users_len":3,"allowed_groups_len":0,"showAccessControl":true},"timestamp":1770799353390}+{"hypothesisId":"H3","location":"chunks/page.tsx:AccessControl","message":"chunks page fileData ACL","data":{"filename":"sample_doc.docx.docx","owner":"107549107036797017082","allowed_users_len":3,"allowed_groups_len":0,"showAccessControl":true},"timestamp":1770799353391}+{"hypothesisId": "H1", "location": "search_service.py:chunks", "message": "backend chunk ACL sample", "data": {"chunk_samples": [{"filename": "openrag-documentation.pdf", "owner": false, "allowed_users_len": 0, "allowed_groups_len": 0}, {"filename": "openrag-documentation.pdf", "owner": false, "allowed_users_len": 0, "allowed_groups_len": 0}, {"filename": "openrag-documentation.pdf", "owner": false, "allowed_users_len": 0, "allowed_groups_len": 0}, {"filename": "openrag-documentation.pdf", "owner": false, "allowed_users_len": 0, "allowed_groups_len": 0}, {"filename": "openrag-documentation.pdf", "owner": false, "allowed_users_len": 0, "allowed_groups_len": 0}]}, "timestamp": 1770799353773.2039}+{"hypothesisId":"H5","location":"useGetSearchQuery.ts:firstChunk","message":"API first chunk ACL","data":{"filename":"openrag-documentation.pdf","hasOwner":false,"allowed_users_len":0,"allowed_groups_len":0},"timestamp":1770799353784}+{"hypothesisId":"H2","location":"useGetSearchQuery.ts:firstFile","message":"aggregated file ACL","data":{"filename":"openrag-documentation.pdf","owner":null,"allowed_users_len":0,"allowed_groups_len":0},"timestamp":1770799353785}+{"hypothesisId":"H3","location":"chunks/page.tsx:AccessControl","message":"chunks page fileData ACL","data":{"filename":"sample_doc.docx.docx","owner":"107549107036797017082","allowed_users_len":3,"allowed_groups_len":0,"showAccessControl":true},"timestamp":1770799353789}+{"hypothesisId":"H3","location":"chunks/page.tsx:AccessControl","message":"chunks page fileData ACL","data":{"filename":"sample_doc.docx.docx","owner":"107549107036797017082","allowed_users_len":3,"allowed_groups_len":0,"showAccessControl":true},"timestamp":1770799353791}diff --git a/frontend/app/api/queries/useGetSearchQuery.ts b/frontend/app/api/queries/useGetSearchQuery.tsindex 72a7c9ee3..c77be72a3 100644--- a/frontend/app/api/queries/useGetSearchQuery.ts+++ b/frontend/app/api/queries/useGetSearchQuery.ts@@ -138,29 +138,6 @@ export const useGetSearchQuery = (}const data = await response.json();- // #region agent log- const firstChunk = (data.results || [])[0] as ChunkResult | undefined;- if (firstChunk) {- const au = firstChunk.allowed_users;- const ag = firstChunk.allowed_groups;- fetch("http://127.0.0.1:7242/ingest/0459c69b-01d9-4ed6-bcff-99a7ce0d82ee", {- method: "POST",- headers: { "Content-Type": "application/json" },- body: JSON.stringify({- hypothesisId: "H5",- location: "useGetSearchQuery.ts:firstChunk",- message: "API first chunk ACL",- data: {- filename: firstChunk.filename,- hasOwner: Boolean(firstChunk.owner),- allowed_users_len: Array.isArray(au) ? au.length : 0,- allowed_groups_len: Array.isArray(ag) ? ag.length : 0,- },- timestamp: Date.now(),- }),- }).catch(() => {});- }- // #endregion// Group chunks by filename to create file results similar to page.tsxconst fileMap = new Map<string,@@ -234,28 +211,6 @@ export const useGetSearchQuery = (allowed_groups: file.allowed_groups || [],}));- // #region agent log- const firstFile = files[0];- if (firstFile) {- fetch("http://127.0.0.1:7242/ingest/0459c69b-01d9-4ed6-bcff-99a7ce0d82ee", {- method: "POST",- headers: { "Content-Type": "application/json" },- body: JSON.stringify({- hypothesisId: "H2",- location: "useGetSearchQuery.ts:firstFile",- message: "aggregated file ACL",- data: {- filename: firstFile.filename,- owner: firstFile.owner || null,- allowed_users_len: (firstFile.allowed_users || []).length,- allowed_groups_len: (firstFile.allowed_groups || []).length,- },- timestamp: Date.now(),- }),- }).catch(() => {});- }- // #endregion-return files;} catch (error) {console.error("Error getting files", error);diff --git a/frontend/app/knowledge/chunks/page.tsx b/frontend/app/knowledge/chunks/page.tsxindex a125620e8..9b3122bab 100644--- a/frontend/app/knowledge/chunks/page.tsx+++ b/frontend/app/knowledge/chunks/page.tsx@@ -317,31 +317,11 @@ function ChunksPageContent() {</dl></div>{(() => {- // #region agent logconst hasOwner = Boolean(fileData?.owner);const hasAllowedUsers = (fileData?.allowed_users?.length ?? 0) > 0;const hasAllowedGroups = (fileData?.allowed_groups?.length ?? 0) > 0;- const showAccessControl = hasOwner || hasAllowedUsers || hasAllowedGroups;- if (fileData && chunks.length > 0) {- fetch("http://127.0.0.1:7242/ingest/0459c69b-01d9-4ed6-bcff-99a7ce0d82ee", {- method: "POST",- headers: { "Content-Type": "application/json" },- body: JSON.stringify({- hypothesisId: "H3",- location: "chunks/page.tsx:AccessControl",- message: "chunks page fileData ACL",- data: {- filename: fileData.filename,- owner: fileData.owner || null,- allowed_users_len: (fileData.allowed_users || []).length,- allowed_groups_len: (fileData.allowed_groups || []).length,- showAccessControl,- },- timestamp: Date.now(),- }),- }).catch(() => {});- }- // #endregion+ const showAccessControl =+ hasOwner || hasAllowedUsers || hasAllowedGroups;return showAccessControl;})() ? (<div className="mb-4">diff --git a/src/connectors/service.py b/src/connectors/service.pyindex 3830e908d..2bc6e9986 100644--- a/src/connectors/service.py+++ b/src/connectors/service.py@@ -85,17 +85,6 @@ class ConnectorService:logger.debug("Document processing result", result=result)- # #region agent log- try:- import json as _json- _log_path = "/Users/edwin.jose/Documents/openrag/.cursor/debug.log"- _acl = document.acl- with open(_log_path, "a") as _f:- _f.write(_json.dumps({"hypothesisId": "H9", "location": "service.py:process_connector_document", "message": "post process_document_standard", "data": {"document_id": document.id, "filename": document.filename, "result_status": result.get("status"), "has_acl": _acl is not None, "acl_owner": _acl.owner if _acl else None, "acl_allowed_users": _acl.allowed_users if _acl else None, "acl_allowed_groups": _acl.allowed_groups if _acl else None}, "timestamp": __import__("time").time() * 1000}) + "\n")- except Exception:- pass- # #endregion-# If successfully indexed or already exists, update the indexed documents with connector metadataif result["status"] in ["indexed", "unchanged"]:# Update all chunks with connector-specific metadatadiff --git a/src/connectors/sharepoint/connector.py b/src/connectors/sharepoint/connector.pyindex 4ce8fb2a3..6900b64c4 100644--- a/src/connectors/sharepoint/connector.py+++ b/src/connectors/sharepoint/connector.py@@ -481,16 +481,6 @@ class SharePointConnector(BaseConnector):# Fallback to user drivepermissions_url = f"{self._graph_base_url}/me/drive/items/{file_id}/permissions"- # #region agent log- try:- import json as _json- _log_path = "/Users/edwin.jose/Documents/openrag/.cursor/debug.log"- with open(_log_path, "a") as _f:- _f.write(_json.dumps({"hypothesisId": "H6", "location": "connector.py:_extract_sharepoint_acl", "message": "ACL extraction attempt", "data": {"file_id": file_id, "permissions_url": permissions_url, "has_token": bool(access_token), "token_type": type(access_token).__name__}, "timestamp": __import__("time").time() * 1000}) + "\n")- except Exception:- pass- # #endregion-# Fetch permissionsasync with httpx.AsyncClient() as client:response = await client.get(@@ -498,17 +488,6 @@ class SharePointConnector(BaseConnector):headers={"Authorization": f"Bearer {access_token}"})- # #region agent log- try:- import json as _json- _log_path = "/Users/edwin.jose/Documents/openrag/.cursor/debug.log"- _resp_preview = response.text[:500] if response.text else ""- with open(_log_path, "a") as _f:- _f.write(_json.dumps({"hypothesisId": "H6", "location": "connector.py:_extract_sharepoint_acl:response", "message": "permissions API response", "data": {"file_id": file_id, "status_code": response.status_code, "response_preview": _resp_preview}, "timestamp": __import__("time").time() * 1000}) + "\n")- except Exception:- pass- # #endregion-if response.status_code != 200:logger.warning(f"Failed to fetch permissions for {file_id}: {response.status_code}")return DocumentACL()@@ -557,16 +536,6 @@ class SharePointConnector(BaseConnector):if group_id or group_display_name:allowed_groups.append(group_display_name or group_id)- # #region agent log- try:- import json as _json- _log_path = "/Users/edwin.jose/Documents/openrag/.cursor/debug.log"- with open(_log_path, "a") as _f:- _f.write(_json.dumps({"hypothesisId": "H6", "location": "connector.py:_extract_sharepoint_acl:result", "message": "ACL extraction result", "data": {"file_id": file_id, "owner": owner, "allowed_users": allowed_users, "allowed_groups": allowed_groups, "perm_count": len(permissions_data.get("value", []))}, "timestamp": __import__("time").time() * 1000}) + "\n")- except Exception:- pass- # #endregion-return DocumentACL(owner=owner,allowed_users=allowed_users,@@ -575,15 +544,6 @@ class SharePointConnector(BaseConnector):except Exception as e:logger.warning(f"Failed to extract ACL for SharePoint item {file_id}: {e}")- # #region agent log- try:- import json as _json, traceback as _tb- _log_path = "/Users/edwin.jose/Documents/openrag/.cursor/debug.log"- with open(_log_path, "a") as _f:- _f.write(_json.dumps({"hypothesisId": "H7", "location": "connector.py:_extract_sharepoint_acl:exception", "message": "ACL extraction failed", "data": {"file_id": file_id, "error": str(e), "traceback": _tb.format_exc()}, "timestamp": __import__("time").time() * 1000}) + "\n")- except Exception:- pass- # #endregionreturn DocumentACL()async def get_file_content(self, file_id: str) -> ConnectorDocument:diff --git a/src/services/search_service.py b/src/services/search_service.pyindex aabe8d2d9..9ef240acc 100644--- a/src/services/search_service.py+++ b/src/services/search_service.py@@ -458,20 +458,6 @@ class SearchService:})- # #region agent log- try:- _log_path = "/Users/edwin.jose/Documents/openrag/.cursor/debug.log"- _samples = []- for c in chunks[:5]:- _au = c.get("allowed_users") or []- _ag = c.get("allowed_groups") or []- _samples.append({"filename": c.get("filename"), "owner": bool(c.get("owner")), "allowed_users_len": len(_au) if isinstance(_au, list) else 0, "allowed_groups_len": len(_ag) if isinstance(_ag, list) else 0})- with open(_log_path, "a") as _f:- _f.write(json.dumps({"hypothesisId": "H1", "location": "search_service.py:chunks", "message": "backend chunk ACL sample", "data": {"chunk_samples": _samples}, "timestamp": __import__("time").time() * 1000}) + "\n")- except Exception:- pass- # #endregion-# Return both transformed results and aggregationsreturn {"results": chunks,diff --git a/src/utils/acl_utils.py b/src/utils/acl_utils.pyindex 4a1c4a541..d8806206e 100644--- a/src/utils/acl_utils.py+++ b/src/utils/acl_utils.py@@ -109,29 +109,9 @@ async def update_document_acl(Returns:Dict with status ("unchanged" or "updated") and chunks_updated count"""- # #region agent log- try:- import json as _json- _log_path = "/Users/edwin.jose/Documents/openrag/.cursor/debug.log"- with open(_log_path, "a") as _f:- _f.write(_json.dumps({"hypothesisId": "H9", "location": "acl_utils.py:update_document_acl:entry", "message": "update_document_acl called", "data": {"document_id": document_id, "acl_owner": acl.owner if acl else None, "acl_allowed_users": acl.allowed_users if acl else None, "acl_allowed_groups": acl.allowed_groups if acl else None}, "timestamp": __import__("time").time() * 1000}) + "\n")- except Exception:- pass- # #endregion-# Check if ACL changed (queries one chunk)should_update = await should_update_acl(document_id, acl, opensearch_client)- # #region agent log- try:- import json as _json- _log_path = "/Users/edwin.jose/Documents/openrag/.cursor/debug.log"- with open(_log_path, "a") as _f:- _f.write(_json.dumps({"hypothesisId": "H10", "location": "acl_utils.py:update_document_acl:should_update", "message": "should_update_acl result", "data": {"document_id": document_id, "should_update": should_update}, "timestamp": __import__("time").time() * 1000}) + "\n")- except Exception:- pass- # #endregion-if not should_update:return {"status": "unchanged", "chunks_updated": 0}@@ -156,31 +136,12 @@ async def update_document_acl(})- # #region agent log- try:- import json as _json- _log_path = "/Users/edwin.jose/Documents/openrag/.cursor/debug.log"- with open(_log_path, "a") as _f:- _f.write(_json.dumps({"hypothesisId": "H11", "location": "acl_utils.py:update_document_acl:result", "message": "update_by_query result", "data": {"document_id": document_id, "updated": response.get("updated", 0), "total": response.get("total", 0), "response_keys": list(response.keys())}, "timestamp": __import__("time").time() * 1000}) + "\n")- except Exception:- pass- # #endregion-return {"status": "updated","chunks_updated": response.get("updated", 0)}except Exception as e:- # #region agent log- try:- import json as _json, traceback as _tb- _log_path = "/Users/edwin.jose/Documents/openrag/.cursor/debug.log"- with open(_log_path, "a") as _f:- _f.write(_json.dumps({"hypothesisId": "H11", "location": "acl_utils.py:update_document_acl:error", "message": "update_by_query failed", "data": {"document_id": document_id, "error": str(e), "traceback": _tb.format_exc()}, "timestamp": __import__("time").time() * 1000}) + "\n")- except Exception:- pass- # #endregionprint(f"Error updating ACL for {document_id}: {e}")return {"status": "error", "chunks_updated": 0, "error": str(e)}