Skip to content

BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.

feat(ingestion): structured ingestion

Christian233e815b2509
diff --git a/Makefile b/Makefile
index cb9dc03..311627e 100644
--- a/Makefile
+++ b/Makefile
@@ -81,7 +81,7 @@ clear-all-volumes:
make delete-neo4j-volumes
make delete-mongo-volumes
-VERSION := $(shell poetry version -s)
+VERSION := $(shell grep '^version = ' pyproject.toml | sed 's/version = "\(.*\)"/\1/')
container-build:
docker build --no-cache --platform linux/amd64 -t ghcr.io/lumen-labs/brainapi:v$(VERSION) ./
docker tag ghcr.io/lumen-labs/brainapi:v$(VERSION) ghcr.io/lumen-labs/brainapi:latest
@@ -91,16 +91,15 @@ container-push:
docker push ghcr.io/lumen-labs/brainapi:latest
container-release:
- VERSION=$$(poetry version -s)
BUILD_DATE=$$(date -u +'%Y-%m-%dT%H:%M:%SZ')
BUILD_SHA=$$(git rev-parse HEAD || echo "unknown")
CACHE_BUST=$$(date +%s)
- docker build --no-cache --build-arg BUILD_DATE="$$BUILD_DATE" --build-arg BUILD_SHA="$$BUILD_SHA" --build-arg CACHE_BUST="$$CACHE_BUST" --platform linux/amd64 -t ghcr.io/lumen-labs/brainapi:v$$VERSION ./
- docker tag ghcr.io/lumen-labs/brainapi:v$$VERSION ghcr.io/lumen-labs/brainapi:latest
- git tag -s v$$VERSION -m "Release v$$VERSION"
- docker push ghcr.io/lumen-labs/brainapi:v$$VERSION
+ docker build --no-cache --build-arg BUILD_DATE="$$BUILD_DATE" --build-arg BUILD_SHA="$$BUILD_SHA" --build-arg CACHE_BUST="$$CACHE_BUST" --platform linux/amd64 -t ghcr.io/lumen-labs/brainapi:v$(VERSION) ./
+ docker tag ghcr.io/lumen-labs/brainapi:v$(VERSION) ghcr.io/lumen-labs/brainapi:latest
+ git tag -s v$(VERSION) -m "Release v$(VERSION)"
+ docker push ghcr.io/lumen-labs/brainapi:v$(VERSION)
docker push ghcr.io/lumen-labs/brainapi:latest
- git push origin v$$VERSION
+ git push origin v$(VERSION)
v-patch:
poetry version patch
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..e69de29
diff --git a/poetry.lock b/poetry.lock
index 6defc31..62512ee 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1458,31 +1458,124 @@ pytest = ["pytest (>=7.0.0)", "rich (>=13.9.4)", "vcrpy (>=7.0.0)"]
vcr = ["vcrpy (>=7.0.0)"]
[[package]]
-name = "mccabe"
-version = "0.7.0"
-description = "McCabe checker, plugin for flake8"
+name = "levenshtein"
+version = "0.27.1"
+description = "Python extension for computing string edit distances and similarities."
optional = false
-python-versions = ">=3.6"
+python-versions = ">=3.9"
groups = ["main"]
files = [
- {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"},
- {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"},
+ {file = "levenshtein-0.27.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:13d6f617cb6fe63714c4794861cfaacd398db58a292f930edb7f12aad931dace"},
+ {file = "levenshtein-0.27.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ca9d54d41075e130c390e61360bec80f116b62d6ae973aec502e77e921e95334"},
+ {file = "levenshtein-0.27.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2de1f822b5c9a20d10411f779dfd7181ce3407261436f8470008a98276a9d07f"},
+ {file = "levenshtein-0.27.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:81270392c2e45d1a7e1b3047c3a272d5e28bb4f1eff0137637980064948929b7"},
+ {file = "levenshtein-0.27.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2d30c3ea23a94dddd56dbe323e1fa8a29ceb24da18e2daa8d0abf78b269a5ad1"},
+ {file = "levenshtein-0.27.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f3e0bea76695b9045bbf9ad5f67ad4cc01c11f783368f34760e068f19b6a6bc"},
+ {file = "levenshtein-0.27.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cdd190e468a68c31a5943368a5eaf4e130256a8707886d23ab5906a0cb98a43c"},
+ {file = "levenshtein-0.27.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:7c3121314bb4b676c011c33f6a0ebb462cfdcf378ff383e6f9e4cca5618d0ba7"},
+ {file = "levenshtein-0.27.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:f8ef378c873efcc5e978026b69b45342d841cd7a2f273447324f1c687cc4dc37"},
+ {file = "levenshtein-0.27.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:ff18d78c5c16bea20876425e1bf5af56c25918fb01bc0f2532db1317d4c0e157"},
+ {file = "levenshtein-0.27.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:13412ff805afbfe619d070280d1a76eb4198c60c5445cd5478bd4c7055bb3d51"},
+ {file = "levenshtein-0.27.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a2adb9f263557f7fb13e19eb2f34595d86929a44c250b2fca6e9b65971e51e20"},
+ {file = "levenshtein-0.27.1-cp310-cp310-win32.whl", hash = "sha256:6278a33d2e0e909d8829b5a72191419c86dd3bb45b82399c7efc53dabe870c35"},
+ {file = "levenshtein-0.27.1-cp310-cp310-win_amd64.whl", hash = "sha256:5b602b8428ee5dc88432a55c5303a739ee2be7c15175bd67c29476a9d942f48e"},
+ {file = "levenshtein-0.27.1-cp310-cp310-win_arm64.whl", hash = "sha256:48334081fddaa0c259ba01ee898640a2cf8ede62e5f7e25fefece1c64d34837f"},
+ {file = "levenshtein-0.27.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2e6f1760108319a108dceb2f02bc7cdb78807ad1f9c673c95eaa1d0fe5dfcaae"},
+ {file = "levenshtein-0.27.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c4ed8400d94ab348099395e050b8ed9dd6a5d6b5b9e75e78b2b3d0b5f5b10f38"},
+ {file = "levenshtein-0.27.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7826efe51be8ff58bc44a633e022fdd4b9fc07396375a6dbc4945a3bffc7bf8f"},
+ {file = "levenshtein-0.27.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ff5afb78719659d353055863c7cb31599fbea6865c0890b2d840ee40214b3ddb"},
+ {file = "levenshtein-0.27.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:201dafd5c004cd52018560cf3213da799534d130cf0e4db839b51f3f06771de0"},
+ {file = "levenshtein-0.27.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e5ddd59f3cfaec216811ee67544779d9e2d6ed33f79337492a248245d6379e3d"},
+ {file = "levenshtein-0.27.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6afc241d27ecf5b921063b796812c55b0115423ca6fa4827aa4b1581643d0a65"},
+ {file = "levenshtein-0.27.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ee2e766277cceb8ca9e584ea03b8dc064449ba588d3e24c1923e4b07576db574"},
+ {file = "levenshtein-0.27.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:920b23d6109453913ce78ec451bc402ff19d020ee8be4722e9d11192ec2fac6f"},
+ {file = "levenshtein-0.27.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:560d7edba126e2eea3ac3f2f12e7bd8bc9c6904089d12b5b23b6dfa98810b209"},
+ {file = "levenshtein-0.27.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:8d5362b6c7aa4896dc0cb1e7470a4ad3c06124e0af055dda30d81d3c5549346b"},
+ {file = "levenshtein-0.27.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:65ba880815b0f80a80a293aeebac0fab8069d03ad2d6f967a886063458f9d7a1"},
+ {file = "levenshtein-0.27.1-cp311-cp311-win32.whl", hash = "sha256:fcc08effe77fec0bc5b0f6f10ff20b9802b961c4a69047b5499f383119ddbe24"},
+ {file = "levenshtein-0.27.1-cp311-cp311-win_amd64.whl", hash = "sha256:0ed402d8902be7df212ac598fc189f9b2d520817fdbc6a05e2ce44f7f3ef6857"},
+ {file = "levenshtein-0.27.1-cp311-cp311-win_arm64.whl", hash = "sha256:7fdaab29af81a8eb981043737f42450efca64b9761ca29385487b29c506da5b5"},
+ {file = "levenshtein-0.27.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:25fb540d8c55d1dc7bdc59b7de518ea5ed9df92eb2077e74bcb9bb6de7b06f69"},
+ {file = "levenshtein-0.27.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f09cfab6387e9c908c7b37961c045e8e10eb9b7ec4a700367f8e080ee803a562"},
+ {file = "levenshtein-0.27.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dafa29c0e616f322b574e0b2aeb5b1ff2f8d9a1a6550f22321f3bd9bb81036e3"},
+ {file = "levenshtein-0.27.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be7a7642ea64392fa1e6ef7968c2e50ef2152c60948f95d0793361ed97cf8a6f"},
+ {file = "levenshtein-0.27.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:060b48c45ed54bcea9582ce79c6365b20a1a7473767e0b3d6be712fa3a22929c"},
+ {file = "levenshtein-0.27.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:712f562c5e64dd0398d3570fe99f8fbb88acec7cc431f101cb66c9d22d74c542"},
+ {file = "levenshtein-0.27.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a6141ad65cab49aa4527a3342d76c30c48adb2393b6cdfeca65caae8d25cb4b8"},
+ {file = "levenshtein-0.27.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:799b8d73cda3265331116f62932f553804eae16c706ceb35aaf16fc2a704791b"},
+ {file = "levenshtein-0.27.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:ec99871d98e517e1cc4a15659c62d6ea63ee5a2d72c5ddbebd7bae8b9e2670c8"},
+ {file = "levenshtein-0.27.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8799164e1f83588dbdde07f728ea80796ea72196ea23484d78d891470241b222"},
+ {file = "levenshtein-0.27.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:583943813898326516ab451a83f734c6f07488cda5c361676150d3e3e8b47927"},
+ {file = "levenshtein-0.27.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5bb22956af44bb4eade93546bf95be610c8939b9a9d4d28b2dfa94abf454fed7"},
+ {file = "levenshtein-0.27.1-cp312-cp312-win32.whl", hash = "sha256:d9099ed1bcfa7ccc5540e8ad27b5dc6f23d16addcbe21fdd82af6440f4ed2b6d"},
+ {file = "levenshtein-0.27.1-cp312-cp312-win_amd64.whl", hash = "sha256:7f071ecdb50aa6c15fd8ae5bcb67e9da46ba1df7bba7c6bf6803a54c7a41fd96"},
+ {file = "levenshtein-0.27.1-cp312-cp312-win_arm64.whl", hash = "sha256:83b9033a984ccace7703f35b688f3907d55490182fd39b33a8e434d7b2e249e6"},
+ {file = "levenshtein-0.27.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ab00c2cae2889166afb7e1af64af2d4e8c1b126f3902d13ef3740df00e54032d"},
+ {file = "levenshtein-0.27.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c27e00bc7527e282f7c437817081df8da4eb7054e7ef9055b851fa3947896560"},
+ {file = "levenshtein-0.27.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5b07de42bfc051136cc8e7f1e7ba2cb73666aa0429930f4218efabfdc5837ad"},
+ {file = "levenshtein-0.27.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fb11ad3c9dae3063405aa50d9c96923722ab17bb606c776b6817d70b51fd7e07"},
+ {file = "levenshtein-0.27.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c5986fb46cb0c063305fd45b0a79924abf2959a6d984bbac2b511d3ab259f3f"},
+ {file = "levenshtein-0.27.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75191e469269ddef2859bc64c4a8cfd6c9e063302766b5cb7e1e67f38cc7051a"},
+ {file = "levenshtein-0.27.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:51b3a7b2266933babc04e4d9821a495142eebd6ef709f90e24bc532b52b81385"},
+ {file = "levenshtein-0.27.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bbac509794afc3e2a9e73284c9e3d0aab5b1d928643f42b172969c3eefa1f2a3"},
+ {file = "levenshtein-0.27.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:8d68714785178347ecb272b94e85cbf7e638165895c4dd17ab57e7742d8872ec"},
+ {file = "levenshtein-0.27.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:8ee74ee31a5ab8f61cd6c6c6e9ade4488dde1285f3c12207afc018393c9b8d14"},
+ {file = "levenshtein-0.27.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f2441b6365453ec89640b85344afd3d602b0d9972840b693508074c613486ce7"},
+ {file = "levenshtein-0.27.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a9be39640a46d8a0f9be729e641651d16a62b2c07d3f4468c36e1cc66b0183b9"},
+ {file = "levenshtein-0.27.1-cp313-cp313-win32.whl", hash = "sha256:a520af67d976761eb6580e7c026a07eb8f74f910f17ce60e98d6e492a1f126c7"},
+ {file = "levenshtein-0.27.1-cp313-cp313-win_amd64.whl", hash = "sha256:7dd60aa49c2d8d23e0ef6452c8329029f5d092f386a177e3385d315cabb78f2a"},
+ {file = "levenshtein-0.27.1-cp313-cp313-win_arm64.whl", hash = "sha256:149cd4f0baf5884ac5df625b7b0d281721b15de00f447080e38f5188106e1167"},
+ {file = "levenshtein-0.27.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0c9231ac7c705a689f12f4fc70286fa698b9c9f06091fcb0daddb245e9259cbe"},
+ {file = "levenshtein-0.27.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:cf9ba080b1a8659d35c11dcfffc7f8c001028c2a3a7b7e6832348cdd60c53329"},
+ {file = "levenshtein-0.27.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:164e3184385caca94ef7da49d373edd7fb52d4253bcc5bd5b780213dae307dfb"},
+ {file = "levenshtein-0.27.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e6024d67de6efbd32aaaafd964864c7fee0569b960556de326c3619d1eeb2ba4"},
+ {file = "levenshtein-0.27.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6fbb234b3b04e04f7b3a2f678e24fd873c86c543d541e9df3ac9ec1cc809e732"},
+ {file = "levenshtein-0.27.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffdd9056c7afb29aea00b85acdb93a3524e43852b934ebb9126c901506d7a1ed"},
+ {file = "levenshtein-0.27.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a1a0918243a313f481f4ba6a61f35767c1230395a187caeecf0be87a7c8f0624"},
+ {file = "levenshtein-0.27.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c57655b20690ffa5168df7f4b7c6207c4ca917b700fb1b142a49749eb1cf37bb"},
+ {file = "levenshtein-0.27.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:079cc78de05d3ded6cf1c5e2c3eadeb1232e12d49be7d5824d66c92b28c3555a"},
+ {file = "levenshtein-0.27.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:ac28c4ced134c0fe2941230ce4fd5c423aa66339e735321665fb9ae970f03a32"},
+ {file = "levenshtein-0.27.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:a2f7688355b22db27588f53c922b4583b8b627c83a8340191bbae1fbbc0f5f56"},
+ {file = "levenshtein-0.27.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:654e8f016cb64ad27263d3364c6536e7644205f20d94748c8b94c586e3362a23"},
+ {file = "levenshtein-0.27.1-cp39-cp39-win32.whl", hash = "sha256:145e6e8744643a3764fed9ab4ab9d3e2b8e5f05d2bcd0ad7df6f22f27a9fbcd4"},
+ {file = "levenshtein-0.27.1-cp39-cp39-win_amd64.whl", hash = "sha256:612f0c90201c318dd113e7e97bd677e6e3e27eb740f242b7ae1a83f13c892b7e"},
+ {file = "levenshtein-0.27.1-cp39-cp39-win_arm64.whl", hash = "sha256:cde09ec5b3cc84a6737113b47e45392b331c136a9e8a8ead8626f3eacae936f8"},
+ {file = "levenshtein-0.27.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:c92a222ab95b8d903eae6d5e7d51fe6c999be021b647715c18d04d0b0880f463"},
+ {file = "levenshtein-0.27.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:71afc36b4ee950fa1140aff22ffda9e5e23280285858e1303260dbb2eabf342d"},
+ {file = "levenshtein-0.27.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58b1daeebfc148a571f09cfe18c16911ea1eaaa9e51065c5f7e7acbc4b866afa"},
+ {file = "levenshtein-0.27.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:105edcb14797d95c77f69bad23104314715a64cafbf4b0e79d354a33d7b54d8d"},
+ {file = "levenshtein-0.27.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d9c58fb1ef8bdc8773d705fbacf628e12c3bb63ee4d065dda18a76e86042444a"},
+ {file = "levenshtein-0.27.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:e52270591854af67217103955a36bd7436b57c801e3354e73ba44d689ed93697"},
+ {file = "levenshtein-0.27.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:909b7b6bce27a4ec90576c9a9bd9af5a41308dfecf364b410e80b58038277bbe"},
+ {file = "levenshtein-0.27.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d193a7f97b8c6a350e36ec58e41a627c06fa4157c3ce4b2b11d90cfc3c2ebb8f"},
+ {file = "levenshtein-0.27.1-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:614be316e3c06118705fae1f717f9072d35108e5fd4e66a7dd0e80356135340b"},
+ {file = "levenshtein-0.27.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31fc0a5bb070722bdabb6f7e14955a294a4a968c68202d294699817f21545d22"},
+ {file = "levenshtein-0.27.1-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9415aa5257227af543be65768a80c7a75e266c3c818468ce6914812f88f9c3df"},
+ {file = "levenshtein-0.27.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:7987ef006a3cf56a4532bd4c90c2d3b7b4ca9ad3bf8ae1ee5713c4a3bdfda913"},
+ {file = "levenshtein-0.27.1-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:e67750653459a8567b5bb10e56e7069b83428d42ff5f306be821ef033b92d1a8"},
+ {file = "levenshtein-0.27.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:93344c2c3812f21fdc46bd9e57171684fc53dd107dae2f648d65ea6225d5ceaf"},
+ {file = "levenshtein-0.27.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da4baef7e7460691006dd2ca6b9e371aecf135130f72fddfe1620ae740b68d94"},
+ {file = "levenshtein-0.27.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8141c8e5bf2bd76ae214c348ba382045d7ed9d0e7ce060a36fc59c6af4b41d48"},
+ {file = "levenshtein-0.27.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:773aa120be48c71e25c08d92a2108786e6537a24081049664463715926c76b86"},
+ {file = "levenshtein-0.27.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:f12a99138fb09eb5606ab9de61dd234dd82a7babba8f227b5dce0e3ae3a9eaf4"},
+ {file = "levenshtein-0.27.1.tar.gz", hash = "sha256:3e18b73564cfc846eec94dd13fab6cb006b5d2e0cc56bad1fd7d5585881302e3"},
]
+[package.dependencies]
+rapidfuzz = ">=3.9.0,<4.0.0"
+
[[package]]
-name = "milvus"
-version = "2.3.9"
-description = "Embeded Milvus"
+name = "mccabe"
+version = "0.7.0"
+description = "McCabe checker, plugin for flake8"
optional = false
python-versions = ">=3.6"
groups = ["main"]
files = [
- {file = "milvus-2.3.9-py3-none-macosx_12_0_arm64.whl", hash = "sha256:2e770b1cb76d8f4ba0e3ce1c2e49bdb9cd38222f8662762db9d794e577dac826"},
+ {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"},
+ {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"},
]
-[package.extras]
-client = ["pymilvus (>=2.3.0b1,<2.4.0)"]
-
[[package]]
name = "neo4j"
version = "6.0.2"
@@ -2548,6 +2641,111 @@ files = [
{file = "pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f"},
]
+[[package]]
+name = "rapidfuzz"
+version = "3.14.1"
+description = "rapid fuzzy string matching"
+optional = false
+python-versions = ">=3.10"
+groups = ["main"]
+files = [
+ {file = "rapidfuzz-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:489440e4b5eea0d150a31076eb183bed0ec84f934df206c72ae4fc3424501758"},
+ {file = "rapidfuzz-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:eff22cc938c3f74d194df03790a6c3325d213b28cf65cdefd6fdeae759b745d5"},
+ {file = "rapidfuzz-3.14.1-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e0307f018b16feaa36074bcec2496f6f120af151a098910296e72e233232a62f"},
+ {file = "rapidfuzz-3.14.1-cp310-cp310-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bc133652da143aca1ab72de235446432888b2b7f44ee332d006f8207967ecb8a"},
+ {file = "rapidfuzz-3.14.1-cp310-cp310-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e9e71b3fe7e4a1590843389a90fe2a8684649fc74b9b7446e17ee504ddddb7de"},
+ {file = "rapidfuzz-3.14.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c51519eb2f20b52eba6fc7d857ae94acc6c2a1f5d0f2d794b9d4977cdc29dd7"},
+ {file = "rapidfuzz-3.14.1-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:fe87d94602624f8f25fff9a0a7b47f33756c4d9fc32b6d3308bb142aa483b8a4"},
+ {file = "rapidfuzz-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2d665380503a575dda52eb712ea521f789e8f8fd629c7a8e6c0f8ff480febc78"},
+ {file = "rapidfuzz-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0f0dd022b8a7cbf3c891f6de96a80ab6a426f1069a085327816cea749e096c2"},
+ {file = "rapidfuzz-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:bf1ba22d36858b265c95cd774ba7fe8991e80a99cd86fe4f388605b01aee81a3"},
+ {file = "rapidfuzz-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:ca1c1494ac9f9386d37f0e50cbaf4d07d184903aed7691549df1b37e9616edc9"},
+ {file = "rapidfuzz-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9e4b12e921b0fa90d7c2248742a536f21eae5562174090b83edd0b4ab8b557d7"},
+ {file = "rapidfuzz-3.14.1-cp310-cp310-win32.whl", hash = "sha256:5e1c1f2292baa4049535b07e9e81feb29e3650d2ba35ee491e64aca7ae4cb15e"},
+ {file = "rapidfuzz-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:59a8694beb9a13c4090ab3d1712cabbd896c6949706d1364e2a2e1713c413760"},
+ {file = "rapidfuzz-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:e94cee93faa792572c574a615abe12912124b4ffcf55876b72312914ab663345"},
+ {file = "rapidfuzz-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4d976701060886a791c8a9260b1d4139d14c1f1e9a6ab6116b45a1acf3baff67"},
+ {file = "rapidfuzz-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5e6ba7e6eb2ab03870dcab441d707513db0b4264c12fba7b703e90e8b4296df2"},
+ {file = "rapidfuzz-3.14.1-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e532bf46de5fd3a1efde73a16a4d231d011bce401c72abe3c6ecf9de681003f"},
+ {file = "rapidfuzz-3.14.1-cp311-cp311-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f9b6a6fb8ed9b951e5f3b82c1ce6b1665308ec1a0da87f799b16e24fc59e4662"},
+ {file = "rapidfuzz-3.14.1-cp311-cp311-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5b6ac3f9810949caef0e63380b11a3c32a92f26bacb9ced5e32c33560fcdf8d1"},
+ {file = "rapidfuzz-3.14.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e52e4c34fd567f77513e886b66029c1ae02f094380d10eba18ba1c68a46d8b90"},
+ {file = "rapidfuzz-3.14.1-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:2ef72e41b1a110149f25b14637f1cedea6df192462120bea3433980fe9d8ac05"},
+ {file = "rapidfuzz-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fb654a35b373d712a6b0aa2a496b2b5cdd9d32410cfbaecc402d7424a90ba72a"},
+ {file = "rapidfuzz-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:2b2c12e5b9eb8fe9a51b92fe69e9ca362c0970e960268188a6d295e1dec91e6d"},
+ {file = "rapidfuzz-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4f069dec5c450bd987481e752f0a9979e8fdf8e21e5307f5058f5c4bb162fa56"},
+ {file = "rapidfuzz-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:4d0d9163725b7ad37a8c46988cae9ebab255984db95ad01bf1987ceb9e3058dd"},
+ {file = "rapidfuzz-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db656884b20b213d846f6bc990c053d1f4a60e6d4357f7211775b02092784ca1"},
+ {file = "rapidfuzz-3.14.1-cp311-cp311-win32.whl", hash = "sha256:4b42f7b9c58cbcfbfaddc5a6278b4ca3b6cd8983e7fd6af70ca791dff7105fb9"},
+ {file = "rapidfuzz-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:e5847f30d7d4edefe0cb37294d956d3495dd127c1c56e9128af3c2258a520bb4"},
+ {file = "rapidfuzz-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:5087d8ad453092d80c042a08919b1cb20c8ad6047d772dc9312acd834da00f75"},
+ {file = "rapidfuzz-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:809515194f628004aac1b1b280c3734c5ea0ccbd45938c9c9656a23ae8b8f553"},
+ {file = "rapidfuzz-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0afcf2d6cb633d0d4260d8df6a40de2d9c93e9546e2c6b317ab03f89aa120ad7"},
+ {file = "rapidfuzz-3.14.1-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5c1c3d07d53dcafee10599da8988d2b1f39df236aee501ecbd617bd883454fcd"},
+ {file = "rapidfuzz-3.14.1-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6e9ee3e1eb0a027717ee72fe34dc9ac5b3e58119f1bd8dd15bc19ed54ae3e62b"},
+ {file = "rapidfuzz-3.14.1-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:70c845b64a033a20c44ed26bc890eeb851215148cc3e696499f5f65529afb6cb"},
+ {file = "rapidfuzz-3.14.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26db0e815213d04234298dea0d884d92b9cb8d4ba954cab7cf67a35853128a33"},
+ {file = "rapidfuzz-3.14.1-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:6ad3395a416f8b126ff11c788531f157c7debeb626f9d897c153ff8980da10fb"},
+ {file = "rapidfuzz-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:61c5b9ab6f730e6478aa2def566223712d121c6f69a94c7cc002044799442afd"},
+ {file = "rapidfuzz-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:13e0ea3d0c533969158727d1bb7a08c2cc9a816ab83f8f0dcfde7e38938ce3e6"},
+ {file = "rapidfuzz-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:6325ca435b99f4001aac919ab8922ac464999b100173317defb83eae34e82139"},
+ {file = "rapidfuzz-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:07a9fad3247e68798424bdc116c1094e88ecfabc17b29edf42a777520347648e"},
+ {file = "rapidfuzz-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8ff5dbe78db0a10c1f916368e21d328935896240f71f721e073cf6c4c8cdedd"},
+ {file = "rapidfuzz-3.14.1-cp312-cp312-win32.whl", hash = "sha256:9c83270e44a6ae7a39fc1d7e72a27486bccc1fa5f34e01572b1b90b019e6b566"},
+ {file = "rapidfuzz-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:e06664c7fdb51c708e082df08a6888fce4c5c416d7e3cc2fa66dd80eb76a149d"},
+ {file = "rapidfuzz-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:6c7c26025f7934a169a23dafea6807cfc3fb556f1dd49229faf2171e5d8101cc"},
+ {file = "rapidfuzz-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8d69f470d63ee824132ecd80b1974e1d15dd9df5193916901d7860cef081a260"},
+ {file = "rapidfuzz-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6f571d20152fc4833b7b5e781b36d5e4f31f3b5a596a3d53cf66a1bd4436b4f4"},
+ {file = "rapidfuzz-3.14.1-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61d77e09b2b6bc38228f53b9ea7972a00722a14a6048be9a3672fb5cb08bad3a"},
+ {file = "rapidfuzz-3.14.1-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8b41d95ef86a6295d353dc3bb6c80550665ba2c3bef3a9feab46074d12a9af8f"},
+ {file = "rapidfuzz-3.14.1-cp313-cp313-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0591df2e856ad583644b40a2b99fb522f93543c65e64b771241dda6d1cfdc96b"},
+ {file = "rapidfuzz-3.14.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f277801f55b2f3923ef2de51ab94689a0671a4524bf7b611de979f308a54cd6f"},
+ {file = "rapidfuzz-3.14.1-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:893fdfd4f66ebb67f33da89eb1bd1674b7b30442fdee84db87f6cb9074bf0ce9"},
+ {file = "rapidfuzz-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:fe2651258c1f1afa9b66f44bf82f639d5f83034f9804877a1bbbae2120539ad1"},
+ {file = "rapidfuzz-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:ace21f7a78519d8e889b1240489cd021c5355c496cb151b479b741a4c27f0a25"},
+ {file = "rapidfuzz-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:cb5acf24590bc5e57027283b015950d713f9e4d155fda5cfa71adef3b3a84502"},
+ {file = "rapidfuzz-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:67ea46fa8cc78174bad09d66b9a4b98d3068e85de677e3c71ed931a1de28171f"},
+ {file = "rapidfuzz-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:44e741d785de57d1a7bae03599c1cbc7335d0b060a35e60c44c382566e22782e"},
+ {file = "rapidfuzz-3.14.1-cp313-cp313-win32.whl", hash = "sha256:b1fe6001baa9fa36bcb565e24e88830718f6c90896b91ceffcb48881e3adddbc"},
+ {file = "rapidfuzz-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:83b8cc6336709fa5db0579189bfd125df280a554af544b2dc1c7da9cdad7e44d"},
+ {file = "rapidfuzz-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:cf75769662eadf5f9bd24e865c19e5ca7718e879273dce4e7b3b5824c4da0eb4"},
+ {file = "rapidfuzz-3.14.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:d937dbeda71c921ef6537c6d41a84f1b8112f107589c9977059de57a1d726dd6"},
+ {file = "rapidfuzz-3.14.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:7a2d80cc1a4fcc7e259ed4f505e70b36433a63fa251f1bb69ff279fe376c5efd"},
+ {file = "rapidfuzz-3.14.1-cp313-cp313t-win32.whl", hash = "sha256:40875e0c06f1a388f1cab3885744f847b557e0b1642dfc31ff02039f9f0823ef"},
+ {file = "rapidfuzz-3.14.1-cp313-cp313t-win_amd64.whl", hash = "sha256:876dc0c15552f3d704d7fb8d61bdffc872ff63bedf683568d6faad32e51bbce8"},
+ {file = "rapidfuzz-3.14.1-cp313-cp313t-win_arm64.whl", hash = "sha256:61458e83b0b3e2abc3391d0953c47d6325e506ba44d6a25c869c4401b3bc222c"},
+ {file = "rapidfuzz-3.14.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e84d9a844dc2e4d5c4cabd14c096374ead006583304333c14a6fbde51f612a44"},
+ {file = "rapidfuzz-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:40301b93b99350edcd02dbb22e37ca5f2a75d0db822e9b3c522da451a93d6f27"},
+ {file = "rapidfuzz-3.14.1-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fedd5097a44808dddf341466866e5c57a18a19a336565b4ff50aa8f09eb528f6"},
+ {file = "rapidfuzz-3.14.1-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e3e61c9e80d8c26709d8aa5c51fdd25139c81a4ab463895f8a567f8347b0548"},
+ {file = "rapidfuzz-3.14.1-cp314-cp314-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da011a373722fac6e64687297a1d17dc8461b82cb12c437845d5a5b161bc24b9"},
+ {file = "rapidfuzz-3.14.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5967d571243cfb9ad3710e6e628ab68c421a237b76e24a67ac22ee0ff12784d6"},
+ {file = "rapidfuzz-3.14.1-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:474f416cbb9099676de54aa41944c154ba8d25033ee460f87bb23e54af6d01c9"},
+ {file = "rapidfuzz-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ae2d57464b59297f727c4e201ea99ec7b13935f1f056c753e8103da3f2fc2404"},
+ {file = "rapidfuzz-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:57047493a1f62f11354c7143c380b02f1b355c52733e6b03adb1cb0fe8fb8816"},
+ {file = "rapidfuzz-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4acc20776f225ee37d69517a237c090b9fa7e0836a0b8bc58868e9168ba6ef6f"},
+ {file = "rapidfuzz-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:4373f914ff524ee0146919dea96a40a8200ab157e5a15e777a74a769f73d8a4a"},
+ {file = "rapidfuzz-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:37017b84953927807847016620d61251fe236bd4bcb25e27b6133d955bb9cafb"},
+ {file = "rapidfuzz-3.14.1-cp314-cp314-win32.whl", hash = "sha256:c8d1dd1146539e093b84d0805e8951475644af794ace81d957ca612e3eb31598"},
+ {file = "rapidfuzz-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:f51c7571295ea97387bac4f048d73cecce51222be78ed808263b45c79c40a440"},
+ {file = "rapidfuzz-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:01eab10ec90912d7d28b3f08f6c91adbaf93458a53f849ff70776ecd70dd7a7a"},
+ {file = "rapidfuzz-3.14.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:60879fcae2f7618403c4c746a9a3eec89327d73148fb6e89a933b78442ff0669"},
+ {file = "rapidfuzz-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f94d61e44db3fc95a74006a394257af90fa6e826c900a501d749979ff495d702"},
+ {file = "rapidfuzz-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:93b6294a3ffab32a9b5f9b5ca048fa0474998e7e8bb0f2d2b5e819c64cb71ec7"},
+ {file = "rapidfuzz-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:6cb56b695421538fdbe2c0c85888b991d833b8637d2f2b41faa79cea7234c000"},
+ {file = "rapidfuzz-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:7cd312c380d3ce9d35c3ec9726b75eee9da50e8a38e89e229a03db2262d3d96b"},
+ {file = "rapidfuzz-3.14.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:673ce55a9be5b772dade911909e42382c0828b8a50ed7f9168763fa6b9f7054d"},
+ {file = "rapidfuzz-3.14.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:45c62ada1980ebf4c64c4253993cc8daa018c63163f91db63bb3af69cb74c2e3"},
+ {file = "rapidfuzz-3.14.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:4d51efb29c0df0d4f7f64f672a7624c2146527f0745e3572098d753676538800"},
+ {file = "rapidfuzz-3.14.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:4a21ccdf1bd7d57a1009030527ba8fae1c74bf832d0a08f6b67de8f5c506c96f"},
+ {file = "rapidfuzz-3.14.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:589fb0af91d3aff318750539c832ea1100dbac2c842fde24e42261df443845f6"},
+ {file = "rapidfuzz-3.14.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:a4f18092db4825f2517d135445015b40033ed809a41754918a03ef062abe88a0"},
+ {file = "rapidfuzz-3.14.1.tar.gz", hash = "sha256:b02850e7f7152bd1edff27e9d584505b84968cacedee7a734ec4050c655a803c"},
+]
+
+[package.extras]
+all = ["numpy"]
+
[[package]]
name = "redis"
version = "6.4.0"
@@ -3560,4 +3758,4 @@ cffi = ["cffi (>=1.17,<2.0) ; platform_python_implementation != \"PyPy\" and pyt
[metadata]
lock-version = "2.1"
python-versions = ">=3.11,<4.0.0"
-content-hash = "a7d1ddad4410905fd6f3d8b9a364aa18395c1f54685d8fb76ad7ba45479d37cf"
+content-hash = "b8f5bd4625042937e907cb79df3a8f1af12e67dcb3dcbb5e52c9afe1b4c4003e"
diff --git a/pyproject.toml b/pyproject.toml
index caaec02..06ff1ca 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "brainapi2"
-version = "0.1.3"
+version = "1.0.0-dev"
description = "Version 1.x.x of the BrainAPI memory layer."
authors = [
{name = "Christian",email = "[email protected]"}
@@ -24,7 +24,8 @@ dependencies = [
"langchain-openai (>=1.0.0,<2.0.0)",
"google-genai (>=1.46.0,<2.0.0)",
"langchain-google-vertexai (>=3.0.1,<4.0.0)",
- "pymilvus (>=2.6.2,<3.0.0)"
+ "pymilvus (>=2.6.2,<3.0.0)",
+ "levenshtein (>=0.27.1,<0.28.0)"
]
[tool.poetry]
diff --git a/src/adapters/data.py b/src/adapters/data.py
index 1acc111..979d2b5 100644
--- a/src/adapters/data.py
+++ b/src/adapters/data.py
@@ -10,7 +10,7 @@ Modified By: the developer formerly known as Christian Nonis at <alch.infoemail@
from typing import List, Tuple
from src.adapters.interfaces.data import DataClient, SearchResult
-from src.constants.data import Observation, TextChunk
+from src.constants.data import Observation, StructuredData, TextChunk
class DataAdapter:
@@ -52,3 +52,9 @@ class DataAdapter:
Get data by their IDs.
"""
return self.data.get_text_chunks_by_ids(ids, with_observations)
+
+ def save_structured_data(self, structured_data: StructuredData) -> StructuredData:
+ """
+ Save a structured data to the data client.
+ """
+ return self.data.save_structured_data(structured_data)
diff --git a/src/adapters/graph.py b/src/adapters/graph.py
index ed16ae6..9e7f184 100644
--- a/src/adapters/graph.py
+++ b/src/adapters/graph.py
@@ -58,11 +58,12 @@ class GraphAdapter:
nodes: list[Node],
identification_params: Optional[dict],
metadata: Optional[dict],
+ database: Optional[str] = None,
) -> list[Node] | str:
"""
Add nodes to the graph.
"""
- return self.graph.add_nodes(nodes, identification_params, metadata)
+ return self.graph.add_nodes(nodes, identification_params, metadata, database)
def add_relationship(
self,
@@ -97,7 +98,7 @@ class GraphAdapter:
relationships_depth: Optional[int] = 1,
relationships_type: Optional[list[str]] = None,
preferred_labels: Optional[list[str]] = None,
- ) -> list[Node]:
+ ) -> list[dict]:
"""
Get nodes by their UUIDs with optional relationships and preferred labels.
"""
@@ -115,5 +116,17 @@ class GraphAdapter:
"""
return self.graph.get_graph_entities()
+ def get_by_uuid(self, uuid: str) -> Node:
+ """
+ Get a node by its UUID.
+ """
+ return self.graph.get_by_uuid(uuid)
+
+ def get_by_uuids(self, uuids: list[str]) -> list[Node]:
+ """
+ Get nodes by their UUIDs.
+ """
+ return self.graph.get_by_uuids(uuids)
+
_graph_adapter = GraphAdapter()
diff --git a/src/adapters/interfaces/data.py b/src/adapters/interfaces/data.py
index 6f426f0..83b6481 100644
--- a/src/adapters/interfaces/data.py
+++ b/src/adapters/interfaces/data.py
@@ -13,7 +13,7 @@ from typing import List, Tuple
from pydantic import BaseModel
-from src.constants.data import Observation, TextChunk
+from src.constants.data import Observation, StructuredData, TextChunk
class SearchResult(BaseModel):
@@ -59,3 +59,10 @@ class DataClient(ABC):
Get data by their IDs.
"""
raise NotImplementedError("get_by_ids method not implemented")
+
+ @abstractmethod
+ def save_structured_data(self, structured_data: StructuredData) -> StructuredData:
+ """
+ Save a structured data to the data client.
+ """
+ raise NotImplementedError("save_structured_data method not implemented")
diff --git a/src/adapters/interfaces/graph.py b/src/adapters/interfaces/graph.py
index 3ae7e6e..642b929 100644
--- a/src/adapters/interfaces/graph.py
+++ b/src/adapters/interfaces/graph.py
@@ -48,6 +48,7 @@ class GraphClient(ABC):
nodes: list[Node],
identification_params: Optional[dict],
metadata: Optional[dict],
+ database: Optional[str] = None,
) -> list[Node] | str:
"""
Add nodes to the graph.
@@ -91,7 +92,7 @@ class GraphClient(ABC):
relationships_depth: Optional[int] = 1,
relationships_type: Optional[list[str]] = None,
preferred_labels: Optional[list[str]] = None,
- ) -> list[Node]:
+ ) -> list[dict]:
"""
Get nodes by their UUIDs with optional relationships and preferred labels.
"""
@@ -103,3 +104,23 @@ class GraphClient(ABC):
Get the entities of the graph.
"""
raise NotImplementedError("get_graph_entities method not implemented")
+
+ @abstractmethod
+ def get_by_uuid(
+ self,
+ uuid: str,
+ ) -> Node:
+ """
+ Get a node by its UUID.
+ """
+ raise NotImplementedError("get_by_uuid method not implemented")
+
+ @abstractmethod
+ def get_by_uuids(
+ self,
+ uuids: list[str],
+ ) -> list[Node]:
+ """
+ Get nodes by their UUIDs.
+ """
+ raise NotImplementedError("get_by_uuids method not implemented")
diff --git a/src/constants/data.py b/src/constants/data.py
index 9066f8c..57e622b 100644
--- a/src/constants/data.py
+++ b/src/constants/data.py
@@ -9,7 +9,7 @@ Modified By: the developer formerly known as Christian Nonis at <alch.infoemail@
"""
from datetime import datetime
-from typing import Optional
+from typing import List, Optional
import uuid
from pydantic import BaseModel, Field
@@ -28,6 +28,23 @@ class TextChunk(BaseModel):
)
+class StructuredData(BaseModel):
+ """
+ Structured data model.
+ """
+
+ id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+ data: dict = Field(description="The json data rapresenting the structured element.")
+ types: List[str] = Field(
+ description="A list of types, used to categorize the data."
+ )
+ metadata: Optional[dict] = None
+ inserted_at: datetime = Field(
+ default_factory=datetime.now,
+ description="The date and time the structured data was inserted.",
+ )
+
+
class Observation(BaseModel):
"""
Observation model.
@@ -43,3 +60,24 @@ class Observation(BaseModel):
default_factory=datetime.now,
description="The date and time the observation was inserted.",
)
+
+
+class KGChanges(BaseModel):
+ """
+ KG changes model.
+ """
+
+ id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+
+ node_ids: Optional[List[str]] = Field(
+ default=None, description="The id of the node that was changed."
+ )
+ predicate_id: Optional[str] = Field(
+ default=None, description="The id of the predicate that was changed."
+ )
+
+ changelog: List[dict] = Field(description="The changelog of the changes.")
+ date: datetime = Field(
+ default_factory=datetime.now,
+ description="The date and time the changes were made.",
+ )
diff --git a/src/constants/kg.py b/src/constants/kg.py
index c775a46..c04acea 100644
--- a/src/constants/kg.py
+++ b/src/constants/kg.py
@@ -8,7 +8,8 @@ Modified By: the developer formerly known as Christian Nonis at <alch.infoemail@
-----
"""
-from typing import Optional
+from datetime import datetime
+from typing import List, Literal, Optional
import uuid
from pydantic import BaseModel, Extra, Field
@@ -19,10 +20,14 @@ class Node(BaseModel):
"""
uuid: str = Field(default_factory=lambda: str(uuid.uuid4()))
- label: str
+ labels: List[str]
name: str
- description: Optional[str]
+ description: Optional[str] = None
properties: dict = Field(default_factory=dict)
+ last_updated: datetime = Field(
+ default_factory=datetime.now,
+ description="The date and time the node was last updated.",
+ )
class Meta:
"""
@@ -39,6 +44,13 @@ class Predicate(BaseModel):
name: str
description: str
+ last_updated: datetime = Field(
+ default_factory=datetime.now,
+ description="The date and time the predicate was last updated.",
+ )
+ deprecated: bool = Field(
+ default=False, description="Whether the predicate is deprecated."
+ )
class Triple(BaseModel):
@@ -49,3 +61,29 @@ class Triple(BaseModel):
subject: Node
predicate: Predicate
object: Node
+
+
+class Relationship(BaseModel):
+ """
+ Relationship model.
+ """
+
+ direction: Literal["in", "out"]
+ predicate: Predicate
+
+
+class IdentificationParams(BaseModel):
+ """
+ Identification params model.
+ Must have a name property.
+ Than can contain other properties.
+ """
+
+ name: str
+
+ class Meta:
+ """
+ Allow extra properties to be added to the identification params.
+ """
+
+ extra = Extra.allow
diff --git a/src/constants/prompts/kg_agent.py b/src/constants/prompts/kg_agent.py
index 71e00e6..5fb1e91 100644
--- a/src/constants/prompts/kg_agent.py
+++ b/src/constants/prompts/kg_agent.py
@@ -55,6 +55,21 @@ identify nodes or be standalone nodes:
Begin!
"""
+KG_AGENT_UPDATE_STRUCTURED_PROMPT = """
+You are given a main node and a textual information about the main node.
+Your triplets must be related somehow to the main node, directly or indirectly.
+
+This is the main node who's information is about:
+{main_node}
+
+This is the information about the main node:
+== START OF INFORMATION ==
+{textual_data}
+== END OF INFORMATION ==
+
+Begin!
+"""
+
KG_AGENT_SYSTEM_PROMPT = """
You are a knowledge extraction model specialized in converting natural language into structured semantic triplets for use in knowledge graphs and memory-based AI systems.
Your output must be precise, unambiguous, and complete, covering every meaningful relation in the text.
diff --git a/src/core/agents/kg_agent.py b/src/core/agents/kg_agent.py
index d96796b..d9dfcb8 100644
--- a/src/core/agents/kg_agent.py
+++ b/src/core/agents/kg_agent.py
@@ -15,9 +15,11 @@ from langchain.tools import BaseTool
from src.adapters.cache import CacheAdapter
from src.adapters.graph import GraphAdapter
from src.adapters.llm import LLMAdapter
+from src.constants.kg import Node
from src.constants.prompts.kg_agent import (
KG_AGENT_SYSTEM_PROMPT,
KG_AGENT_UPDATE_PROMPT,
+ KG_AGENT_UPDATE_STRUCTURED_PROMPT,
)
from src.core.agents.tools.kg_agent import (
KGAgentAddTripletsTool,
@@ -131,3 +133,29 @@ class KGAgent:
config={"recursion_limit": 50},
)
return response
+
+ def structured_update_kg(
+ self, main_node: Node, textual_data: dict, identification_params: dict
+ ) -> str:
+ """
+ Update the knowledge graph with new structured information.
+ """
+
+ self._get_agent(identification_params, metadata={})
+
+ response = self.agent.invoke(
+ {
+ "messages": [
+ {
+ "role": "user",
+ "content": KG_AGENT_UPDATE_STRUCTURED_PROMPT.format(
+ textual_data=textual_data,
+ main_node=main_node,
+ ),
+ }
+ ],
+ },
+ print_mode="debug",
+ config={"recursion_limit": 50},
+ )
+ return response
diff --git a/src/core/agents/tools/kg_agent/KGAgentAddTripletsTool.py b/src/core/agents/tools/kg_agent/KGAgentAddTripletsTool.py
index 9b438fc..ff59407 100644
--- a/src/core/agents/tools/kg_agent/KGAgentAddTripletsTool.py
+++ b/src/core/agents/tools/kg_agent/KGAgentAddTripletsTool.py
@@ -9,12 +9,14 @@ Modified By: the developer formerly known as Christian Nonis at <alch.infoemail@
"""
from typing import List, Optional
+from uuid import uuid4
from langchain.tools import BaseTool
from src.adapters.graph import GraphAdapter
from src.adapters.embeddings import EmbeddingsAdapter, VectorStoreAdapter
from src.constants.kg import Node, Predicate, Triple
from src.services.api.constants.tool_schemas import TRIPLE_SCHEMA
+from src.utils.nlp.names import most_similar_name_with_labels_or_none
class KGAgentAddTripletsTool(BaseTool):
@@ -67,8 +69,74 @@ class KGAgentAddTripletsTool(BaseTool):
def _run(self, *args, **kwargs) -> str:
triplets: List[Triple] = []
for triplet_data in kwargs.get("triplets", []):
- subject = Node(**triplet_data["subject"])
- object_node = Node(**triplet_data["object"])
+ try:
+ subject = Node(**triplet_data["subject"])
+ object_node = Node(**triplet_data["object"])
+ except Exception as e:
+ print(f"Error creating nodes: {e} - {triplet_data}")
+ continue
+
+ v_sub = self.embeddings.embed_text(subject.name)
+ v_obj = self.embeddings.embed_text(object_node.name)
+
+ v_sim_sub = self.vector_store.search_vectors(v_sub.embeddings, "nodes", k=5)
+ v_sim_obj = self.vector_store.search_vectors(v_obj.embeddings, "nodes", k=5)
+ print("[v_sim_x]", v_sim_sub, v_sim_obj)
+ sim_sub = most_similar_name_with_labels_or_none(
+ subject.name,
+ [v.metadata.get("name", []) for v in v_sim_sub],
+ subject.labels,
+ [v.metadata.get("labels", []) for v in v_sim_sub],
+ )
+ sim_obj = most_similar_name_with_labels_or_none(
+ object_node.name,
+ [v.metadata.get("name", []) for v in v_sim_obj],
+ object_node.labels,
+ [v.metadata.get("labels", []) for v in v_sim_obj],
+ )
+ print("[sim_x]", sim_sub, sim_obj)
+ if sim_sub:
+ sim_sub_vector = next(
+ (v for v in v_sim_sub if v.metadata.get("name", []) == sim_sub),
+ None,
+ )
+ print("[sim_sub_vector]", sim_sub_vector)
+ subject = Node(
+ name=sim_sub_vector.metadata.get("name", []),
+ uuid=sim_sub_vector.metadata.get("uuid"),
+ labels=sim_sub_vector.metadata.get("labels", []),
+ )
+ # TODO: update changelog to record any eventual merge if sim_sub is not the same as subject.name
+ else:
+ v_sub.metadata = {
+ "name": subject.name,
+ "labels": subject.labels,
+ "uuid": str(uuid4()),
+ }
+ vector = self.vector_store.add_vectors([v_sub], store="nodes")
+ subject.uuid = v_sub.metadata.get("uuid")
+
+ if sim_obj:
+ sim_obj_vector = next(
+ (v for v in v_sim_obj if v.metadata.get("name", []) == sim_obj),
+ None,
+ )
+ print("[sim_obj_vector]", sim_obj_vector)
+ object_node = Node(
+ name=sim_obj_vector.metadata.get("name", []),
+ uuid=sim_obj_vector.metadata.get("uuid"),
+ labels=sim_obj_vector.metadata.get("labels", []),
+ )
+ # TODO: update changelog to record any eventual merge if sim_obj is not the same as object_node.name
+ else:
+ v_obj.metadata = {
+ "name": object_node.name,
+ "labels": object_node.labels,
+ "uuid": str(uuid4()),
+ }
+ vector = self.vector_store.add_vectors([v_obj], store="nodes")
+ object_node.uuid = v_obj.metadata.get("uuid")
+
predicate = Predicate(
name=triplet_data["predicate"]["name"].replace(" ", "_").upper(),
description=triplet_data["predicate"]["description"],
@@ -85,6 +153,7 @@ class KGAgentAddTripletsTool(BaseTool):
"predicate": triplet.predicate.name,
}
self.vector_store.add_vectors([vector], "triplets")
+
self.kg.add_nodes(
[triplet.subject, triplet.object],
self.identification_params,
@@ -95,4 +164,7 @@ class KGAgentAddTripletsTool(BaseTool):
triplet.predicate,
triplet.object,
)
+
+ # TODO: add changelog relationship created
+
return f"Triplets added successfully: {triplets}"
diff --git a/src/core/agents/tools/kg_agent/KGAgentSearchGraphTool.py b/src/core/agents/tools/kg_agent/KGAgentSearchGraphTool.py
index 391427a..7edfa59 100644
--- a/src/core/agents/tools/kg_agent/KGAgentSearchGraphTool.py
+++ b/src/core/agents/tools/kg_agent/KGAgentSearchGraphTool.py
@@ -37,7 +37,8 @@ class KGAgentSearchGraphTool(BaseTool):
"query": {
"type": "string",
"description": (
- "Add this parameter if you want to search textually into the knowledge graph."
+ "Add this parameter if you want to search textually into the knowledge graph. "
+ "This will perform hybrid search combining semantic and textual search."
),
},
"nodes": {
@@ -49,16 +50,23 @@ class KGAgentSearchGraphTool(BaseTool):
"items": {
"type": "object",
"properties": {
- "label": {
- "type": "string",
- "description": "The label of the node to search for.",
+ "labels": {
+ "type": "array",
+ "items": {
+ "type": "string",
+ "description": (
+ "The category of the node to search for. "
+ "(eg: Person, Organization, "
+ "Location, Product, Service, Event, etc.)"
+ ),
+ },
},
"name": {
"type": "string",
"description": "The name of the node to search for.",
},
},
- "required": ["label", "name"],
+ "required": ["labels", "name"],
},
},
},
@@ -99,18 +107,40 @@ class KGAgentSearchGraphTool(BaseTool):
if query:
query_embedding = self.embeddings.embed_text(query)
- v_results = self.vector_store.search_vectors(
- query_embedding.embeddings, "triplets"
+ v_triplets_results = self.vector_store.search_vectors(
+ query_embedding.embeddings, "triplets", k=5
+ )
+ v_nodes_results = self.vector_store.search_vectors(
+ query_embedding.embeddings, "nodes"
)
nodes.extend(self.kg.node_text_search(query))
+ nodes.extend(
+ self.kg.get_by_uuids(
+ [
+ *[
+ v_node_result.metadata.get("uuid")
+ for v_node_result in v_nodes_results
+ if v_node_result.metadata.get("uuid") is not None
+ ],
+ *[
+ node_id
+ for v_triplet_result in v_triplets_results
+ for node_id in v_triplet_result.metadata.get("node_ids", [])
+ if node_id is not None
+ ],
+ ]
+ )
+ )
- nodes = kwargs.get("nodes", [])
- _nodes = [Node(name=node["name"], label=node["label"]) for node in nodes]
+ _nodes = [
+ Node(name=node["name"], labels=node["labels"])
+ for node in kwargs.get("nodes", [])
+ ]
nodes.extend(self.kg.search_graph(_nodes))
nodes.extend(
self.kg.get_nodes_by_uuid(
- [v_result.id for v_result in v_results],
+ [v_result.id for v_result in v_results if v_result.id is not None],
with_relationships=True,
relationships_depth=1,
relationships_type=[
@@ -121,4 +151,13 @@ class KGAgentSearchGraphTool(BaseTool):
)
)
- return json.dumps([node.model_dump(mode="json") for node in nodes])
+ result_nodes = []
+ for item in nodes:
+ if isinstance(item, dict) and "node" in item:
+ result_nodes.append(item["node"])
+ elif isinstance(item, Node):
+ result_nodes.append(item)
+ else:
+ continue
+
+ return json.dumps([node.model_dump(mode="json") for node in result_nodes])
diff --git a/src/lib/milvus/client.py b/src/lib/milvus/client.py
index ed058fe..783b692 100644
--- a/src/lib/milvus/client.py
+++ b/src/lib/milvus/client.py
@@ -8,14 +8,22 @@ Modified By: the developer formerly known as Christian Nonis at <alch.infoemail@
-----
"""
+import os
+import hashlib
+
+os.environ["GRPC_DNS_RESOLVER"] = "native"
+
+if os.getenv("MILVUS_URI") and not os.getenv("MILVUS_URI").startswith(
+ ("http://", "https://")
+):
+ os.environ["MILVUS_URI"] = f"https://{os.getenv('MILVUS_URI')}"
+
from pymilvus import MilvusClient as Milvus
from src.adapters.interfaces.embeddings import VectorStoreClient
from src.config import config
from src.constants.embeddings import EMBEDDING_STORES_SIZES, Vector
-import hashlib
-
def string_to_int64(s: str) -> int:
sha256 = hashlib.sha256(s.encode()).digest()
@@ -29,6 +37,14 @@ class MilvusClient(VectorStoreClient):
def __init__(self):
self._client = None
+ self._lock = None
+
+ def _get_lock(self):
+ if self._lock is None:
+ import threading
+
+ self._lock = threading.Lock()
+ return self._lock
@property
def client(self):
@@ -36,28 +52,27 @@ class MilvusClient(VectorStoreClient):
Lazy initialization of the Milvus client.
"""
if self._client is None:
- if config.milvus.uri and config.milvus.token:
- if not config.milvus.uri.startswith(("http://", "https://")):
- uri = f"https://{config.milvus.uri}"
- else:
- uri = config.milvus.uri
- self._client = Milvus(
- uri=uri,
- token=config.milvus.token,
- )
- elif config.milvus.host and config.milvus.port:
- self._client = Milvus(
- host=config.milvus.host,
- port=config.milvus.port,
- token=config.milvus.token if config.milvus.token else None,
- )
- else:
- raise ValueError("Invalid Milvus configuration")
+ with self._get_lock():
+ if self._client is None:
+ if config.milvus.uri and config.milvus.token:
+ print("using uri", config.milvus.uri)
+ self._client = Milvus(
+ uri=config.milvus.uri,
+ token=config.milvus.token,
+ )
+ elif config.milvus.host and config.milvus.port:
+ self._client = Milvus(
+ host=config.milvus.host,
+ port=config.milvus.port,
+ token=config.milvus.token if config.milvus.token else None,
+ )
+ else:
+ raise ValueError("Invalid Milvus configuration")
return self._client
def _ensure_store(self, store: str) -> None:
"""
- Ensure the store exists.
+ Ensure the store exists and is loaded.
"""
if store not in EMBEDDING_STORES_SIZES:
raise ValueError(f"Store {store} not available")
@@ -69,18 +84,28 @@ class MilvusClient(VectorStoreClient):
vector_field_name="embeddings",
)
+ try:
+ has_partition = self.client.has_partition(store, "_default")
+ if not has_partition:
+ self.client.load_collection(store)
+ except Exception:
+ try:
+ self.client.load_collection(store)
+ except Exception:
+ pass
+
def add_vectors(self, vectors: list[Vector], store: str) -> None:
"""
Add vectors to the vector store.
"""
self._ensure_store(store)
- print("adding vectors", vectors)
self.client.insert(
store,
[
{
"id": hash(vector.id) % (2**63),
"embeddings": vector.embeddings,
+ "uuid": vector.id,
**(vector.metadata or {}),
}
for vector in vectors
@@ -94,6 +119,12 @@ class MilvusClient(VectorStoreClient):
Search vectors in the vector store and return the top k vectors.
"""
self._ensure_store(store)
+
+ try:
+ self.client.load_collection(store)
+ except Exception:
+ pass
+
_results = self.client.search(
store, data=[data_vector], limit=k, output_fields=["$meta"]
)
@@ -113,5 +144,19 @@ class MilvusClient(VectorStoreClient):
)
return results
+ def get_by_ids(self, ids: list[str], store: str) -> list[Vector]:
+ """
+ Get vectors by their IDs.
+ """
+ self._ensure_store(store)
+ collection = self.client.query(
+ collection_name=store,
+ ids=[hash(id) % (2**63) for id in ids],
+ output_fields=["$meta"],
+ )
+ return [
+ Vector(id=result["id"], metadata=result["entity"]) for result in collection
+ ]
+
_milvus_client = MilvusClient()
diff --git a/src/lib/mongo/client.py b/src/lib/mongo/client.py
index 081781d..058bd0e 100644
--- a/src/lib/mongo/client.py
+++ b/src/lib/mongo/client.py
@@ -14,7 +14,7 @@ from pymongo.database import Database
from pymongo.collection import Collection
from src.adapters.interfaces.data import DataClient, SearchResult
from src.config import config
-from src.constants.data import Observation, TextChunk
+from src.constants.data import Observation, StructuredData, TextChunk
class MongoClient(DataClient):
@@ -119,5 +119,10 @@ class MongoClient(DataClient):
),
)
+ def save_structured_data(self, structured_data: StructuredData) -> StructuredData:
+ collection = self.get_collection("structured_data")
+ collection.insert_one(structured_data.model_dump(mode="json"))
+ return structured_data
+
_mongo_client = MongoClient()
diff --git a/src/lib/neo4j/client.py b/src/lib/neo4j/client.py
index ad534a7..5b15a95 100644
--- a/src/lib/neo4j/client.py
+++ b/src/lib/neo4j/client.py
@@ -17,14 +17,40 @@ from src.constants.kg import Node, Predicate
class Neo4jClient(GraphClient):
"""
- Neo4j client.
+ Neo4j client with support for multiple databases.
"""
- def __init__(self):
+ def __init__(self, database: Optional[str] = None):
self.driver = GraphDatabase.driver(
f"bolt://{config.neo4j.host}:{config.neo4j.port}",
auth=(config.neo4j.username, config.neo4j.password),
)
+ self.database = database or "neo4j"
+
+ def execute_operation(self, operation: str, database: Optional[str] = None) -> str:
+ """
+ Execute a Neo4j operation with database override.
+ """
+ db = database or self.database
+ return self.driver.execute_query(operation, database_=db)
+
+ def ensure_database(self, database: str) -> None:
+ """
+ Ensure a database exists.
+ """
+ try:
+ cypher_query = f"CREATE DATABASE {database}"
+ self.execute_operation(cypher_query, database="neo4j")
+ except Exception as e:
+ error_msg = str(e).lower()
+ if (
+ "already exists" in error_msg
+ or "not supported in community edition" in error_msg
+ or "unsupported administration command" in error_msg
+ ):
+ pass
+ else:
+ raise
@property
def graphdb_type(self) -> str:
@@ -41,14 +67,12 @@ class Neo4jClient(GraphClient):
"""
return "The graph database is Neo4j. Cyphter is the language used to operate with it."
- def execute_operation(self, operation: str) -> str:
- """
- Execute a Neo4j operation.
- """
- return self.driver.execute_query(operation)
-
def add_nodes(
- self, nodes: list[Node], identification_params: dict, metadata: dict
+ self,
+ nodes: list[Node],
+ identification_params: dict,
+ metadata: dict,
+ database: Optional[str] = None,
) -> list[Node] | str:
"""
Add nodes to the graph.
@@ -67,33 +91,44 @@ class Neo4jClient(GraphClient):
property_assignments = []
for key, value in all_properties.items():
+ sanitized_key = key.rstrip("`").lstrip("`")
+ needs_quoting = any(
+ char in sanitized_key
+ for char in ["-", " ", ".", "+", "*", "/", "%", ":", "@", "#", "$"]
+ )
+ cypher_key = f"`{sanitized_key}`" if needs_quoting else sanitized_key
+
if isinstance(value, str):
escaped_value = value.replace("'", "\\'")
- property_assignments.append(f"n.{key} = '{escaped_value}'")
+ property_assignments.append(f"n.{cypher_key} = '{escaped_value}'")
elif isinstance(value, (int, float, bool)):
- property_assignments.append(f"n.{key} = {value}")
+ property_assignments.append(f"n.{cypher_key} = {value}")
elif value is None:
- property_assignments.append(f"n.{key} = null")
+ property_assignments.append(f"n.{cypher_key} = null")
else:
escaped_value = str(value).replace("'", "\\'")
- property_assignments.append(f"n.{key} = '{escaped_value}'")
+ property_assignments.append(f"n.{cypher_key} = '{escaped_value}'")
property_assignments.append(f"n.uuid = '{node.uuid}'")
properties_set = f"{', '.join(property_assignments)}"
+ labels_expression = ":".join(
+ label.replace(" ", "_") for label in node.labels
+ )
cypher_query = f"""
- MERGE (n:{node.label.replace(" ", "_")} {identification_set})
+ MERGE (n:{labels_expression} {identification_set})
SET {properties_set}
RETURN n
"""
- result = self.driver.execute_query(cypher_query)
+ self.ensure_database(database)
+ result = self.driver.execute_query(cypher_query, database_=database)
results.append(result)
return [
Node(
uuid=node.uuid,
- label=node.label,
+ labels=node.labels,
name=node.name,
description=node.description,
properties={**node.properties, **(metadata or {})},
@@ -110,10 +145,12 @@ class Neo4jClient(GraphClient):
"""
Add a relationship between two nodes to the graph.
"""
+ safe_desc = predicate.description.replace("'", "\\'")
+
cypher_query = f"""
- MATCH (a:{subject.label.replace(" ", "")}) WHERE a.name = '{subject.name}'
- MATCH (b:{to_object.label.replace(" ", "")}) WHERE b.name = '{to_object.name}'
- CREATE (a)-[:{predicate.name.replace(" ", "_").upper()}]->(b)
+ MATCH (a:{":".join([lb.replace(" ", "") for lb in subject.labels])}) WHERE a.name = '{subject.name}'
+ MATCH (b:{":".join([lb.replace(" ", "") for lb in to_object.labels])}) WHERE b.name = '{to_object.name}'
+ CREATE (a)-[:{predicate.name.replace(" ", "_").replace("-", "_").upper()} {{ description: '{safe_desc}' }}]->(b)
RETURN a, b
"""
result = self.driver.execute_query(cypher_query)
@@ -128,7 +165,7 @@ class Neo4jClient(GraphClient):
queries = []
for node in nodes:
- query = f"""MATCH (n:{node.label}) WHERE n.name = '{node.name}'
+ query = f"""MATCH (n:{":".join(node.labels)}) WHERE n.name = '{node.name}'
OPTIONAL MATCH (n)-[r*1]-(m)
RETURN n, r, m"""
queries.append(query)
@@ -151,7 +188,7 @@ class Neo4jClient(GraphClient):
Node(
uuid=node["uuid"],
name=node["name"],
- label=(node["labels"][0] if node["labels"] else None),
+ labels=node["labels"],
description=node["description"],
properties=node["properties"],
)
@@ -165,27 +202,27 @@ class Neo4jClient(GraphClient):
relationships_depth: Optional[int] = 1,
relationships_type: Optional[list[str]] = None,
preferred_labels: Optional[list[str]] = None,
- ) -> list[Node]:
+ ) -> list[dict]:
"""
Get nodes by their UUIDs with optional relationships.
"""
cypher_query = f"""
- MATCH (n) WHERE n.uuid IN {uuids}
+ MATCH (n) WHERE n.uuid IN ["{'","'.join(uuids)}"]
"""
if with_relationships:
if relationships_type and len(relationships_type) > 0:
- rel_pattern = "|".join(relationships_type)
+ rel_pattern = "|".join([f"`{rt}`" for rt in relationships_type])
cypher_query += f"""
- OPTIONAL MATCH (n)-[r:{rel_pattern}*1..{relationships_depth}]-(m)
- """
+ OPTIONAL MATCH (n)-[r:{rel_pattern}*1..{relationships_depth}]-(m)
+ """
else:
cypher_query += f"""
OPTIONAL MATCH (n)-[r*1..{relationships_depth}]-(m)
"""
if preferred_labels and len(preferred_labels) > 0:
cypher_query += f"""
- WHERE any(lbl IN labels(m) WHERE lbl IN {preferred_labels})
+ WHERE any(lbl IN labels(m) WHERE lbl IN ["{'","'.join(preferred_labels)}"])
"""
cypher_query += """
WITH n, r, m
@@ -207,7 +244,7 @@ class Neo4jClient(GraphClient):
"node": Node(
uuid=record["uuid"],
name=record["name"],
- label=(record["labels"][0] if record["labels"] else None),
+ labels=record["labels"],
description=record["description"],
properties=record["properties"],
),
@@ -221,7 +258,7 @@ class Neo4jClient(GraphClient):
Node(
uuid=record["uuid"],
name=record["name"],
- label=(record["labels"][0] if record["labels"] else None),
+ labels=record["labels"],
description=record["description"],
properties=record["properties"],
)
@@ -239,5 +276,46 @@ class Neo4jClient(GraphClient):
result = self.driver.execute_query(cypher_query)
return [label for record in result.records for label in record["labels"]]
+ def get_by_uuid(self, uuid: str) -> Node:
+ """
+ Get a node by its UUID.
+ """
+ cypher_query = f"""
+ MATCH (n) WHERE n.uuid = '{uuid}'
+ RETURN n.uuid as uuid, n.name as name, labels(n) as labels, n.description as description, properties(n) as properties
+ """
+ result = self.driver.execute_query(cypher_query)
+ return (
+ Node(
+ uuid=result.records[0]["uuid"],
+ name=result.records[0]["name"],
+ labels=result.records[0]["labels"],
+ description=result.records[0]["description"],
+ properties=result.records[0]["properties"],
+ )
+ if result.records
+ else None
+ )
+
+ def get_by_uuids(self, uuids: list[str]) -> list[Node]:
+ """
+ Get nodes by their UUIDs.
+ """
+ cypher_query = f"""
+ MATCH (n) WHERE n.uuid IN ["{'","'.join(uuids)}"]
+ RETURN n.uuid as uuid, n.name as name, labels(n) as labels, n.description as description, properties(n) as properties
+ """
+ result = self.driver.execute_query(cypher_query)
+ return [
+ Node(
+ uuid=record["uuid"],
+ name=record["name"],
+ labels=record["labels"],
+ description=record["description"],
+ properties=record["properties"],
+ )
+ for record in result.records
+ ]
+
_neo4j_client = Neo4jClient()
diff --git a/src/services/api/constants/requests.py b/src/services/api/constants/requests.py
index 4c05a76..bc955a8 100644
--- a/src/services/api/constants/requests.py
+++ b/src/services/api/constants/requests.py
@@ -8,9 +8,10 @@ Modified By: the developer formerly known as Christian Nonis at <alch.infoemail@
-----
"""
-from typing import List, Any
-from pydantic import BaseModel, field_serializer
+from typing import List, Any, Optional
+from pydantic import BaseModel, Field, field_serializer
from src.constants.data import Observation, TextChunk
+from src.constants.kg import IdentificationParams, Node, Relationship
from src.constants.tasks.ingestion import IngestionTaskArgs
@@ -20,6 +21,46 @@ class IngestionRequestBody(IngestionTaskArgs):
"""
+class IngestionStructuredDataElement(BaseModel):
+ """
+ Element for the structured ingestion endpoint.
+ """
+
+ json_data: dict = Field(
+ default={},
+ description="The data rapresenting the structured element.",
+ )
+ types: List[str] = Field(
+ default=[],
+ description="A list of types, used to categorize the data.",
+ )
+ identification_params: Optional[IdentificationParams] = Field(
+ default=None,
+ description="The parameters used to identify the structured element.",
+ )
+ textual_data: Optional[dict] = Field(
+ default={},
+ description="The textual descriptive data rapresenting the structured element. Could be a description, a summary, some notes, etc.",
+ )
+
+
+class IngestionStructuredRequestBody(BaseModel):
+ """
+ Request body for the structured ingestion endpoint.
+ """
+
+ data: List[IngestionStructuredDataElement]
+ store: Optional[str] = Field(
+ default="default",
+ description="The store where the data will be stored, if not provided, the data will be stored in the default store.",
+ )
+ observate_for: Optional[List[str]] = Field(
+ default=[],
+ description="What to look for and describe in the data during observation. "
+ "If not provided, the observations will be generic",
+ )
+
+
class RetrieveRequestResponse(BaseModel):
"""
Response for the retrieve endpoint.
@@ -79,3 +120,20 @@ class RetrieveRequestResponse(BaseModel):
return str(obj)
return [_ser(v) for v in value] if value is not None else []
+
+
+class RetrievedNeighborNode(Node):
+ """
+ Node model for the retrieve neighbors endpoint.
+ """
+
+ relation: Relationship
+ observations: List[str]
+
+
+class RetrieveNeighborsRequestResponse(BaseModel):
+ """
+ Response for the retrieve neighbors endpoint.
+ """
+
+ neighbors: List[RetrievedNeighborNode]
diff --git a/src/services/api/constants/tool_schemas.py b/src/services/api/constants/tool_schemas.py
index 81c6e88..68baa4e 100644
--- a/src/services/api/constants/tool_schemas.py
+++ b/src/services/api/constants/tool_schemas.py
@@ -11,14 +11,17 @@ Modified By: the developer formerly known as Christian Nonis at <alch.infoemail@
NODE_SCHEMA = {
"type": "object",
"properties": {
- "label": {
- "type": "string",
- "description": (
- "The category of the node, "
- "identifying the type of node. "
- "(eg: Person, Organization, "
- "Location, Product, Service, Event, etc.)"
- ),
+ "labels": {
+ "type": "array",
+ "items": {
+ "type": "string",
+ "description": (
+ "The category of the node, "
+ "identifying the type of node. "
+ "(eg: Person, Organization, "
+ "Location, Product, Service, Event, etc.)"
+ ),
+ },
},
"name": {
"type": "string",
@@ -45,7 +48,7 @@ NODE_SCHEMA = {
"additionalProperties": True,
},
},
- "required": ["label", "name"],
+ "required": ["labels", "name"],
}
TRIPLE_SCHEMA = {
diff --git a/src/services/api/controllers/retrieve.py b/src/services/api/controllers/retrieve.py
new file mode 100644
index 0000000..47824ad
--- /dev/null
+++ b/src/services/api/controllers/retrieve.py
@@ -0,0 +1,136 @@
+"""
+File: /retrieve.py
+Created Date: Sunday October 26th 2025
+Author: Christian Nonis <[email protected]>
+-----
+Last Modified: Sunday October 26th 2025 4:03:07 pm
+Modified By: the developer formerly known as Christian Nonis at <[email protected]>
+-----
+"""
+
+import asyncio
+import json
+
+from fastapi import HTTPException
+
+from src.constants.kg import Node, Predicate, Relationship
+from src.services.api.constants.requests import (
+ RetrieveRequestResponse,
+ RetrieveNeighborsRequestResponse,
+ RetrievedNeighborNode,
+)
+from src.services.kg_agent.main import graph_adapter
+from src.services.data.main import data_adapter
+from src.services.kg_agent.main import embeddings_adapter, vector_store_adapter
+
+
+async def retrieve_data(
+ text: str, limit: int, preferred_entities: str
+) -> RetrieveRequestResponse:
+ """
+ Retrieve data from the knowledge graph and data store.
+ """
+ if preferred_entities:
+ preferred_entities = [
+ e.strip() for e in preferred_entities.split(",") if e.strip()
+ ]
+ else:
+ preferred_entities = []
+
+ def _get_data():
+ text_embeddings = embeddings_adapter.embed_text(text)
+ data_vectors = vector_store_adapter.search_vectors(
+ text_embeddings.embeddings, "data", limit
+ )
+ triple_vectors = vector_store_adapter.search_vectors(
+ text_embeddings.embeddings, "triplets", limit
+ )
+
+ search_result = data_adapter.search(text)
+
+ ts_text_chunks = search_result.text_chunks
+ ts_observations = search_result.observations
+
+ v_text_chunks, v_observations = data_adapter.get_text_chunks_by_ids(
+ [dv.metadata.get("resource_id") for dv in data_vectors], True
+ )
+
+ node_ids = [
+ node_id
+ for tv in triple_vectors
+ for node_id in tv.metadata.get("node_ids", [])
+ ]
+ print("[preferred entities]", preferred_entities, type(preferred_entities))
+ nodes = graph_adapter.get_nodes_by_uuid(
+ uuids=node_ids,
+ with_relationships=True,
+ relationships_depth=1,
+ relationships_type=[
+ tv.metadata.get("predicate")
+ for tv in triple_vectors
+ if tv.metadata.get("predicate", None)
+ ],
+ preferred_labels=preferred_entities or [],
+ )
+ return ts_text_chunks, ts_observations, v_text_chunks, v_observations, nodes
+
+ ts_text_chunks, ts_observations, v_text_chunks, v_observations, nodes = (
+ await asyncio.to_thread(_get_data)
+ )
+
+ return RetrieveRequestResponse(
+ data=[*ts_text_chunks, *v_text_chunks],
+ observations=[*ts_observations, *v_observations],
+ relationships=nodes,
+ )
+
+
+async def retrieve_neighbors(uuid: str, limit: int) -> RetrieveNeighborsRequestResponse:
+ """
+ Retrieve neighbors of an entity from the knowledge graph.
+ """
+ node = graph_adapter.get_by_uuid(uuid)
+ if not node:
+ raise HTTPException(status_code=404, detail="Entity not found")
+ results = graph_adapter.get_nodes_by_uuid(
+ uuids=[node.uuid],
+ with_relationships=True,
+ relationships_depth=5,
+ # preferred_labels=[node.label],
+ )
+
+ neighbors = []
+
+ for result in results:
+ related_node = result.get("related_nodes")
+ relationships = result.get("relationships")
+
+ if not related_node or not relationships:
+ continue
+
+ relationship = relationships[0] if relationships else None
+
+ neighbor_node = RetrievedNeighborNode(
+ uuid=related_node.get("uuid"),
+ name=related_node.get("name"),
+ labels=related_node.get("labels") if related_node.get("labels") else [],
+ description=related_node.get("description"),
+ properties=(
+ related_node.get("properties") if related_node.get("properties") else {}
+ ),
+ observations=[],
+ relation=Relationship(
+ predicate=Predicate(
+ name=(
+ relationship.type
+ if hasattr(relationship, "type")
+ else str(relationship)
+ ),
+ description="",
+ ),
+ direction="in",
+ ),
+ )
+ neighbors.append(neighbor_node)
+
+ return RetrieveNeighborsRequestResponse(neighbors=neighbors)
diff --git a/src/services/api/routes/ingest.py b/src/services/api/routes/ingest.py
index 544f2f4..c3fc811 100644
--- a/src/services/api/routes/ingest.py
+++ b/src/services/api/routes/ingest.py
@@ -11,8 +11,14 @@ Modified By: the developer formerly known as Christian Nonis at <alch.infoemail@
from fastapi import APIRouter
from starlette.responses import JSONResponse
-from src.services.api.constants.requests import IngestionRequestBody
+from src.services.api.constants.requests import (
+ IngestionRequestBody,
+ IngestionStructuredRequestBody,
+)
from src.workers.tasks.ingestion import ingest_data as ingest_data_task
+from src.workers.tasks.ingestion import (
+ ingest_structured_data as ingest_structured_data_task,
+)
ingest_router = APIRouter(prefix="/ingest", tags=["ingest"])
@@ -28,3 +34,16 @@ async def ingest_data(data: IngestionRequestBody):
return JSONResponse(
content={"message": "Data ingested successfully", "task_id": task.id}
)
+
+
+@ingest_router.post(path="/structured")
+async def ingest_structured_data(data: IngestionStructuredRequestBody):
+ """
+ Ingest structured data to the processing pipeline and save to the memory.
+ """
+
+ task = ingest_structured_data_task.delay(data.model_dump())
+
+ return JSONResponse(
+ content={"message": "Structured data ingested successfully", "task_id": task.id}
+ )
diff --git a/src/services/api/routes/retrieve.py b/src/services/api/routes/retrieve.py
index ac6d493..bfd6a44 100644
--- a/src/services/api/routes/retrieve.py
+++ b/src/services/api/routes/retrieve.py
@@ -8,14 +8,18 @@ Modified By: the developer formerly known as Christian Nonis at <alch.infoemail@
-----
"""
-import asyncio
-from typing import List
from fastapi import APIRouter, Query
-from src.services.api.constants.requests import RetrieveRequestResponse
-from src.services.data.main import data_adapter
-from src.services.kg_agent.main import embeddings_adapter, vector_store_adapter
-from src.services.kg_agent.main import graph_adapter
+from src.services.api.constants.requests import (
+ RetrieveRequestResponse,
+ RetrieveNeighborsRequestResponse,
+)
+from src.services.api.controllers.retrieve import (
+ retrieve_neighbors as retrieve_neighbors_controller,
+)
+from src.services.api.controllers.retrieve import (
+ retrieve_data as retrieve_data_controller,
+)
retrieve_router = APIRouter(prefix="/retrieve", tags=["retrieve"])
@@ -24,58 +28,32 @@ retrieve_router = APIRouter(prefix="/retrieve", tags=["retrieve"])
async def retrieve(
text: str = Query(..., description="The text to search for."),
limit: int = Query(10, description="The number of results to return."),
- preferred_entities: List[str] = Query(
- ..., description="The entities to prioritize in the relationships."
+ preferred_entities: str = Query(
+ ...,
+ description="The entities to prioritize in the relationships, separated by commas.",
),
):
"""
Retrieve data from the knowledge graph and data store.
"""
+ return await retrieve_data_controller(text, limit, preferred_entities)
- def _get_data():
- text_embeddings = embeddings_adapter.embed_text(text)
- data_vectors = vector_store_adapter.search_vectors(
- text_embeddings.embeddings, "data", limit
- )
- triple_vectors = vector_store_adapter.search_vectors(
- text_embeddings.embeddings, "triplets", limit
- )
- search_result = data_adapter.search(text)
-
- ts_text_chunks = search_result.text_chunks
- ts_observations = search_result.observations
-
- v_text_chunks, v_observations = data_adapter.get_text_chunks_by_ids(
- [dv.metadata.get("resource_id") for dv in data_vectors], True
- )
-
- node_ids = [
- node_id
- for tv in triple_vectors
- for node_id in tv.metadata.get("node_ids", [])
- ]
-
- nodes = graph_adapter.get_nodes_by_uuid(
- uuids=node_ids,
- with_relationships=True,
- relationships_depth=1,
- relationships_type=[
- tv.metadata.get("predicate")
- for tv in triple_vectors
- if tv.metadata.get("predicate", None)
- ],
- preferred_labels=preferred_entities or [],
- )
-
- return ts_text_chunks, ts_observations, v_text_chunks, v_observations, nodes
+@retrieve_router.get(
+ "/entities/neighbors", response_model=RetrieveNeighborsRequestResponse
+)
+async def get_neighbors(
+ uuid: str = Query(..., description="The UUID of the entity to get neighbors for."),
+ limit: int = Query(10, description="The number of neighbors to return."),
+):
+ """
+ Get the neighbors of an entity.
+ """
+ return await retrieve_neighbors_controller(uuid, limit)
- ts_text_chunks, ts_observations, v_text_chunks, v_observations, nodes = (
- await asyncio.to_thread(_get_data)
- )
- return RetrieveRequestResponse(
- data=[*ts_text_chunks, *v_text_chunks],
- observations=[*ts_observations, *v_observations],
- relationships=nodes,
- )
+@retrieve_router.post("/context")
+async def get_context(request):
+ """
+ Get the context of an entity.
+ """
diff --git a/src/utils/nlp/__init__.py b/src/utils/nlp/__init__.py
new file mode 100644
index 0000000..eb28c5b
--- /dev/null
+++ b/src/utils/nlp/__init__.py
@@ -0,0 +1,9 @@
+"""
+File: /__init__.py
+Created Date: Monday October 27th 2025
+Author: Christian Nonis <[email protected]>
+-----
+Last Modified: Monday October 27th 2025 8:20:39 pm
+Modified By: the developer formerly known as Christian Nonis at <[email protected]>
+-----
+"""
diff --git a/src/utils/nlp/names.py b/src/utils/nlp/names.py
new file mode 100644
index 0000000..ec1529f
--- /dev/null
+++ b/src/utils/nlp/names.py
@@ -0,0 +1,71 @@
+"""
+File: /names.py
+Created Date: Monday October 27th 2025
+Author: Christian Nonis <[email protected]>
+-----
+Last Modified: Monday October 27th 2025 8:20:50 pm
+Modified By: the developer formerly known as Christian Nonis at <[email protected]>
+-----
+"""
+
+import Levenshtein
+
+
+def levenshtein_similarity(name1: str, name2: str) -> float:
+ """
+ Measure the similarity between two names using the Levenshtein distance.
+ """
+ return Levenshtein.ratio(name1, name2)
+
+
+def labels_similarity(labels_a: list[str], labels_b: list[str]) -> float:
+ """
+ Calculate the similarity between two lists of labels using Jaccard similarity.
+ """
+ set_a = set(labels_a)
+ set_b = set(labels_b)
+
+ if not set_a and not set_b:
+ return 1.0
+
+ intersection = len(set_a & set_b)
+ union = len(set_a | set_b)
+
+ return intersection / union if union > 0 else 0.0
+
+
+def find_most_similar_names(
+ name: str, names: list[str], labels_a: list[str], labels_list: list[list[str]]
+) -> tuple[str, float]:
+ """
+ Find the most similar name to the given name.
+ """
+ most_similar_name = None
+ max_similarity = 0
+
+ for n, labels in zip(names, labels_list):
+ name_sim = levenshtein_similarity(name, n)
+ labels_sim = labels_similarity(labels_a, labels)
+ similarity = (name_sim * 0.6) + (labels_sim * 0.4)
+ if similarity > max_similarity:
+ max_similarity = similarity
+ most_similar_name = n
+
+ return most_similar_name, max_similarity
+
+
+def most_similar_name_with_labels_or_none(
+ name: str, names: list[str], labels_a: list[str], labels_list: list[list[str]]
+) -> str:
+ """
+ Find the most similar name to the given name.
+ """
+ most_similar_name, max_similarity = find_most_similar_names(
+ name, names, labels_a, labels_list
+ )
+ print(
+ f"[SIMILARITY] {max_similarity} for {name} w/ labels {labels_a} and {most_similar_name} w/ labels {labels_list}"
+ )
+ if max_similarity > 0.5:
+ return most_similar_name
+ return None
diff --git a/src/workers/tasks/ingestion.py b/src/workers/tasks/ingestion.py
index 5ae2ef1..70aad6f 100644
--- a/src/workers/tasks/ingestion.py
+++ b/src/workers/tasks/ingestion.py
@@ -9,10 +9,14 @@ Modified By: the developer formerly known as Christian Nonis at <alch.infoemail@
"""
import json
-from src.constants.data import Observation, TextChunk
+from uuid import uuid4
+from src.constants.data import Observation, StructuredData, TextChunk
+from src.constants.kg import Node
+from src.services.api.constants.requests import IngestionStructuredRequestBody
from src.services.data.main import data_adapter
from src.services.kg_agent.main import (
embeddings_adapter,
+ graph_adapter,
kg_agent,
vector_store_adapter,
)
@@ -31,6 +35,26 @@ def ingest_data(self, args: dict):
"""
payload = IngestionTaskArgs(**args)
+ payload.meta_keys = (
+ {
+ f"{k.replace(' ', '_').lower()}": v
+ for k, v in payload.meta_keys.items()
+ if v is not None
+ }
+ if payload.meta_keys
+ else None
+ )
+
+ payload.identification_params = (
+ {
+ f"{k.replace(' ', '_').lower()}": v
+ for k, v in payload.identification_params.items()
+ if v is not None
+ }
+ if payload.identification_params
+ else None
+ )
+
# ================================================
# --------------- Data Saving --------------------
# ================================================
@@ -65,6 +89,7 @@ def ingest_data(self, args: dict):
),
observate_for=payload.observate_for,
)
+ print("[observations]", observations, type(observations))
data_adapter.save_observations(
[
Observation(
@@ -96,3 +121,120 @@ def ingest_data(self, args: dict):
)
return self.request.id
+
+
+@ingestion_app.task(bind=True)
+def ingest_structured_data(self, args: dict):
+ """
+ Ingest structured data into the database.
+ """
+ payload = IngestionStructuredRequestBody(**args)
+
+ for element in payload.data:
+ uuid = str(uuid4())
+ print("[uuid]", uuid)
+ node = graph_adapter.add_nodes(
+ [
+ Node(
+ uuid=uuid,
+ labels=element.types,
+ name=element.identification_params.name,
+ properties=element.json_data,
+ )
+ ],
+ identification_params=element.identification_params,
+ metadata=element.textual_data,
+ )[0]
+ vector = embeddings_adapter.embed_text(node.name)
+ vector.id = node.uuid
+ vector.metadata = {
+ "labels": node.labels,
+ "name": node.name,
+ "uuid": node.uuid,
+ }
+ vector_store_adapter.add_vectors(vectors=[vector], store="nodes")
+
+ data_adapter.save_structured_data(
+ StructuredData(
+ id=uuid,
+ data=element.json_data,
+ types=element.types,
+ metadata={
+ "identification_params": element.identification_params,
+ "textual_data": element.textual_data,
+ "labels": element.types,
+ "name": element.identification_params.name,
+ },
+ )
+ )
+ vector = embeddings_adapter.embed_text(
+ "\n".join(
+ [
+ (
+ v
+ if isinstance(v, str)
+ else (", ".join(v) if isinstance(v, list) else str(v))
+ )
+ for v in element.textual_data.values()
+ ]
+ )
+ + "\n"
+ + (", ".join(node.labels) if isinstance(node.labels, list) else node.labels)
+ + "\n"
+ + node.name
+ )
+ vector.id = node.uuid
+ vector.metadata = {
+ "labels": node.labels,
+ "name": node.name,
+ "uuid": node.uuid,
+ }
+ vector_store_adapter.add_vectors(vectors=[vector], store="data")
+
+ text_content = "\n".join(
+ [
+ (
+ f"{k}: {v}"
+ if isinstance(v, str)
+ else (f"{k}: {', '.join(v) if isinstance(v, list) else str(v)}")
+ )
+ for k, v in element.textual_data.items()
+ ]
+ )
+ observations = observations_agent.observe(
+ text=text_content,
+ observate_for=payload.observate_for,
+ )
+ for obs in observations:
+ obs_vector = embeddings_adapter.embed_text(obs)
+ obs_vector.id = str(uuid4())
+ obs_vector.metadata = {
+ "resource_id": node.uuid,
+ "labels": node.labels,
+ "name": node.name,
+ }
+ vector_store_adapter.add_vectors(vectors=[obs_vector], store="observations")
+ if len(observations) > 0:
+ data_adapter.save_observations(
+ [
+ Observation(
+ id=node.uuid,
+ text=obs,
+ metadata={
+ "labels": node.labels,
+ "name": node.name,
+ },
+ resource_id=node.uuid,
+ )
+ for obs in observations
+ ]
+ )
+
+ # TODO: + vectorize the predicates and new nodes + save changelog&vectors
+ kg_agent.structured_update_kg(
+ main_node=node,
+ textual_data=element.textual_data,
+ identification_params=element.identification_params.model_dump(mode="json"),
+ )
+
+ return self.request.id