LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
feat: Added an SFTP connector, (#674)
Chandrasekharan Md60868a0cd38
diff --git a/backend/pdm.lock b/backend/pdm.lockindex 656514b42..378cebd6a 100644--- a/backend/pdm.lock+++ b/backend/pdm.lock@@ -4,8 +4,8 @@[metadata]groups = ["default", "deploy", "dev", "test"]strategy = ["cross_platform", "inherit_metadata"]-lock_version = "4.4.1"-content_hash = "sha256:dcf5f16cb2201a698efff3daf588c5dea4bda976aae3a86054780e01d0548549"+lock_version = "4.4.2"+content_hash = "sha256:2a64b25fbdae60bb1332d0486b0f4296d4b338fc8b72057694e9d4d47ad71dcb"[[package]]name = "adlfs"@@ -437,6 +437,42 @@ files = [{file = "azure_storage_blob-12.23.0.tar.gz", hash = "sha256:2fadbceda1d99c4a72dfd32e0122d7bca8b5e8d2563f5c624d634aeaff49c9df"},]+[[package]]+name = "bcrypt"+version = "4.2.0"+requires_python = ">=3.7"+summary = "Modern password hashing for your software and your servers"+groups = ["default", "dev"]+files = [+ {file = "bcrypt-4.2.0-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:096a15d26ed6ce37a14c1ac1e48119660f21b24cba457f160a4b830f3fe6b5cb"},+ {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c02d944ca89d9b1922ceb8a46460dd17df1ba37ab66feac4870f6862a1533c00"},+ {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d84cf6d877918620b687b8fd1bf7781d11e8a0998f576c7aa939776b512b98d"},+ {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:1bb429fedbe0249465cdd85a58e8376f31bb315e484f16e68ca4c786dcc04291"},+ {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:655ea221910bcac76ea08aaa76df427ef8625f92e55a8ee44fbf7753dbabb328"},+ {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:1ee38e858bf5d0287c39b7a1fc59eec64bbf880c7d504d3a06a96c16e14058e7"},+ {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:0da52759f7f30e83f1e30a888d9163a81353ef224d82dc58eb5bb52efcabc399"},+ {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3698393a1b1f1fd5714524193849d0c6d524d33523acca37cd28f02899285060"},+ {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:762a2c5fb35f89606a9fde5e51392dad0cd1ab7ae64149a8b935fe8d79dd5ed7"},+ {file = "bcrypt-4.2.0-cp37-abi3-win32.whl", hash = "sha256:5a1e8aa9b28ae28020a3ac4b053117fb51c57a010b9f969603ed885f23841458"},+ {file = "bcrypt-4.2.0-cp37-abi3-win_amd64.whl", hash = "sha256:8f6ede91359e5df88d1f5c1ef47428a4420136f3ce97763e31b86dd8280fbdf5"},+ {file = "bcrypt-4.2.0-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:c52aac18ea1f4a4f65963ea4f9530c306b56ccd0c6f8c8da0c06976e34a6e841"},+ {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3bbbfb2734f0e4f37c5136130405332640a1e46e6b23e000eeff2ba8d005da68"},+ {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3413bd60460f76097ee2e0a493ccebe4a7601918219c02f503984f0a7ee0aebe"},+ {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:8d7bb9c42801035e61c109c345a28ed7e84426ae4865511eb82e913df18f58c2"},+ {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3d3a6d28cb2305b43feac298774b997e372e56c7c7afd90a12b3dc49b189151c"},+ {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:9c1c4ad86351339c5f320ca372dfba6cb6beb25e8efc659bedd918d921956bae"},+ {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:27fe0f57bb5573104b5a6de5e4153c60814c711b29364c10a75a54bb6d7ff48d"},+ {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:8ac68872c82f1add6a20bd489870c71b00ebacd2e9134a8aa3f98a0052ab4b0e"},+ {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cb2a8ec2bc07d3553ccebf0746bbf3d19426d1c6d1adbd4fa48925f66af7b9e8"},+ {file = "bcrypt-4.2.0-cp39-abi3-win32.whl", hash = "sha256:77800b7147c9dc905db1cba26abe31e504d8247ac73580b4aa179f98e6608f34"},+ {file = "bcrypt-4.2.0-cp39-abi3-win_amd64.whl", hash = "sha256:61ed14326ee023917ecd093ee6ef422a72f3aec6f07e21ea5f10622b735538a9"},+ {file = "bcrypt-4.2.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:39e1d30c7233cfc54f5c3f2c825156fe044efdd3e0b9d309512cc514a263ec2a"},+ {file = "bcrypt-4.2.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:f4f4acf526fcd1c34e7ce851147deedd4e26e6402369304220250598b26448db"},+ {file = "bcrypt-4.2.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:1ff39b78a52cf03fdf902635e4c81e544714861ba3f0efc56558979dd4f09170"},+ {file = "bcrypt-4.2.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:373db9abe198e8e2c70d12b479464e0d5092cc122b20ec504097b5f2297ed184"},+ {file = "bcrypt-4.2.0.tar.gz", hash = "sha256:cf69eaf5185fd58f268f805b505ce31f9b9fc2d64b376642164e9244540c1221"},+]+[[package]]name = "beautifulsoup4"version = "4.12.3"@@ -1247,6 +1283,22 @@ files = [{file = "fsspec-2024.6.0.tar.gz", hash = "sha256:f579960a56e6d8038a9efc8f9c77279ec12e6299aa86b0769a7e9c46b94527c2"},]+[[package]]+name = "fsspec"+version = "2024.6.0"+extras = ["sftp"]+requires_python = ">=3.8"+summary = "File-system specification"+groups = ["default", "dev"]+dependencies = [+ "fsspec==2024.6.0",+ "paramiko",+]+files = [+ {file = "fsspec-2024.6.0-py3-none-any.whl", hash = "sha256:58d7122eb8a1a46f7f13453187bfea4972d66bf01618d37366521b1998034cee"},+ {file = "fsspec-2024.6.0.tar.gz", hash = "sha256:f579960a56e6d8038a9efc8f9c77279ec12e6299aa86b0769a7e9c46b94527c2"},+]+[[package]]name = "funcy"version = "2.0"@@ -1885,7 +1937,7 @@ files = [[[package]]name = "huggingface-hub"-version = "0.25.0"+version = "0.25.1"requires_python = ">=3.8.0"summary = "Client library to download and publish models, datasets and other repos on the huggingface.co hub"groups = ["default", "dev"]@@ -1899,8 +1951,8 @@ dependencies = ["typing-extensions>=3.7.4.3",]files = [- {file = "huggingface_hub-0.25.0-py3-none-any.whl", hash = "sha256:e2f357b35d72d5012cfd127108c4e14abcd61ba4ebc90a5a374dc2456cb34e12"},- {file = "huggingface_hub-0.25.0.tar.gz", hash = "sha256:fb5fbe6c12fcd99d187ec7db95db9110fb1a20505f23040a5449a717c1a0db4d"},+ {file = "huggingface_hub-0.25.1-py3-none-any.whl", hash = "sha256:a5158ded931b3188f54ea9028097312cb0acd50bffaaa2612014c3c526b44972"},+ {file = "huggingface_hub-0.25.1.tar.gz", hash = "sha256:9ff7cb327343211fbd06e2b149b8f362fd1e389454f3f14c6db75a4999ee20ff"},][[package]]@@ -2088,7 +2140,7 @@ files = [[[package]]name = "llama-cloud"-version = "0.0.17"+version = "0.1.0"requires_python = "<4,>=3.8"summary = ""groups = ["default", "dev"]@@ -2097,8 +2149,8 @@ dependencies = ["pydantic>=1.10",]files = [- {file = "llama_cloud-0.0.17-py3-none-any.whl", hash = "sha256:da898dcc98de84f29886f979b1ccae1e96d9f73d1b0e07146a51d315b161e45c"},- {file = "llama_cloud-0.0.17.tar.gz", hash = "sha256:7fd6857bbbb91937535572ccb48daa38189f55cdd7411185d8083dab29ba1299"},+ {file = "llama_cloud-0.1.0-py3-none-any.whl", hash = "sha256:e315046d856780d996886864cd315d5f216cd398b3bd3f005fcae393c66f5494"},+ {file = "llama_cloud-0.1.0.tar.gz", hash = "sha256:8b31e1fb87f48c397c1a6415816cf37afcb4abe9fe7a5eb4101a2dc6d5c6ebbc"},][[package]]@@ -2908,7 +2960,7 @@ files = [[[package]]name = "openai"-version = "1.47.0"+version = "1.47.1"requires_python = ">=3.7.1"summary = "The official Python library for the openai API"groups = ["default", "dev"]@@ -2923,8 +2975,8 @@ dependencies = ["typing-extensions<5,>=4.11",]files = [- {file = "openai-1.47.0-py3-none-any.whl", hash = "sha256:9ccc8737dfa791f7bd903db4758c176b8544a8cd89d3a3d2add3cea02a34c3a0"},- {file = "openai-1.47.0.tar.gz", hash = "sha256:6e14d6f77c8cf546646afcd87a2ef752505b3710d2564a2e433e17307dfa86a0"},+ {file = "openai-1.47.1-py3-none-any.whl", hash = "sha256:34277583bf268bb2494bc03f48ac123788c5e2a914db1d5a23d5edc29d35c825"},+ {file = "openai-1.47.1.tar.gz", hash = "sha256:62c8f5f478f82ffafc93b33040f8bb16a45948306198bd0cba2da2ecd9cf7323"},][[package]]@@ -3013,6 +3065,22 @@ files = [{file = "pandas-2.1.4.tar.gz", hash = "sha256:fcb68203c833cc735321512e13861358079a96c174a61f5116a1de89c58c0ef7"},]+[[package]]+name = "paramiko"+version = "3.5.0"+requires_python = ">=3.6"+summary = "SSH2 protocol library"+groups = ["default", "dev"]+dependencies = [+ "bcrypt>=3.2",+ "cryptography>=3.3",+ "pynacl>=1.5",+]+files = [+ {file = "paramiko-3.5.0-py3-none-any.whl", hash = "sha256:1fedf06b085359051cd7d0d270cebe19e755a8a921cc2ddbfa647fb0cd7d68f9"},+ {file = "paramiko-3.5.0.tar.gz", hash = "sha256:ad11e540da4f55cedda52931f1a3f812a8238a7af7f62a60de538cd80bb28124"},+]+[[package]]name = "pdfminer-six"version = "20231228"@@ -3549,6 +3617,28 @@ files = [{file = "PyMySQL-1.1.0.tar.gz", hash = "sha256:4f13a7df8bf36a51e81dd9f3605fede45a4878fe02f9236349fd82a3f0612f96"},]+[[package]]+name = "pynacl"+version = "1.5.0"+requires_python = ">=3.6"+summary = "Python binding to the Networking and Cryptography (NaCl) library"+groups = ["default", "dev"]+dependencies = [+ "cffi>=1.4.1",+]+files = [+ {file = "PyNaCl-1.5.0-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1"},+ {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92"},+ {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394"},+ {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:0c84947a22519e013607c9be43706dd42513f9e6ae5d39d3613ca1e142fba44d"},+ {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06b8f6fa7f5de8d5d2f7573fe8c863c051225a27b61e6860fd047b1775807858"},+ {file = "PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b"},+ {file = "PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:61f642bf2378713e2c2e1de73444a3778e5f0a38be6fee0fe532fe30060282ff"},+ {file = "PyNaCl-1.5.0-cp36-abi3-win32.whl", hash = "sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543"},+ {file = "PyNaCl-1.5.0-cp36-abi3-win_amd64.whl", hash = "sha256:20f42270d27e1b6a29f54032090b972d97f0a1b0948cc52392041ef7831fee93"},+ {file = "PyNaCl-1.5.0.tar.gz", hash = "sha256:8ac7448f09ab85811607bdd21ec2464495ac8b7c66d146bf545b0f08fb9220ba"},+]+[[package]]name = "pyopenssl"version = "23.3.0"@@ -4717,13 +4807,13 @@ files = [[[package]]name = "tzdata"-version = "2024.1"+version = "2024.2"requires_python = ">=2"summary = "Provider of IANA time zone data"groups = ["default", "dev"]files = [- {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"},- {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"},+ {file = "tzdata-2024.2-py2.py3-none-any.whl", hash = "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd"},+ {file = "tzdata-2024.2.tar.gz", hash = "sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc"},][[package]]@@ -4797,6 +4887,7 @@ dependencies = ["adlfs==2023.8.0","boxfs==0.2.1","dropboxdrivefs==1.4.1",+ "fsspec[sftp]~=2024.6.0","gcsfs==2024.6.0","google-auth==2.20.0","google-cloud-bigquery==3.11.4",@@ -4806,6 +4897,7 @@ dependencies = ["psycopg2-binary==2.9.9","pymssql==2.2.8","s3fs[boto3]==2024.6.0",+ "singleton-decorator~=1.0.0","snowflake-connector-python[pandas]~=3.6.0",]@@ -5070,7 +5162,7 @@ files = [[[package]]name = "yarl"-version = "1.11.1"+version = "1.12.1"requires_python = ">=3.8"summary = "Yet another URL library"groups = ["default", "dev"]@@ -5079,51 +5171,51 @@ dependencies = ["multidict>=4.0",]files = [- {file = "yarl-1.11.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:400cd42185f92de559d29eeb529e71d80dfbd2f45c36844914a4a34297ca6f00"},- {file = "yarl-1.11.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8258c86f47e080a258993eed877d579c71da7bda26af86ce6c2d2d072c11320d"},- {file = "yarl-1.11.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2164cd9725092761fed26f299e3f276bb4b537ca58e6ff6b252eae9631b5c96e"},- {file = "yarl-1.11.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08ea567c16f140af8ddc7cb58e27e9138a1386e3e6e53982abaa6f2377b38cc"},- {file = "yarl-1.11.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:768ecc550096b028754ea28bf90fde071c379c62c43afa574edc6f33ee5daaec"},- {file = "yarl-1.11.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2909fa3a7d249ef64eeb2faa04b7957e34fefb6ec9966506312349ed8a7e77bf"},- {file = "yarl-1.11.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01a8697ec24f17c349c4f655763c4db70eebc56a5f82995e5e26e837c6eb0e49"},- {file = "yarl-1.11.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e286580b6511aac7c3268a78cdb861ec739d3e5a2a53b4809faef6b49778eaff"},- {file = "yarl-1.11.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4179522dc0305c3fc9782549175c8e8849252fefeb077c92a73889ccbcd508ad"},- {file = "yarl-1.11.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:27fcb271a41b746bd0e2a92182df507e1c204759f460ff784ca614e12dd85145"},- {file = "yarl-1.11.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:f61db3b7e870914dbd9434b560075e0366771eecbe6d2b5561f5bc7485f39efd"},- {file = "yarl-1.11.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:c92261eb2ad367629dc437536463dc934030c9e7caca861cc51990fe6c565f26"},- {file = "yarl-1.11.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d95b52fbef190ca87d8c42f49e314eace4fc52070f3dfa5f87a6594b0c1c6e46"},- {file = "yarl-1.11.1-cp310-cp310-win32.whl", hash = "sha256:489fa8bde4f1244ad6c5f6d11bb33e09cf0d1d0367edb197619c3e3fc06f3d91"},- {file = "yarl-1.11.1-cp310-cp310-win_amd64.whl", hash = "sha256:476e20c433b356e16e9a141449f25161e6b69984fb4cdbd7cd4bd54c17844998"},- {file = "yarl-1.11.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:946eedc12895873891aaceb39bceb484b4977f70373e0122da483f6c38faaa68"},- {file = "yarl-1.11.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:21a7c12321436b066c11ec19c7e3cb9aec18884fe0d5b25d03d756a9e654edfe"},- {file = "yarl-1.11.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c35f493b867912f6fda721a59cc7c4766d382040bdf1ddaeeaa7fa4d072f4675"},- {file = "yarl-1.11.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25861303e0be76b60fddc1250ec5986c42f0a5c0c50ff57cc30b1be199c00e63"},- {file = "yarl-1.11.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e4b53f73077e839b3f89c992223f15b1d2ab314bdbdf502afdc7bb18e95eae27"},- {file = "yarl-1.11.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:327c724b01b8641a1bf1ab3b232fb638706e50f76c0b5bf16051ab65c868fac5"},- {file = "yarl-1.11.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4307d9a3417eea87715c9736d050c83e8c1904e9b7aada6ce61b46361b733d92"},- {file = "yarl-1.11.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:48a28bed68ab8fb7e380775f0029a079f08a17799cb3387a65d14ace16c12e2b"},- {file = "yarl-1.11.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:067b961853c8e62725ff2893226fef3d0da060656a9827f3f520fb1d19b2b68a"},- {file = "yarl-1.11.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8215f6f21394d1f46e222abeb06316e77ef328d628f593502d8fc2a9117bde83"},- {file = "yarl-1.11.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:498442e3af2a860a663baa14fbf23fb04b0dd758039c0e7c8f91cb9279799bff"},- {file = "yarl-1.11.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:69721b8effdb588cb055cc22f7c5105ca6fdaa5aeb3ea09021d517882c4a904c"},- {file = "yarl-1.11.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1e969fa4c1e0b1a391f3fcbcb9ec31e84440253325b534519be0d28f4b6b533e"},- {file = "yarl-1.11.1-cp311-cp311-win32.whl", hash = "sha256:7d51324a04fc4b0e097ff8a153e9276c2593106a811704025bbc1d6916f45ca6"},- {file = "yarl-1.11.1-cp311-cp311-win_amd64.whl", hash = "sha256:15061ce6584ece023457fb8b7a7a69ec40bf7114d781a8c4f5dcd68e28b5c53b"},- {file = "yarl-1.11.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:884eab2ce97cbaf89f264372eae58388862c33c4f551c15680dd80f53c89a269"},- {file = "yarl-1.11.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8a336eaa7ee7e87cdece3cedb395c9657d227bfceb6781295cf56abcd3386a26"},- {file = "yarl-1.11.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:87f020d010ba80a247c4abc335fc13421037800ca20b42af5ae40e5fd75e7909"},- {file = "yarl-1.11.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:637c7ddb585a62d4469f843dac221f23eec3cbad31693b23abbc2c366ad41ff4"},- {file = "yarl-1.11.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:48dfd117ab93f0129084577a07287376cc69c08138694396f305636e229caa1a"},- {file = "yarl-1.11.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:75e0ae31fb5ccab6eda09ba1494e87eb226dcbd2372dae96b87800e1dcc98804"},- {file = "yarl-1.11.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f46f81501160c28d0c0b7333b4f7be8983dbbc161983b6fb814024d1b4952f79"},- {file = "yarl-1.11.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:04293941646647b3bfb1719d1d11ff1028e9c30199509a844da3c0f5919dc520"},- {file = "yarl-1.11.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:250e888fa62d73e721f3041e3a9abf427788a1934b426b45e1b92f62c1f68366"},- {file = "yarl-1.11.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:e8f63904df26d1a66aabc141bfd258bf738b9bc7bc6bdef22713b4f5ef789a4c"},- {file = "yarl-1.11.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:aac44097d838dda26526cffb63bdd8737a2dbdf5f2c68efb72ad83aec6673c7e"},- {file = "yarl-1.11.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:267b24f891e74eccbdff42241c5fb4f974de2d6271dcc7d7e0c9ae1079a560d9"},- {file = "yarl-1.11.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:6907daa4b9d7a688063ed098c472f96e8181733c525e03e866fb5db480a424df"},- {file = "yarl-1.11.1-cp39-cp39-win32.whl", hash = "sha256:14438dfc5015661f75f85bc5adad0743678eefee266ff0c9a8e32969d5d69f74"},- {file = "yarl-1.11.1-cp39-cp39-win_amd64.whl", hash = "sha256:94d0caaa912bfcdc702a4204cd5e2bb01eb917fc4f5ea2315aa23962549561b0"},- {file = "yarl-1.11.1-py3-none-any.whl", hash = "sha256:72bf26f66456baa0584eff63e44545c9f0eaed9b73cb6601b647c91f14c11f38"},- {file = "yarl-1.11.1.tar.gz", hash = "sha256:1bb2d9e212fb7449b8fb73bc461b51eaa17cc8430b4a87d87be7b25052d92f53"},+ {file = "yarl-1.12.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:64c5b0f2b937fe40d0967516eee5504b23cb247b8b7ffeba7213a467d9646fdc"},+ {file = "yarl-1.12.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2e430ac432f969ef21770645743611c1618362309e3ad7cab45acd1ad1a540ff"},+ {file = "yarl-1.12.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3e26e64f42bce5ddf9002092b2c37b13071c2e6413d5c05f9fa9de58ed2f7749"},+ {file = "yarl-1.12.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0103c52f8dfe5d573c856322149ddcd6d28f51b4d4a3ee5c4b3c1b0a05c3d034"},+ {file = "yarl-1.12.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b63465b53baeaf2122a337d4ab57d6bbdd09fcadceb17a974cfa8a0300ad9c67"},+ {file = "yarl-1.12.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17d4dc4ff47893a06737b8788ed2ba2f5ac4e8bb40281c8603920f7d011d5bdd"},+ {file = "yarl-1.12.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8b54949267bd5704324397efe9fbb6aa306466dee067550964e994d309db5f1"},+ {file = "yarl-1.12.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:10b690cd78cbaca2f96a7462f303fdd2b596d3978b49892e4b05a7567c591572"},+ {file = "yarl-1.12.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c85ab016e96a975afbdb9d49ca90f3bca9920ef27c64300843fe91c3d59d8d20"},+ {file = "yarl-1.12.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:c1caa5763d1770216596e0a71b5567f27aac28c95992110212c108ec74589a48"},+ {file = "yarl-1.12.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:595bbcdbfc4a9c6989d7489dca8510cba053ff46b16c84ffd95ac8e90711d419"},+ {file = "yarl-1.12.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e64f0421892a207d3780903085c1b04efeb53b16803b23d947de5a7261b71355"},+ {file = "yarl-1.12.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:319c206e83e46ec2421b25b300c8482b6fe8a018baca246be308c736d9dab267"},+ {file = "yarl-1.12.1-cp310-cp310-win32.whl", hash = "sha256:da045bd1147d12bd43fb032296640a7cc17a7f2eaba67495988362e99db24fd2"},+ {file = "yarl-1.12.1-cp310-cp310-win_amd64.whl", hash = "sha256:aebbd47df77190ada603157f0b3670d578c110c31746ecc5875c394fdcc59a99"},+ {file = "yarl-1.12.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:28389a68981676bf74e2e199fe42f35d1aa27a9c98e3a03e6f58d2d3d054afe1"},+ {file = "yarl-1.12.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f736f54565f8dd7e3ab664fef2bc461d7593a389a7f28d4904af8d55a91bd55f"},+ {file = "yarl-1.12.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6dee0496d5f1a8f57f0f28a16f81a2033fc057a2cf9cd710742d11828f8c80e2"},+ {file = "yarl-1.12.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8981a94a27ac520a398302afb74ae2c0be1c3d2d215c75c582186a006c9e7b0"},+ {file = "yarl-1.12.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ff54340fc1129e8e181827e2234af3ff659b4f17d9bbe77f43bc19e6577fadec"},+ {file = "yarl-1.12.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:54c8cee662b5f8c30ad7eedfc26123f845f007798e4ff1001d9528fe959fd23c"},+ {file = "yarl-1.12.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e97a29b37830ba1262d8dfd48ddb5b28ad4d3ebecc5d93a9c7591d98641ec737"},+ {file = "yarl-1.12.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c89894cc6f6ddd993813e79244b36b215c14f65f9e4f1660b1f2ba9e5594b95"},+ {file = "yarl-1.12.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:712ba8722c0699daf186de089ddc4677651eb9875ed7447b2ad50697522cbdd9"},+ {file = "yarl-1.12.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6e9a9f50892153bad5046c2a6df153224aa6f0573a5a8ab44fc54a1e886f6e21"},+ {file = "yarl-1.12.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:1d4017e78fb22bc797c089b746230ad78ecd3cdb215bc0bd61cb72b5867da57e"},+ {file = "yarl-1.12.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:f494c01b28645c431239863cb17af8b8d15b93b0d697a0320d5dd34cd9d7c2fa"},+ {file = "yarl-1.12.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:de4544b1fb29cf14870c4e2b8a897c0242449f5dcebd3e0366aa0aa3cf58a23a"},+ {file = "yarl-1.12.1-cp311-cp311-win32.whl", hash = "sha256:7564525a4673fde53dee7d4c307a961c0951918f0b8c7f09b2c9e02067cf6504"},+ {file = "yarl-1.12.1-cp311-cp311-win_amd64.whl", hash = "sha256:f23bb1a7a6e8e8b612a164fdd08e683bcc16c76f928d6dbb7bdbee2374fbfee6"},+ {file = "yarl-1.12.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:30ffc046ebddccb3c4cac72c1a3e1bc343492336f3ca86d24672e90ccc5e788a"},+ {file = "yarl-1.12.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f10954b233d4df5cc3137ffa5ced97f8894152df817e5d149bf05a0ef2ab8134"},+ {file = "yarl-1.12.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2e912b282466444023610e4498e3795c10e7cfd641744524876239fcf01d538d"},+ {file = "yarl-1.12.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6af871f70cfd5b528bd322c65793b5fd5659858cdfaa35fbe563fb99b667ed1f"},+ {file = "yarl-1.12.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c3e4e1f7b08d1ec6b685ccd3e2d762219c550164fbf524498532e39f9413436e"},+ {file = "yarl-1.12.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9a7ee79183f0b17dcede8b6723e7da2ded529cf159a878214be9a5d3098f5b1e"},+ {file = "yarl-1.12.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96c8ff1e1dd680e38af0887927cab407a4e51d84a5f02ae3d6eb87233036c763"},+ {file = "yarl-1.12.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e9905fc2dc1319e4c39837b906a024cf71b1261cc66b0cd89678f779c0c61f5"},+ {file = "yarl-1.12.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:01549468858b87d36f967c97d02e6e54106f444aeb947ed76f8f71f85ed07cec"},+ {file = "yarl-1.12.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:96b34830bd6825ca0220bf005ea99ac83eb9ce51301ddb882dcf613ae6cd95fb"},+ {file = "yarl-1.12.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:2aee7594d2c2221c717a8e394bbed4740029df4c0211ceb0f04815686e99c795"},+ {file = "yarl-1.12.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:15871130439ad10abb25a4631120d60391aa762b85fcab971411e556247210a0"},+ {file = "yarl-1.12.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:838dde2cb570cfbb4cab8a876a0974e8b90973ea40b3ac27a79b8a74c8a2db15"},+ {file = "yarl-1.12.1-cp39-cp39-win32.whl", hash = "sha256:eacbcf30efaca7dc5cb264228ffecdb95fdb1e715b1ec937c0ce6b734161e0c8"},+ {file = "yarl-1.12.1-cp39-cp39-win_amd64.whl", hash = "sha256:76a59d1b63de859398bc7764c860a769499511463c1232155061fe0147f13e01"},+ {file = "yarl-1.12.1-py3-none-any.whl", hash = "sha256:dc3192a81ecd5ff954cecd690327badd5a84d00b877e1573f7c9097ce13e5bfb"},+ {file = "yarl-1.12.1.tar.gz", hash = "sha256:5b860055199aec8d6fe4dcee3c5196ce506ca198a50aab0059ffd26e8e815828"},]diff --git a/backend/workflow_manager/endpoint/destination.py b/backend/workflow_manager/endpoint/destination.pyindex 4005fc5af..27a01dd23 100644--- a/backend/workflow_manager/endpoint/destination.py+++ b/backend/workflow_manager/endpoint/destination.py@@ -37,7 +37,7 @@ from workflow_manager.workflow.models.file_history import FileHistoryfrom workflow_manager.workflow.models.workflow import Workflowfrom backend.exceptions import UnstractFSException-from unstract.connectors.exceptions import AzureHttpError+from unstract.connectors.exceptions import ConnectorErrorlogger = logging.getLogger(__name__)@@ -224,19 +224,17 @@ class DestinationConnector(BaseConnector):try:destination_fs.create_dir_if_not_exists(input_dir=output_directory)- destination_fsspec = destination_fs.get_fsspec_fs()# Traverse local directory and create the same structure in the# output_directoryfor root, dirs, files in os.walk(destination_volume_path):for dir_name in dirs:- destination_fsspec.mkdir(- os.path.join(- output_directory,- os.path.relpath(root, destination_volume_path),- dir_name,- )+ current_dir = os.path.join(+ output_directory,+ os.path.relpath(root, destination_volume_path),+ dir_name,)+ destination_fs.create_dir_if_not_exists(input_dir=current_dir)for file_name in files:source_path = os.path.join(root, file_name)@@ -248,7 +246,7 @@ class DestinationConnector(BaseConnector):destination_fs.upload_file_to_storage(source_path=source_path, destination_path=destination_path)- except AzureHttpError as e:+ except ConnectorError as e:raise UnstractFSException(core_err=e) from edef insert_into_db(self, input_file_path: str) -> None:diff --git a/docker/connector-db/README.md b/docker/connector-db/README.mddeleted file mode 100644index 44d445814..000000000--- a/docker/connector-db/README.md+++ /dev/null@@ -1,13 +0,0 @@-# Docker Run--Useful while running database connectors like mysql, mariadb, mssql--**NOTE**: Copy `sample.env` into `.env` and update the necessary variables.--```bash-# Up all connector db-docker compose -f docker-compose-db-connector.yaml up -d--# Up a specific service alone-docker compose -f docker-compose-db-connector.yaml up -d mysql-```diff --git a/docker/connector/README.md b/docker/connector/README.mdnew file mode 100644index 000000000..0ba69fb59--- /dev/null+++ b/docker/connector/README.md@@ -0,0 +1,15 @@+# Docker Run++Useful while running connectors+- Database connectors: Includes db connectors like mysql, mariadb, mssql+- File system connectors: Includes fs connectors like sftp++**NOTE**: Copy `sample.env` into `.env` and update the necessary variables.++```bash+# Up all connector db+docker compose -f docker-compose-connector.yaml up -d++# Up a specific service alone+docker compose -f docker-compose-connector.yaml up -d mysql+```diff --git a/docker/connector-db/docker-compose-db-connector.yaml b/docker/connector/docker-compose-connector.yamlsimilarity index 77%rename from docker/connector-db/docker-compose-db-connector.yamlrename to docker/connector/docker-compose-connector.yamlindex cf7a2389c..b07fa5ed9 100644--- a/docker/connector-db/docker-compose-db-connector.yaml+++ b/docker/connector/docker-compose-connector.yaml@@ -31,7 +31,16 @@ services:volumes:- mssql_data:/var/opt/mssql+ sftp:+ image: atmoz/sftp:debian+ ports:+ - "22:22"+ volumes:+ - sftp_data:/home/foo+ command: "${SFTP_USER}:${SFTP_PASS}:${SFTP_USER_ID}::${SFTP_DIR}"+volumes:mariadb_data:mysql_data:mssql_data:+ sftp_data:diff --git a/docker/connector-db/sample.env b/docker/connector/sample.envsimilarity index 73%rename from docker/connector-db/sample.envrename to docker/connector/sample.envindex c51ddc5a6..d6b160d41 100644--- a/docker/connector-db/sample.env+++ b/docker/connector/sample.env@@ -4,3 +4,7 @@ MYSQL_USER=unstract_devMYSQL_PASSWORD=unstract_passACCEPT_EULA=YMSSQL_SA_PASSWORD=unstractPassword@123+SFTP_USER=foo+SFTP_PASS=pass+SFTP_USER_ID=1001+SFTP_DIR=uploaddiff --git a/frontend/public/icons/connector-icons/SFTP.png b/frontend/public/icons/connector-icons/SFTP.pngnew file mode 100644index 000000000..f50fe5f4cBinary files /dev/null and b/frontend/public/icons/connector-icons/SFTP.png differdiff --git a/unstract/connectors/pyproject.toml b/unstract/connectors/pyproject.tomlindex e353be043..1f51c257c 100644--- a/unstract/connectors/pyproject.toml+++ b/unstract/connectors/pyproject.toml@@ -8,6 +8,7 @@ name = "unstract-connectors"version = "0.0.3"description = "All connectors that are part of the Unstract platform"dependencies = [+ "singleton-decorator~=1.0.0",# For GCSHelper"google-auth==2.20.0","google-cloud-secret-manager==2.16.1",@@ -25,7 +26,9 @@ dependencies = ["snowflake-connector-python[pandas]~=3.6.0", # For Snowflake"google-cloud-bigquery==3.11.4", # For Bigquery"pymssql==2.2.8", # For MSSQL- "PyMySQL==1.1.0" # For MySQL+ "PyMySQL==1.1.0", # For MySQL+ "fsspec[sftp]~=2024.6.0", # For SFTP+]readme = "README.md"urls = { Homepage = "https://unstract.com", Source = "https://github.com/Zipstack/unstract/tree/main/unstract/connectors" }diff --git a/unstract/connectors/src/unstract/connectors/connectorkit.py b/unstract/connectors/src/unstract/connectors/connectorkit.pyindex 5b1d574c1..69ea233d9 100644--- a/unstract/connectors/src/unstract/connectors/connectorkit.py+++ b/unstract/connectors/src/unstract/connectors/connectorkit.py@@ -1,6 +1,8 @@import loggingfrom typing import Any, Optional+from singleton_decorator import singleton+from unstract.connectors import ConnectorDict # type: ignorefrom unstract.connectors.base import UnstractConnectorfrom unstract.connectors.constants import Common@@ -12,6 +14,7 @@ from unstract.connectors.queues import connectors as q_connectorslogger = logging.getLogger(__name__)+@singletonclass Connectorkit:def __init__(self) -> None:self._connectors: ConnectorDict = fs_connectors | db_connectors | q_connectorsdiff --git a/unstract/connectors/src/unstract/connectors/exceptions.py b/unstract/connectors/src/unstract/connectors/exceptions.pyindex def0cd90e..cfeeb6f4c 100644--- a/unstract/connectors/src/unstract/connectors/exceptions.py+++ b/unstract/connectors/src/unstract/connectors/exceptions.py@@ -47,3 +47,11 @@ class AzureHttpError(ConnectorError):Args:ConnectorError (Class): Inherits class ConnectorError"""+++class PermissionDeniedError(ConnectorError):+ """Handles permission denied error from fs connector.++ Args:+ ConnectorError (Class): Inherits class ConnectorError+ """diff --git a/unstract/connectors/src/unstract/connectors/filesystems/sftp/__init__.py b/unstract/connectors/src/unstract/connectors/filesystems/sftp/__init__.pynew file mode 100644index 000000000..7e887f0e8--- /dev/null+++ b/unstract/connectors/src/unstract/connectors/filesystems/sftp/__init__.py@@ -0,0 +1,11 @@+from .sftp import SftpFS++__all__ = ["SftpFS"]++metadata = {+ "name": SftpFS.__name__,+ "version": "1.0.0",+ "connector": SftpFS,+ "description": "SftpFS connector",+ "is_active": True,+}diff --git a/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py b/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.pynew file mode 100644index 000000000..feef4d62a--- /dev/null+++ b/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py@@ -0,0 +1,143 @@+import logging+import os+from typing import Any++from fsspec.implementations.sftp import SFTPFileSystem++from unstract.connectors.exceptions import ConnectorError, PermissionDeniedError+from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem++logger = logging.getLogger(__name__)+++class SettingsKey:+ HOST = "host"+ PORT = "port"+ USERNAME = "username"+ PASSWORD = "password"+ USER_DIRECTORY = "user_dir"+++class SettingsDefault:+ PORT = 22+++class SftpFS(UnstractFileSystem):+ def __init__(self, settings: dict[str, Any]):+ super().__init__("SFTP")+ host = settings.get(SettingsKey.HOST)+ port = int(settings.get(SettingsKey.PORT, SettingsDefault.PORT))+ username = settings.get(SettingsKey.USERNAME)+ password = settings.get(SettingsKey.PASSWORD)+ self.directory = str(settings.get(SettingsKey.USER_DIRECTORY))++ self.sftp_fs = SFTPFileSystem(+ host=host,+ port=port,+ username=username,+ password=password,+ )++ @staticmethod+ def get_id() -> str:+ return "sftp|e68fa828-2988-4cce-9e3d-4285348e3227"++ @staticmethod+ def get_name() -> str:+ return "SFTP/SSH"++ @staticmethod+ def get_description() -> str:+ return "Fetch data via SFTP/SSH"++ @staticmethod+ def get_icon() -> str:+ return "/icons/connector-icons/SFTP.png"++ @staticmethod+ def can_write() -> bool:+ return True++ @staticmethod+ def can_read() -> bool:+ return True++ def get_fsspec_fs(self) -> SFTPFileSystem:+ return self.sftp_fs++ # TODO: Check if this method can be removed, and use it from parent class+ # (class UnstractFileSystem)+ @staticmethod+ def get_json_schema() -> str:+ f = open(f"{os.path.dirname(__file__)}/static/json_schema.json")+ schema = f.read()+ f.close()+ return schema++ # TODO: Check if this method can be removed, and use it from parent class+ # (class UnstractFileSystem)+ @staticmethod+ def requires_oauth() -> bool:+ return False++ # TODO: Check if this method can be removed, and use it from parent class+ # (class UnstractFileSystem)+ @staticmethod+ def python_social_auth_backend() -> str:+ return ""++ def test_credentials(self) -> bool:+ """To test credentials for SFTP."""+ is_dir = False+ user_dir = f"{self.directory.strip('/')}/"+ try:+ is_dir = bool(self.get_fsspec_fs().isdir(user_dir))+ except Exception as e:+ raise ConnectorError(+ f"Error while connecting to SFTP server: {str(e)}"+ ) from e+ if not is_dir:+ raise ConnectorError(+ "Unable to connect to SFTP server, "+ "please check the connection settings."+ )+ return True++ def create_dir_if_not_exists(self, input_dir: str) -> None:+ """Method to create dir of a connector if not exists.++ Args:+ input_dir (str): input directory of source connector+ """+ try:+ super().create_dir_if_not_exists(input_dir=input_dir)+ except PermissionError as e:+ self.raise_permission_error(input_dir=input_dir, error=e)++ def upload_file_to_storage(self, source_path: str, destination_path: str) -> None:+ """Method to upload filepath from tool to sftp connector directory.++ Args:+ source_path (str): local path of file to be uploaded, coming from tool+ destination_path (str): target path in the storage where the file will be+ uploaded+ """+ try:+ normalized_path = os.path.normpath(destination_path)+ super().upload_file_to_storage(+ source_path=source_path, destination_path=destination_path+ )+ except PermissionError as e:+ self.raise_permission_error(input_dir=normalized_path, error=e)++ def raise_permission_error(+ self, input_dir: str, error: PermissionError+ ) -> PermissionDeniedError:+ user_message = (+ "Please verify your SFTP credentials and ensure you "+ f" ensure you have the necessary permissions for the path '{input_dir}'. "+ )+ raise PermissionDeniedError(+ user_message,+ treat_as_user_message=True,+ ) from errordiff --git a/unstract/connectors/src/unstract/connectors/filesystems/sftp/static/json_schema.json b/unstract/connectors/src/unstract/connectors/filesystems/sftp/static/json_schema.jsonnew file mode 100644index 000000000..9a131f030--- /dev/null+++ b/unstract/connectors/src/unstract/connectors/filesystems/sftp/static/json_schema.json@@ -0,0 +1,48 @@+{+ "title": "SFTP",+ "description": "A connector that allows for fetching data via SFTP",+ "type": "object",+ "required": [+ "connectorName",+ "host",+ "port",+ "username",+ "password",+ "user_dir"++ ],+ "properties": {+ "connectorName": {+ "type": "string",+ "title": "Name of the connector",+ "default": "Unstract's SFTP connector"+ },+ "host": {+ "type": "string",+ "title": "Host/IP",+ "description": "The hostname or IP address of the SFTP server. Example: `sftp.example.com` or `192.168.1.10`."+ },+ "port": {+ "type": "string",+ "title": "Port",+ "description": "The port number to connect to the SFTP server. Example: `22` (default SSH port).",+ "default": "22"+ },+ "username": {+ "type": "string",+ "title": "Username",+ "description": "The username used to authenticate with the SFTP server"+ },+ "password": {+ "type": "string",+ "title": "Password",+ "description": "The password used to authenticate with the SFTP server",+ "format": "password"+ },+ "user_dir": {+ "type": "string",+ "title": "User directory",+ "description": "The directory that the user has access to"+ }+ }+}diff --git a/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py b/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.pyindex 73f8a38b4..75389c772 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py@@ -93,8 +93,9 @@ class UnstractFileSystem(UnstractConnector, ABC):directory.Args:- source_path (str): _description_- destination_path (str): _description_+ source_path (str): local path of file to be uploaded, coming from tool+ destination_path (str): target path in the storage where the file will be+ uploaded"""normalized_path = os.path.normpath(destination_path)fs = self.get_fsspec_fs()