Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

feat: Added an SFTP connector, (#674)

Chandrasekharan Md60868a0cd38
diff --git a/backend/pdm.lock b/backend/pdm.lock
index 656514b42..378cebd6a 100644
--- a/backend/pdm.lock
+++ b/backend/pdm.lock
@@ -4,8 +4,8 @@
[metadata]
groups = ["default", "deploy", "dev", "test"]
strategy = ["cross_platform", "inherit_metadata"]
-lock_version = "4.4.1"
-content_hash = "sha256:dcf5f16cb2201a698efff3daf588c5dea4bda976aae3a86054780e01d0548549"
+lock_version = "4.4.2"
+content_hash = "sha256:2a64b25fbdae60bb1332d0486b0f4296d4b338fc8b72057694e9d4d47ad71dcb"
[[package]]
name = "adlfs"
@@ -437,6 +437,42 @@ files = [
{file = "azure_storage_blob-12.23.0.tar.gz", hash = "sha256:2fadbceda1d99c4a72dfd32e0122d7bca8b5e8d2563f5c624d634aeaff49c9df"},
]
+[[package]]
+name = "bcrypt"
+version = "4.2.0"
+requires_python = ">=3.7"
+summary = "Modern password hashing for your software and your servers"
+groups = ["default", "dev"]
+files = [
+ {file = "bcrypt-4.2.0-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:096a15d26ed6ce37a14c1ac1e48119660f21b24cba457f160a4b830f3fe6b5cb"},
+ {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c02d944ca89d9b1922ceb8a46460dd17df1ba37ab66feac4870f6862a1533c00"},
+ {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d84cf6d877918620b687b8fd1bf7781d11e8a0998f576c7aa939776b512b98d"},
+ {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:1bb429fedbe0249465cdd85a58e8376f31bb315e484f16e68ca4c786dcc04291"},
+ {file = "bcrypt-4.2.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:655ea221910bcac76ea08aaa76df427ef8625f92e55a8ee44fbf7753dbabb328"},
+ {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:1ee38e858bf5d0287c39b7a1fc59eec64bbf880c7d504d3a06a96c16e14058e7"},
+ {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:0da52759f7f30e83f1e30a888d9163a81353ef224d82dc58eb5bb52efcabc399"},
+ {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3698393a1b1f1fd5714524193849d0c6d524d33523acca37cd28f02899285060"},
+ {file = "bcrypt-4.2.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:762a2c5fb35f89606a9fde5e51392dad0cd1ab7ae64149a8b935fe8d79dd5ed7"},
+ {file = "bcrypt-4.2.0-cp37-abi3-win32.whl", hash = "sha256:5a1e8aa9b28ae28020a3ac4b053117fb51c57a010b9f969603ed885f23841458"},
+ {file = "bcrypt-4.2.0-cp37-abi3-win_amd64.whl", hash = "sha256:8f6ede91359e5df88d1f5c1ef47428a4420136f3ce97763e31b86dd8280fbdf5"},
+ {file = "bcrypt-4.2.0-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:c52aac18ea1f4a4f65963ea4f9530c306b56ccd0c6f8c8da0c06976e34a6e841"},
+ {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3bbbfb2734f0e4f37c5136130405332640a1e46e6b23e000eeff2ba8d005da68"},
+ {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3413bd60460f76097ee2e0a493ccebe4a7601918219c02f503984f0a7ee0aebe"},
+ {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:8d7bb9c42801035e61c109c345a28ed7e84426ae4865511eb82e913df18f58c2"},
+ {file = "bcrypt-4.2.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3d3a6d28cb2305b43feac298774b997e372e56c7c7afd90a12b3dc49b189151c"},
+ {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:9c1c4ad86351339c5f320ca372dfba6cb6beb25e8efc659bedd918d921956bae"},
+ {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:27fe0f57bb5573104b5a6de5e4153c60814c711b29364c10a75a54bb6d7ff48d"},
+ {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:8ac68872c82f1add6a20bd489870c71b00ebacd2e9134a8aa3f98a0052ab4b0e"},
+ {file = "bcrypt-4.2.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cb2a8ec2bc07d3553ccebf0746bbf3d19426d1c6d1adbd4fa48925f66af7b9e8"},
+ {file = "bcrypt-4.2.0-cp39-abi3-win32.whl", hash = "sha256:77800b7147c9dc905db1cba26abe31e504d8247ac73580b4aa179f98e6608f34"},
+ {file = "bcrypt-4.2.0-cp39-abi3-win_amd64.whl", hash = "sha256:61ed14326ee023917ecd093ee6ef422a72f3aec6f07e21ea5f10622b735538a9"},
+ {file = "bcrypt-4.2.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:39e1d30c7233cfc54f5c3f2c825156fe044efdd3e0b9d309512cc514a263ec2a"},
+ {file = "bcrypt-4.2.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:f4f4acf526fcd1c34e7ce851147deedd4e26e6402369304220250598b26448db"},
+ {file = "bcrypt-4.2.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:1ff39b78a52cf03fdf902635e4c81e544714861ba3f0efc56558979dd4f09170"},
+ {file = "bcrypt-4.2.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:373db9abe198e8e2c70d12b479464e0d5092cc122b20ec504097b5f2297ed184"},
+ {file = "bcrypt-4.2.0.tar.gz", hash = "sha256:cf69eaf5185fd58f268f805b505ce31f9b9fc2d64b376642164e9244540c1221"},
+]
+
[[package]]
name = "beautifulsoup4"
version = "4.12.3"
@@ -1247,6 +1283,22 @@ files = [
{file = "fsspec-2024.6.0.tar.gz", hash = "sha256:f579960a56e6d8038a9efc8f9c77279ec12e6299aa86b0769a7e9c46b94527c2"},
]
+[[package]]
+name = "fsspec"
+version = "2024.6.0"
+extras = ["sftp"]
+requires_python = ">=3.8"
+summary = "File-system specification"
+groups = ["default", "dev"]
+dependencies = [
+ "fsspec==2024.6.0",
+ "paramiko",
+]
+files = [
+ {file = "fsspec-2024.6.0-py3-none-any.whl", hash = "sha256:58d7122eb8a1a46f7f13453187bfea4972d66bf01618d37366521b1998034cee"},
+ {file = "fsspec-2024.6.0.tar.gz", hash = "sha256:f579960a56e6d8038a9efc8f9c77279ec12e6299aa86b0769a7e9c46b94527c2"},
+]
+
[[package]]
name = "funcy"
version = "2.0"
@@ -1885,7 +1937,7 @@ files = [
[[package]]
name = "huggingface-hub"
-version = "0.25.0"
+version = "0.25.1"
requires_python = ">=3.8.0"
summary = "Client library to download and publish models, datasets and other repos on the huggingface.co hub"
groups = ["default", "dev"]
@@ -1899,8 +1951,8 @@ dependencies = [
"typing-extensions>=3.7.4.3",
]
files = [
- {file = "huggingface_hub-0.25.0-py3-none-any.whl", hash = "sha256:e2f357b35d72d5012cfd127108c4e14abcd61ba4ebc90a5a374dc2456cb34e12"},
- {file = "huggingface_hub-0.25.0.tar.gz", hash = "sha256:fb5fbe6c12fcd99d187ec7db95db9110fb1a20505f23040a5449a717c1a0db4d"},
+ {file = "huggingface_hub-0.25.1-py3-none-any.whl", hash = "sha256:a5158ded931b3188f54ea9028097312cb0acd50bffaaa2612014c3c526b44972"},
+ {file = "huggingface_hub-0.25.1.tar.gz", hash = "sha256:9ff7cb327343211fbd06e2b149b8f362fd1e389454f3f14c6db75a4999ee20ff"},
]
[[package]]
@@ -2088,7 +2140,7 @@ files = [
[[package]]
name = "llama-cloud"
-version = "0.0.17"
+version = "0.1.0"
requires_python = "<4,>=3.8"
summary = ""
groups = ["default", "dev"]
@@ -2097,8 +2149,8 @@ dependencies = [
"pydantic>=1.10",
]
files = [
- {file = "llama_cloud-0.0.17-py3-none-any.whl", hash = "sha256:da898dcc98de84f29886f979b1ccae1e96d9f73d1b0e07146a51d315b161e45c"},
- {file = "llama_cloud-0.0.17.tar.gz", hash = "sha256:7fd6857bbbb91937535572ccb48daa38189f55cdd7411185d8083dab29ba1299"},
+ {file = "llama_cloud-0.1.0-py3-none-any.whl", hash = "sha256:e315046d856780d996886864cd315d5f216cd398b3bd3f005fcae393c66f5494"},
+ {file = "llama_cloud-0.1.0.tar.gz", hash = "sha256:8b31e1fb87f48c397c1a6415816cf37afcb4abe9fe7a5eb4101a2dc6d5c6ebbc"},
]
[[package]]
@@ -2908,7 +2960,7 @@ files = [
[[package]]
name = "openai"
-version = "1.47.0"
+version = "1.47.1"
requires_python = ">=3.7.1"
summary = "The official Python library for the openai API"
groups = ["default", "dev"]
@@ -2923,8 +2975,8 @@ dependencies = [
"typing-extensions<5,>=4.11",
]
files = [
- {file = "openai-1.47.0-py3-none-any.whl", hash = "sha256:9ccc8737dfa791f7bd903db4758c176b8544a8cd89d3a3d2add3cea02a34c3a0"},
- {file = "openai-1.47.0.tar.gz", hash = "sha256:6e14d6f77c8cf546646afcd87a2ef752505b3710d2564a2e433e17307dfa86a0"},
+ {file = "openai-1.47.1-py3-none-any.whl", hash = "sha256:34277583bf268bb2494bc03f48ac123788c5e2a914db1d5a23d5edc29d35c825"},
+ {file = "openai-1.47.1.tar.gz", hash = "sha256:62c8f5f478f82ffafc93b33040f8bb16a45948306198bd0cba2da2ecd9cf7323"},
]
[[package]]
@@ -3013,6 +3065,22 @@ files = [
{file = "pandas-2.1.4.tar.gz", hash = "sha256:fcb68203c833cc735321512e13861358079a96c174a61f5116a1de89c58c0ef7"},
]
+[[package]]
+name = "paramiko"
+version = "3.5.0"
+requires_python = ">=3.6"
+summary = "SSH2 protocol library"
+groups = ["default", "dev"]
+dependencies = [
+ "bcrypt>=3.2",
+ "cryptography>=3.3",
+ "pynacl>=1.5",
+]
+files = [
+ {file = "paramiko-3.5.0-py3-none-any.whl", hash = "sha256:1fedf06b085359051cd7d0d270cebe19e755a8a921cc2ddbfa647fb0cd7d68f9"},
+ {file = "paramiko-3.5.0.tar.gz", hash = "sha256:ad11e540da4f55cedda52931f1a3f812a8238a7af7f62a60de538cd80bb28124"},
+]
+
[[package]]
name = "pdfminer-six"
version = "20231228"
@@ -3549,6 +3617,28 @@ files = [
{file = "PyMySQL-1.1.0.tar.gz", hash = "sha256:4f13a7df8bf36a51e81dd9f3605fede45a4878fe02f9236349fd82a3f0612f96"},
]
+[[package]]
+name = "pynacl"
+version = "1.5.0"
+requires_python = ">=3.6"
+summary = "Python binding to the Networking and Cryptography (NaCl) library"
+groups = ["default", "dev"]
+dependencies = [
+ "cffi>=1.4.1",
+]
+files = [
+ {file = "PyNaCl-1.5.0-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1"},
+ {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92"},
+ {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394"},
+ {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:0c84947a22519e013607c9be43706dd42513f9e6ae5d39d3613ca1e142fba44d"},
+ {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06b8f6fa7f5de8d5d2f7573fe8c863c051225a27b61e6860fd047b1775807858"},
+ {file = "PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b"},
+ {file = "PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:61f642bf2378713e2c2e1de73444a3778e5f0a38be6fee0fe532fe30060282ff"},
+ {file = "PyNaCl-1.5.0-cp36-abi3-win32.whl", hash = "sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543"},
+ {file = "PyNaCl-1.5.0-cp36-abi3-win_amd64.whl", hash = "sha256:20f42270d27e1b6a29f54032090b972d97f0a1b0948cc52392041ef7831fee93"},
+ {file = "PyNaCl-1.5.0.tar.gz", hash = "sha256:8ac7448f09ab85811607bdd21ec2464495ac8b7c66d146bf545b0f08fb9220ba"},
+]
+
[[package]]
name = "pyopenssl"
version = "23.3.0"
@@ -4717,13 +4807,13 @@ files = [
[[package]]
name = "tzdata"
-version = "2024.1"
+version = "2024.2"
requires_python = ">=2"
summary = "Provider of IANA time zone data"
groups = ["default", "dev"]
files = [
- {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"},
- {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"},
+ {file = "tzdata-2024.2-py2.py3-none-any.whl", hash = "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd"},
+ {file = "tzdata-2024.2.tar.gz", hash = "sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc"},
]
[[package]]
@@ -4797,6 +4887,7 @@ dependencies = [
"adlfs==2023.8.0",
"boxfs==0.2.1",
"dropboxdrivefs==1.4.1",
+ "fsspec[sftp]~=2024.6.0",
"gcsfs==2024.6.0",
"google-auth==2.20.0",
"google-cloud-bigquery==3.11.4",
@@ -4806,6 +4897,7 @@ dependencies = [
"psycopg2-binary==2.9.9",
"pymssql==2.2.8",
"s3fs[boto3]==2024.6.0",
+ "singleton-decorator~=1.0.0",
"snowflake-connector-python[pandas]~=3.6.0",
]
@@ -5070,7 +5162,7 @@ files = [
[[package]]
name = "yarl"
-version = "1.11.1"
+version = "1.12.1"
requires_python = ">=3.8"
summary = "Yet another URL library"
groups = ["default", "dev"]
@@ -5079,51 +5171,51 @@ dependencies = [
"multidict>=4.0",
]
files = [
- {file = "yarl-1.11.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:400cd42185f92de559d29eeb529e71d80dfbd2f45c36844914a4a34297ca6f00"},
- {file = "yarl-1.11.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8258c86f47e080a258993eed877d579c71da7bda26af86ce6c2d2d072c11320d"},
- {file = "yarl-1.11.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2164cd9725092761fed26f299e3f276bb4b537ca58e6ff6b252eae9631b5c96e"},
- {file = "yarl-1.11.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08ea567c16f140af8ddc7cb58e27e9138a1386e3e6e53982abaa6f2377b38cc"},
- {file = "yarl-1.11.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:768ecc550096b028754ea28bf90fde071c379c62c43afa574edc6f33ee5daaec"},
- {file = "yarl-1.11.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2909fa3a7d249ef64eeb2faa04b7957e34fefb6ec9966506312349ed8a7e77bf"},
- {file = "yarl-1.11.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01a8697ec24f17c349c4f655763c4db70eebc56a5f82995e5e26e837c6eb0e49"},
- {file = "yarl-1.11.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e286580b6511aac7c3268a78cdb861ec739d3e5a2a53b4809faef6b49778eaff"},
- {file = "yarl-1.11.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4179522dc0305c3fc9782549175c8e8849252fefeb077c92a73889ccbcd508ad"},
- {file = "yarl-1.11.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:27fcb271a41b746bd0e2a92182df507e1c204759f460ff784ca614e12dd85145"},
- {file = "yarl-1.11.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:f61db3b7e870914dbd9434b560075e0366771eecbe6d2b5561f5bc7485f39efd"},
- {file = "yarl-1.11.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:c92261eb2ad367629dc437536463dc934030c9e7caca861cc51990fe6c565f26"},
- {file = "yarl-1.11.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d95b52fbef190ca87d8c42f49e314eace4fc52070f3dfa5f87a6594b0c1c6e46"},
- {file = "yarl-1.11.1-cp310-cp310-win32.whl", hash = "sha256:489fa8bde4f1244ad6c5f6d11bb33e09cf0d1d0367edb197619c3e3fc06f3d91"},
- {file = "yarl-1.11.1-cp310-cp310-win_amd64.whl", hash = "sha256:476e20c433b356e16e9a141449f25161e6b69984fb4cdbd7cd4bd54c17844998"},
- {file = "yarl-1.11.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:946eedc12895873891aaceb39bceb484b4977f70373e0122da483f6c38faaa68"},
- {file = "yarl-1.11.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:21a7c12321436b066c11ec19c7e3cb9aec18884fe0d5b25d03d756a9e654edfe"},
- {file = "yarl-1.11.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c35f493b867912f6fda721a59cc7c4766d382040bdf1ddaeeaa7fa4d072f4675"},
- {file = "yarl-1.11.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25861303e0be76b60fddc1250ec5986c42f0a5c0c50ff57cc30b1be199c00e63"},
- {file = "yarl-1.11.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e4b53f73077e839b3f89c992223f15b1d2ab314bdbdf502afdc7bb18e95eae27"},
- {file = "yarl-1.11.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:327c724b01b8641a1bf1ab3b232fb638706e50f76c0b5bf16051ab65c868fac5"},
- {file = "yarl-1.11.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4307d9a3417eea87715c9736d050c83e8c1904e9b7aada6ce61b46361b733d92"},
- {file = "yarl-1.11.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:48a28bed68ab8fb7e380775f0029a079f08a17799cb3387a65d14ace16c12e2b"},
- {file = "yarl-1.11.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:067b961853c8e62725ff2893226fef3d0da060656a9827f3f520fb1d19b2b68a"},
- {file = "yarl-1.11.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8215f6f21394d1f46e222abeb06316e77ef328d628f593502d8fc2a9117bde83"},
- {file = "yarl-1.11.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:498442e3af2a860a663baa14fbf23fb04b0dd758039c0e7c8f91cb9279799bff"},
- {file = "yarl-1.11.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:69721b8effdb588cb055cc22f7c5105ca6fdaa5aeb3ea09021d517882c4a904c"},
- {file = "yarl-1.11.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1e969fa4c1e0b1a391f3fcbcb9ec31e84440253325b534519be0d28f4b6b533e"},
- {file = "yarl-1.11.1-cp311-cp311-win32.whl", hash = "sha256:7d51324a04fc4b0e097ff8a153e9276c2593106a811704025bbc1d6916f45ca6"},
- {file = "yarl-1.11.1-cp311-cp311-win_amd64.whl", hash = "sha256:15061ce6584ece023457fb8b7a7a69ec40bf7114d781a8c4f5dcd68e28b5c53b"},
- {file = "yarl-1.11.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:884eab2ce97cbaf89f264372eae58388862c33c4f551c15680dd80f53c89a269"},
- {file = "yarl-1.11.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8a336eaa7ee7e87cdece3cedb395c9657d227bfceb6781295cf56abcd3386a26"},
- {file = "yarl-1.11.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:87f020d010ba80a247c4abc335fc13421037800ca20b42af5ae40e5fd75e7909"},
- {file = "yarl-1.11.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:637c7ddb585a62d4469f843dac221f23eec3cbad31693b23abbc2c366ad41ff4"},
- {file = "yarl-1.11.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:48dfd117ab93f0129084577a07287376cc69c08138694396f305636e229caa1a"},
- {file = "yarl-1.11.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:75e0ae31fb5ccab6eda09ba1494e87eb226dcbd2372dae96b87800e1dcc98804"},
- {file = "yarl-1.11.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f46f81501160c28d0c0b7333b4f7be8983dbbc161983b6fb814024d1b4952f79"},
- {file = "yarl-1.11.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:04293941646647b3bfb1719d1d11ff1028e9c30199509a844da3c0f5919dc520"},
- {file = "yarl-1.11.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:250e888fa62d73e721f3041e3a9abf427788a1934b426b45e1b92f62c1f68366"},
- {file = "yarl-1.11.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:e8f63904df26d1a66aabc141bfd258bf738b9bc7bc6bdef22713b4f5ef789a4c"},
- {file = "yarl-1.11.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:aac44097d838dda26526cffb63bdd8737a2dbdf5f2c68efb72ad83aec6673c7e"},
- {file = "yarl-1.11.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:267b24f891e74eccbdff42241c5fb4f974de2d6271dcc7d7e0c9ae1079a560d9"},
- {file = "yarl-1.11.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:6907daa4b9d7a688063ed098c472f96e8181733c525e03e866fb5db480a424df"},
- {file = "yarl-1.11.1-cp39-cp39-win32.whl", hash = "sha256:14438dfc5015661f75f85bc5adad0743678eefee266ff0c9a8e32969d5d69f74"},
- {file = "yarl-1.11.1-cp39-cp39-win_amd64.whl", hash = "sha256:94d0caaa912bfcdc702a4204cd5e2bb01eb917fc4f5ea2315aa23962549561b0"},
- {file = "yarl-1.11.1-py3-none-any.whl", hash = "sha256:72bf26f66456baa0584eff63e44545c9f0eaed9b73cb6601b647c91f14c11f38"},
- {file = "yarl-1.11.1.tar.gz", hash = "sha256:1bb2d9e212fb7449b8fb73bc461b51eaa17cc8430b4a87d87be7b25052d92f53"},
+ {file = "yarl-1.12.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:64c5b0f2b937fe40d0967516eee5504b23cb247b8b7ffeba7213a467d9646fdc"},
+ {file = "yarl-1.12.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2e430ac432f969ef21770645743611c1618362309e3ad7cab45acd1ad1a540ff"},
+ {file = "yarl-1.12.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3e26e64f42bce5ddf9002092b2c37b13071c2e6413d5c05f9fa9de58ed2f7749"},
+ {file = "yarl-1.12.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0103c52f8dfe5d573c856322149ddcd6d28f51b4d4a3ee5c4b3c1b0a05c3d034"},
+ {file = "yarl-1.12.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b63465b53baeaf2122a337d4ab57d6bbdd09fcadceb17a974cfa8a0300ad9c67"},
+ {file = "yarl-1.12.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17d4dc4ff47893a06737b8788ed2ba2f5ac4e8bb40281c8603920f7d011d5bdd"},
+ {file = "yarl-1.12.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8b54949267bd5704324397efe9fbb6aa306466dee067550964e994d309db5f1"},
+ {file = "yarl-1.12.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:10b690cd78cbaca2f96a7462f303fdd2b596d3978b49892e4b05a7567c591572"},
+ {file = "yarl-1.12.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c85ab016e96a975afbdb9d49ca90f3bca9920ef27c64300843fe91c3d59d8d20"},
+ {file = "yarl-1.12.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:c1caa5763d1770216596e0a71b5567f27aac28c95992110212c108ec74589a48"},
+ {file = "yarl-1.12.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:595bbcdbfc4a9c6989d7489dca8510cba053ff46b16c84ffd95ac8e90711d419"},
+ {file = "yarl-1.12.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e64f0421892a207d3780903085c1b04efeb53b16803b23d947de5a7261b71355"},
+ {file = "yarl-1.12.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:319c206e83e46ec2421b25b300c8482b6fe8a018baca246be308c736d9dab267"},
+ {file = "yarl-1.12.1-cp310-cp310-win32.whl", hash = "sha256:da045bd1147d12bd43fb032296640a7cc17a7f2eaba67495988362e99db24fd2"},
+ {file = "yarl-1.12.1-cp310-cp310-win_amd64.whl", hash = "sha256:aebbd47df77190ada603157f0b3670d578c110c31746ecc5875c394fdcc59a99"},
+ {file = "yarl-1.12.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:28389a68981676bf74e2e199fe42f35d1aa27a9c98e3a03e6f58d2d3d054afe1"},
+ {file = "yarl-1.12.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f736f54565f8dd7e3ab664fef2bc461d7593a389a7f28d4904af8d55a91bd55f"},
+ {file = "yarl-1.12.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6dee0496d5f1a8f57f0f28a16f81a2033fc057a2cf9cd710742d11828f8c80e2"},
+ {file = "yarl-1.12.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8981a94a27ac520a398302afb74ae2c0be1c3d2d215c75c582186a006c9e7b0"},
+ {file = "yarl-1.12.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ff54340fc1129e8e181827e2234af3ff659b4f17d9bbe77f43bc19e6577fadec"},
+ {file = "yarl-1.12.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:54c8cee662b5f8c30ad7eedfc26123f845f007798e4ff1001d9528fe959fd23c"},
+ {file = "yarl-1.12.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e97a29b37830ba1262d8dfd48ddb5b28ad4d3ebecc5d93a9c7591d98641ec737"},
+ {file = "yarl-1.12.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c89894cc6f6ddd993813e79244b36b215c14f65f9e4f1660b1f2ba9e5594b95"},
+ {file = "yarl-1.12.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:712ba8722c0699daf186de089ddc4677651eb9875ed7447b2ad50697522cbdd9"},
+ {file = "yarl-1.12.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6e9a9f50892153bad5046c2a6df153224aa6f0573a5a8ab44fc54a1e886f6e21"},
+ {file = "yarl-1.12.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:1d4017e78fb22bc797c089b746230ad78ecd3cdb215bc0bd61cb72b5867da57e"},
+ {file = "yarl-1.12.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:f494c01b28645c431239863cb17af8b8d15b93b0d697a0320d5dd34cd9d7c2fa"},
+ {file = "yarl-1.12.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:de4544b1fb29cf14870c4e2b8a897c0242449f5dcebd3e0366aa0aa3cf58a23a"},
+ {file = "yarl-1.12.1-cp311-cp311-win32.whl", hash = "sha256:7564525a4673fde53dee7d4c307a961c0951918f0b8c7f09b2c9e02067cf6504"},
+ {file = "yarl-1.12.1-cp311-cp311-win_amd64.whl", hash = "sha256:f23bb1a7a6e8e8b612a164fdd08e683bcc16c76f928d6dbb7bdbee2374fbfee6"},
+ {file = "yarl-1.12.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:30ffc046ebddccb3c4cac72c1a3e1bc343492336f3ca86d24672e90ccc5e788a"},
+ {file = "yarl-1.12.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f10954b233d4df5cc3137ffa5ced97f8894152df817e5d149bf05a0ef2ab8134"},
+ {file = "yarl-1.12.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2e912b282466444023610e4498e3795c10e7cfd641744524876239fcf01d538d"},
+ {file = "yarl-1.12.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6af871f70cfd5b528bd322c65793b5fd5659858cdfaa35fbe563fb99b667ed1f"},
+ {file = "yarl-1.12.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c3e4e1f7b08d1ec6b685ccd3e2d762219c550164fbf524498532e39f9413436e"},
+ {file = "yarl-1.12.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9a7ee79183f0b17dcede8b6723e7da2ded529cf159a878214be9a5d3098f5b1e"},
+ {file = "yarl-1.12.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96c8ff1e1dd680e38af0887927cab407a4e51d84a5f02ae3d6eb87233036c763"},
+ {file = "yarl-1.12.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e9905fc2dc1319e4c39837b906a024cf71b1261cc66b0cd89678f779c0c61f5"},
+ {file = "yarl-1.12.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:01549468858b87d36f967c97d02e6e54106f444aeb947ed76f8f71f85ed07cec"},
+ {file = "yarl-1.12.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:96b34830bd6825ca0220bf005ea99ac83eb9ce51301ddb882dcf613ae6cd95fb"},
+ {file = "yarl-1.12.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:2aee7594d2c2221c717a8e394bbed4740029df4c0211ceb0f04815686e99c795"},
+ {file = "yarl-1.12.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:15871130439ad10abb25a4631120d60391aa762b85fcab971411e556247210a0"},
+ {file = "yarl-1.12.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:838dde2cb570cfbb4cab8a876a0974e8b90973ea40b3ac27a79b8a74c8a2db15"},
+ {file = "yarl-1.12.1-cp39-cp39-win32.whl", hash = "sha256:eacbcf30efaca7dc5cb264228ffecdb95fdb1e715b1ec937c0ce6b734161e0c8"},
+ {file = "yarl-1.12.1-cp39-cp39-win_amd64.whl", hash = "sha256:76a59d1b63de859398bc7764c860a769499511463c1232155061fe0147f13e01"},
+ {file = "yarl-1.12.1-py3-none-any.whl", hash = "sha256:dc3192a81ecd5ff954cecd690327badd5a84d00b877e1573f7c9097ce13e5bfb"},
+ {file = "yarl-1.12.1.tar.gz", hash = "sha256:5b860055199aec8d6fe4dcee3c5196ce506ca198a50aab0059ffd26e8e815828"},
]
diff --git a/backend/workflow_manager/endpoint/destination.py b/backend/workflow_manager/endpoint/destination.py
index 4005fc5af..27a01dd23 100644
--- a/backend/workflow_manager/endpoint/destination.py
+++ b/backend/workflow_manager/endpoint/destination.py
@@ -37,7 +37,7 @@ from workflow_manager.workflow.models.file_history import FileHistory
from workflow_manager.workflow.models.workflow import Workflow
from backend.exceptions import UnstractFSException
-from unstract.connectors.exceptions import AzureHttpError
+from unstract.connectors.exceptions import ConnectorError
logger = logging.getLogger(__name__)
@@ -224,19 +224,17 @@ class DestinationConnector(BaseConnector):
try:
destination_fs.create_dir_if_not_exists(input_dir=output_directory)
- destination_fsspec = destination_fs.get_fsspec_fs()
# Traverse local directory and create the same structure in the
# output_directory
for root, dirs, files in os.walk(destination_volume_path):
for dir_name in dirs:
- destination_fsspec.mkdir(
- os.path.join(
- output_directory,
- os.path.relpath(root, destination_volume_path),
- dir_name,
- )
+ current_dir = os.path.join(
+ output_directory,
+ os.path.relpath(root, destination_volume_path),
+ dir_name,
)
+ destination_fs.create_dir_if_not_exists(input_dir=current_dir)
for file_name in files:
source_path = os.path.join(root, file_name)
@@ -248,7 +246,7 @@ class DestinationConnector(BaseConnector):
destination_fs.upload_file_to_storage(
source_path=source_path, destination_path=destination_path
)
- except AzureHttpError as e:
+ except ConnectorError as e:
raise UnstractFSException(core_err=e) from e
def insert_into_db(self, input_file_path: str) -> None:
diff --git a/docker/connector-db/README.md b/docker/connector-db/README.md
deleted file mode 100644
index 44d445814..000000000
--- a/docker/connector-db/README.md
+++ /dev/null
@@ -1,13 +0,0 @@
-# Docker Run
-
-Useful while running database connectors like mysql, mariadb, mssql
-
-**NOTE**: Copy `sample.env` into `.env` and update the necessary variables.
-
-```bash
-# Up all connector db
-docker compose -f docker-compose-db-connector.yaml up -d
-
-# Up a specific service alone
-docker compose -f docker-compose-db-connector.yaml up -d mysql
-```
diff --git a/docker/connector/README.md b/docker/connector/README.md
new file mode 100644
index 000000000..0ba69fb59
--- /dev/null
+++ b/docker/connector/README.md
@@ -0,0 +1,15 @@
+# Docker Run
+
+Useful while running connectors
+- Database connectors: Includes db connectors like mysql, mariadb, mssql
+- File system connectors: Includes fs connectors like sftp
+
+**NOTE**: Copy `sample.env` into `.env` and update the necessary variables.
+
+```bash
+# Up all connector db
+docker compose -f docker-compose-connector.yaml up -d
+
+# Up a specific service alone
+docker compose -f docker-compose-connector.yaml up -d mysql
+```
diff --git a/docker/connector-db/docker-compose-db-connector.yaml b/docker/connector/docker-compose-connector.yaml
similarity index 77%
rename from docker/connector-db/docker-compose-db-connector.yaml
rename to docker/connector/docker-compose-connector.yaml
index cf7a2389c..b07fa5ed9 100644
--- a/docker/connector-db/docker-compose-db-connector.yaml
+++ b/docker/connector/docker-compose-connector.yaml
@@ -31,7 +31,16 @@ services:
volumes:
- mssql_data:/var/opt/mssql
+ sftp:
+ image: atmoz/sftp:debian
+ ports:
+ - "22:22"
+ volumes:
+ - sftp_data:/home/foo
+ command: "${SFTP_USER}:${SFTP_PASS}:${SFTP_USER_ID}::${SFTP_DIR}"
+
volumes:
mariadb_data:
mysql_data:
mssql_data:
+ sftp_data:
diff --git a/docker/connector-db/sample.env b/docker/connector/sample.env
similarity index 73%
rename from docker/connector-db/sample.env
rename to docker/connector/sample.env
index c51ddc5a6..d6b160d41 100644
--- a/docker/connector-db/sample.env
+++ b/docker/connector/sample.env
@@ -4,3 +4,7 @@ MYSQL_USER=unstract_dev
MYSQL_PASSWORD=unstract_pass
ACCEPT_EULA=Y
MSSQL_SA_PASSWORD=unstractPassword@123
+SFTP_USER=foo
+SFTP_PASS=pass
+SFTP_USER_ID=1001
+SFTP_DIR=upload
diff --git a/frontend/public/icons/connector-icons/SFTP.png b/frontend/public/icons/connector-icons/SFTP.png
new file mode 100644
index 000000000..f50fe5f4c
Binary files /dev/null and b/frontend/public/icons/connector-icons/SFTP.png differ
diff --git a/unstract/connectors/pyproject.toml b/unstract/connectors/pyproject.toml
index e353be043..1f51c257c 100644
--- a/unstract/connectors/pyproject.toml
+++ b/unstract/connectors/pyproject.toml
@@ -8,6 +8,7 @@ name = "unstract-connectors"
version = "0.0.3"
description = "All connectors that are part of the Unstract platform"
dependencies = [
+ "singleton-decorator~=1.0.0",
# For GCSHelper
"google-auth==2.20.0",
"google-cloud-secret-manager==2.16.1",
@@ -25,7 +26,9 @@ dependencies = [
"snowflake-connector-python[pandas]~=3.6.0", # For Snowflake
"google-cloud-bigquery==3.11.4", # For Bigquery
"pymssql==2.2.8", # For MSSQL
- "PyMySQL==1.1.0" # For MySQL
+ "PyMySQL==1.1.0", # For MySQL
+ "fsspec[sftp]~=2024.6.0", # For SFTP
+
]
readme = "README.md"
urls = { Homepage = "https://unstract.com", Source = "https://github.com/Zipstack/unstract/tree/main/unstract/connectors" }
diff --git a/unstract/connectors/src/unstract/connectors/connectorkit.py b/unstract/connectors/src/unstract/connectors/connectorkit.py
index 5b1d574c1..69ea233d9 100644
--- a/unstract/connectors/src/unstract/connectors/connectorkit.py
+++ b/unstract/connectors/src/unstract/connectors/connectorkit.py
@@ -1,6 +1,8 @@
import logging
from typing import Any, Optional
+from singleton_decorator import singleton
+
from unstract.connectors import ConnectorDict # type: ignore
from unstract.connectors.base import UnstractConnector
from unstract.connectors.constants import Common
@@ -12,6 +14,7 @@ from unstract.connectors.queues import connectors as q_connectors
logger = logging.getLogger(__name__)
+@singleton
class Connectorkit:
def __init__(self) -> None:
self._connectors: ConnectorDict = fs_connectors | db_connectors | q_connectors
diff --git a/unstract/connectors/src/unstract/connectors/exceptions.py b/unstract/connectors/src/unstract/connectors/exceptions.py
index def0cd90e..cfeeb6f4c 100644
--- a/unstract/connectors/src/unstract/connectors/exceptions.py
+++ b/unstract/connectors/src/unstract/connectors/exceptions.py
@@ -47,3 +47,11 @@ class AzureHttpError(ConnectorError):
Args:
ConnectorError (Class): Inherits class ConnectorError
"""
+
+
+class PermissionDeniedError(ConnectorError):
+ """Handles permission denied error from fs connector.
+
+ Args:
+ ConnectorError (Class): Inherits class ConnectorError
+ """
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/sftp/__init__.py b/unstract/connectors/src/unstract/connectors/filesystems/sftp/__init__.py
new file mode 100644
index 000000000..7e887f0e8
--- /dev/null
+++ b/unstract/connectors/src/unstract/connectors/filesystems/sftp/__init__.py
@@ -0,0 +1,11 @@
+from .sftp import SftpFS
+
+__all__ = ["SftpFS"]
+
+metadata = {
+ "name": SftpFS.__name__,
+ "version": "1.0.0",
+ "connector": SftpFS,
+ "description": "SftpFS connector",
+ "is_active": True,
+}
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py b/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py
new file mode 100644
index 000000000..feef4d62a
--- /dev/null
+++ b/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py
@@ -0,0 +1,143 @@
+import logging
+import os
+from typing import Any
+
+from fsspec.implementations.sftp import SFTPFileSystem
+
+from unstract.connectors.exceptions import ConnectorError, PermissionDeniedError
+from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem
+
+logger = logging.getLogger(__name__)
+
+
+class SettingsKey:
+ HOST = "host"
+ PORT = "port"
+ USERNAME = "username"
+ PASSWORD = "password"
+ USER_DIRECTORY = "user_dir"
+
+
+class SettingsDefault:
+ PORT = 22
+
+
+class SftpFS(UnstractFileSystem):
+ def __init__(self, settings: dict[str, Any]):
+ super().__init__("SFTP")
+ host = settings.get(SettingsKey.HOST)
+ port = int(settings.get(SettingsKey.PORT, SettingsDefault.PORT))
+ username = settings.get(SettingsKey.USERNAME)
+ password = settings.get(SettingsKey.PASSWORD)
+ self.directory = str(settings.get(SettingsKey.USER_DIRECTORY))
+
+ self.sftp_fs = SFTPFileSystem(
+ host=host,
+ port=port,
+ username=username,
+ password=password,
+ )
+
+ @staticmethod
+ def get_id() -> str:
+ return "sftp|e68fa828-2988-4cce-9e3d-4285348e3227"
+
+ @staticmethod
+ def get_name() -> str:
+ return "SFTP/SSH"
+
+ @staticmethod
+ def get_description() -> str:
+ return "Fetch data via SFTP/SSH"
+
+ @staticmethod
+ def get_icon() -> str:
+ return "/icons/connector-icons/SFTP.png"
+
+ @staticmethod
+ def can_write() -> bool:
+ return True
+
+ @staticmethod
+ def can_read() -> bool:
+ return True
+
+ def get_fsspec_fs(self) -> SFTPFileSystem:
+ return self.sftp_fs
+
+ # TODO: Check if this method can be removed, and use it from parent class
+ # (class UnstractFileSystem)
+ @staticmethod
+ def get_json_schema() -> str:
+ f = open(f"{os.path.dirname(__file__)}/static/json_schema.json")
+ schema = f.read()
+ f.close()
+ return schema
+
+ # TODO: Check if this method can be removed, and use it from parent class
+ # (class UnstractFileSystem)
+ @staticmethod
+ def requires_oauth() -> bool:
+ return False
+
+ # TODO: Check if this method can be removed, and use it from parent class
+ # (class UnstractFileSystem)
+ @staticmethod
+ def python_social_auth_backend() -> str:
+ return ""
+
+ def test_credentials(self) -> bool:
+ """To test credentials for SFTP."""
+ is_dir = False
+ user_dir = f"{self.directory.strip('/')}/"
+ try:
+ is_dir = bool(self.get_fsspec_fs().isdir(user_dir))
+ except Exception as e:
+ raise ConnectorError(
+ f"Error while connecting to SFTP server: {str(e)}"
+ ) from e
+ if not is_dir:
+ raise ConnectorError(
+ "Unable to connect to SFTP server, "
+ "please check the connection settings."
+ )
+ return True
+
+ def create_dir_if_not_exists(self, input_dir: str) -> None:
+ """Method to create dir of a connector if not exists.
+
+ Args:
+ input_dir (str): input directory of source connector
+ """
+ try:
+ super().create_dir_if_not_exists(input_dir=input_dir)
+ except PermissionError as e:
+ self.raise_permission_error(input_dir=input_dir, error=e)
+
+ def upload_file_to_storage(self, source_path: str, destination_path: str) -> None:
+ """Method to upload filepath from tool to sftp connector directory.
+
+ Args:
+ source_path (str): local path of file to be uploaded, coming from tool
+ destination_path (str): target path in the storage where the file will be
+ uploaded
+ """
+ try:
+ normalized_path = os.path.normpath(destination_path)
+ super().upload_file_to_storage(
+ source_path=source_path, destination_path=destination_path
+ )
+ except PermissionError as e:
+ self.raise_permission_error(input_dir=normalized_path, error=e)
+
+ def raise_permission_error(
+ self, input_dir: str, error: PermissionError
+ ) -> PermissionDeniedError:
+ user_message = (
+ "Please verify your SFTP credentials and ensure you "
+ f" ensure you have the necessary permissions for the path '{input_dir}'. "
+ )
+ raise PermissionDeniedError(
+ user_message,
+ treat_as_user_message=True,
+ ) from error
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/sftp/static/json_schema.json b/unstract/connectors/src/unstract/connectors/filesystems/sftp/static/json_schema.json
new file mode 100644
index 000000000..9a131f030
--- /dev/null
+++ b/unstract/connectors/src/unstract/connectors/filesystems/sftp/static/json_schema.json
@@ -0,0 +1,48 @@
+{
+ "title": "SFTP",
+ "description": "A connector that allows for fetching data via SFTP",
+ "type": "object",
+ "required": [
+ "connectorName",
+ "host",
+ "port",
+ "username",
+ "password",
+ "user_dir"
+
+ ],
+ "properties": {
+ "connectorName": {
+ "type": "string",
+ "title": "Name of the connector",
+ "default": "Unstract's SFTP connector"
+ },
+ "host": {
+ "type": "string",
+ "title": "Host/IP",
+ "description": "The hostname or IP address of the SFTP server. Example: `sftp.example.com` or `192.168.1.10`."
+ },
+ "port": {
+ "type": "string",
+ "title": "Port",
+ "description": "The port number to connect to the SFTP server. Example: `22` (default SSH port).",
+ "default": "22"
+ },
+ "username": {
+ "type": "string",
+ "title": "Username",
+ "description": "The username used to authenticate with the SFTP server"
+ },
+ "password": {
+ "type": "string",
+ "title": "Password",
+ "description": "The password used to authenticate with the SFTP server",
+ "format": "password"
+ },
+ "user_dir": {
+ "type": "string",
+ "title": "User directory",
+ "description": "The directory that the user has access to"
+ }
+ }
+}
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py b/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py
index 73f8a38b4..75389c772 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py
@@ -93,8 +93,9 @@ class UnstractFileSystem(UnstractConnector, ABC):
directory.
Args:
- source_path (str): _description_
- destination_path (str): _description_
+ source_path (str): local path of file to be uploaded, coming from tool
+ destination_path (str): target path in the storage where the file will be
+ uploaded
"""
normalized_path = os.path.normpath(destination_path)
fs = self.get_fsspec_fs()