Skip to content

Graph-Native Infrastructure for Context and Accountable AI Systems

fix: prevent fallback recursion and write proper Parquet in embed generate command

Varun Sahnid42af280e857
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
deleted file mode 100644
index 0157ee24..00000000
--- a/.github/workflows/benchmark.yml
+++ /dev/null
@@ -1,51 +0,0 @@
-name: Semantica Performance Suite
-
-on:
- workflow_dispatch:
-
-permissions:
- contents: read
-
-jobs:
- performance-test:
- name: Benchmark Runner (Ubuntu/Python 3.12)
- runs-on: ubuntu-latest
-
- steps:
- - name: Checkout Code
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- with:
- fetch-depth: 0
-
- - name: Set up Python 3.11
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
- with:
- python-version: "3.11"
- cache: 'pip'
-
- - name: Install Dependencies
- env:
-
- BENCHMARK_REAL_LIBS: "1"
- run: |
- python -m pip install --upgrade pip
- pip install -e .
- pip install -r benchmarks/requirements.txt
- python -m spacy download en_core_web_sm
- pip install rdflib neo4j faiss-cpu torch pyarrow pdfplumber python-pptx openpyxl lxml python-docx beautifulsoup4 chardet langdetect
-
- - name: Execute Benchmarks (Real Mode)
- env:
- BENCHMARK_REAL_LIBS: "1"
- run: |
- python benchmarks/benchmarks_runner.py
- # Optional: Compare to baseline (requires previous run artifact)
- # pytest-benchmark --storage file://benchmarks/results --benchmark-compare
-
- - name: Upload Benchmark Results
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
- if: always()
- with:
- name: benchmark-report-${{ github.run_id }}
- path: benchmarks/results
- retention-days: 30
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
deleted file mode 100644
index 4ea31ff8..00000000
--- a/.github/workflows/ci.yml
+++ /dev/null
@@ -1,83 +0,0 @@
-name: CI
-
-permissions:
- contents: read
-
-on:
- push:
- branches: [main]
- paths-ignore:
- - 'docs/**'
- - 'docs_check.py'
- - '**/*.md'
- pull_request:
- branches: [main]
- paths-ignore:
- - 'docs/**'
- - 'docs_check.py'
- - '**/*.md'
-
-jobs:
- build:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
- with:
- python-version: '3.11'
- - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
- with:
- node-version: '20'
- cache: 'npm'
- cache-dependency-path: explorer/package-lock.json
- - name: Install Explorer frontend dependencies
- working-directory: explorer
- run: npm ci
- - name: Test Explorer frontend
- working-directory: explorer
- run: |
- npm run test:graph-store
- npm run test:graph-workspace
- npm run test:plugin-registry
- - name: Build Explorer frontend
- working-directory: explorer
- run: npm run build
- - name: Install pinned Python dependencies
- run: |
- pip install -r requirements-ci.txt
- - name: Verify requirements-ci.txt is up to date
- run: |
- pip install uv==0.12.1
- # Re-resolve with the committed file as a constraint: upstream package
- # releases must NOT fail CI (deps only change when pyproject.toml
- # changes intentionally). Compare only version lines (pkg==ver),
- # ignoring the -c constraint comments and the `\` line continuations
- # that --generate-hashes emits.
- uv pip compile pyproject.toml --python-version 3.11 --extra all \
- --constraint requirements-ci.txt -o /tmp/requirements-ci-check.txt
- diff \
- <(grep -E '^[a-zA-Z0-9._-]+==' requirements-ci.txt | sed 's/ \\$//') \
- <(grep -E '^[a-zA-Z0-9._-]+==' /tmp/requirements-ci-check.txt)
- - run: pip install build
- # wheel is build-time only (not in requirements-ci.txt) — install the
- # same pinned version [build-system] declares so --no-isolation works.
- - run: pip install wheel==0.48.0
- - name: Build package (no isolation — pinned deps)
- run: python -m build --no-isolation
- - name: Verify Explorer frontend is packaged
- run: |
- python - <<'PY'
- import zipfile
- from pathlib import Path
-
- wheels = list(Path("dist").glob("*.whl"))
- assert wheels, "No wheel was built"
-
- with zipfile.ZipFile(wheels[0]) as wheel:
- names = set(wheel.namelist())
-
- assert "semantica/static/index.html" in names, "Explorer index.html missing from wheel"
- assert any(name.startswith("semantica/static/assets/") for name in names), "Explorer assets missing from wheel"
-
- print("Explorer frontend is packaged")
- PY
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
deleted file mode 100644
index 0c15e84c..00000000
--- a/.github/workflows/codeql.yml
+++ /dev/null
@@ -1,97 +0,0 @@
-name: CodeQL
-
-on:
- push:
- branches: [main]
- pull_request:
- branches: [main]
- schedule:
- - cron: '30 1 * * 1' # Every Monday 7 AM IST
-
-permissions:
- contents: read
- security-events: write
- actions: read
-
-jobs:
- analyze:
- name: Analyze Python
- runs-on: ubuntu-latest
-
- steps:
- - name: Checkout repository
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
-
- # The CodeQL bundle download (github/codeql-action/init's "Setup CodeQL
- # tools" step) streams a ~1GB tarball from GitHub's release CDN and
- # does not retry on a transient connection reset (ECONNRESET) itself
- # (github/codeql-action, unresolved as of v4 / CLI 2.26.1: the HTTP
- # error is retryable but isn't retried internally). Since a `uses:`
- # step can't be wrapped by a shell-level retry action, attempt init
- # up to 3 times; each retry is a fresh download attempt with no
- # meaningful state carried over from a failed attempt.
- - name: Initialize CodeQL (attempt 1)
- id: codeql-init-1
- uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
- continue-on-error: true
- with:
- languages: python
- queries: security-and-quality
- config-file: .github/codeql/codeql-config.yml
-
- - name: Initialize CodeQL (attempt 2)
- id: codeql-init-2
- if: steps.codeql-init-1.outcome == 'failure'
- uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
- continue-on-error: true
- with:
- languages: python
- queries: security-and-quality
- config-file: .github/codeql/codeql-config.yml
-
- - name: Initialize CodeQL (attempt 3)
- id: codeql-init-3
- if: steps.codeql-init-2.outcome == 'failure'
- uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
- with:
- languages: python
- queries: security-and-quality
- config-file: .github/codeql/codeql-config.yml
-
- - name: Autobuild
- uses: github/codeql-action/autobuild@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
-
- - name: Perform CodeQL Analysis
- uses: github/codeql-action/analyze@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
- with:
- category: "/language:python"
- upload: false
- id: codeql
-
- - name: Upload SARIF (Advanced Setup only)
- # Uploads results only when Default Setup is not active.
- # If Default Setup is still enabled, this step skips gracefully
- # instead of failing the workflow with HTTP 409.
- uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
- with:
- sarif_file: ${{ steps.codeql.outputs.sarif-output }}
- category: "/language:python"
- wait-for-processing: true
- continue-on-error: true
-
- # NOTE: Auto-dismissal by rule-id is intentionally removed.
- # Dismissing every alert that matches a rule ID would silently suppress
- # future real vulnerabilities of the same type. The alerts below were
- # individually triaged and dismissed manually in the security-enhancement
- # PR (alerts #12–#18). New alerts must be reviewed and dismissed by hand,
- # or will auto-close when the underlying code no longer triggers them.
- #
- # If you need to dismiss a specific known-safe alert, pin its alert NUMBER
- # here and remove it once CodeQL stops reporting it naturally. Example:
- #
- # PINNED_ALERT_NUMBERS=(12 13 14 15 16 17 18)
- # for NUM in "${PINNED_ALERT_NUMBERS[@]}"; do
- # gh api repos/$REPO/code-scanning/alerts/$NUM \
- # -X PATCH -f state=dismissed -f dismissed_reason="false positive" \
- # -f dismissed_comment="<reason>"
- # done
diff --git a/.github/workflows/defender-for-devops.yml b/.github/workflows/defender-for-devops.yml
deleted file mode 100644
index becb7d64..00000000
--- a/.github/workflows/defender-for-devops.yml
+++ /dev/null
@@ -1,88 +0,0 @@
-# This workflow uses actions that are not certified by GitHub.
-# They are provided by a third-party and are governed by
-# separate terms of service, privacy policy, and support
-# documentation.
-#
-# Microsoft Security DevOps (MSDO) is a command line application which integrates static analysis tools into the development cycle.
-# MSDO installs, configures and runs the latest versions of static analysis tools
-# (including, but not limited to, SDL/security and compliance tools).
-#
-# The Microsoft Security DevOps action is currently in beta and runs on the windows-latest queue,
-# as well as Windows self hosted agents. ubuntu-latest support coming soon.
-#
-# For more information about the action , check out https://github.com/microsoft/security-devops-action
-#
-# Please note this workflow do not integrate your GitHub Org with Microsoft Defender For DevOps. You have to create an integration
-# and provide permission before this can report data back to azure.
-# Read the official documentation here : https://learn.microsoft.com/en-us/azure/defender-for-cloud/quickstart-onboard-github
-
-name: "Microsoft Defender For Devops"
-
-on:
- push:
- branches: [ "main" ]
- pull_request:
- branches: [ "main" ]
- schedule:
- - cron: '43 17 * * 6'
-
-permissions:
- contents: read
- security-events: write
-
-jobs:
- MSDO:
- # currently only windows-latest is supported
- runs-on: windows-latest
-
- steps:
- - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6
- with:
- dotnet-version: |
- 5.0.x
- 6.0.x
- - name: Run Microsoft Security DevOps
- uses: microsoft/security-devops-action@08976cb623803b1b36d7112d4ff9f59eae704de0 # v1.12.0
- id: msdo
- with:
- # checkov is intentionally excluded from this MSDO step.
- # MSDO 0.215.0's guardian.cmd wrapper treats checkov's exit code 1
- # (emitted whenever any violation is found, even below the active severity
- # threshold) as a fatal "tool error" and breaks the build even when
- # "Active results: 0" and "Found no breaking results." The .checkov.yaml
- # soft-fail setting is never read by the guardian wrapper.
- # IaC security scanning continues below in this same MSDO job identity.
- # That preserves the existing GitHub code-scanning configuration while
- # avoiding the guardian.cmd/checkov exit-code bug in the MSDO wrapper.
- tools: eslint,templateanalyzer,terrascan
- - name: Upload results to Security tab
- uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
- with:
- sarif_file: ${{ steps.msdo.outputs.sarifFile }}
-
- - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
- with:
- python-version: "3.12"
-
- - name: Install Checkov
- run: python -m pip install checkov==3.3.1
-
- - name: Run Checkov
- shell: pwsh
- env:
- PYTHONUTF8: "1"
- run: |
- New-Item -ItemType Directory -Force reports | Out-Null
- checkov --directory . --framework kubernetes helm dockerfile github_actions secrets bicep arm --soft-fail --output sarif --output-file-path reports/checkov.sarif
- if (-not (Test-Path reports/checkov.sarif)) {
- $sarif = Get-ChildItem -Path reports -Recurse -Filter *.sarif | Select-Object -First 1
- if ($null -eq $sarif) { throw "Checkov did not produce a SARIF file" }
- Copy-Item $sarif.FullName reports/checkov.sarif
- }
-
- - name: Upload Checkov results to Security tab
- uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
- if: always()
- with:
- sarif_file: reports/checkov.sarif
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
deleted file mode 100644
index ca149d7e..00000000
--- a/.github/workflows/docs.yml
+++ /dev/null
@@ -1,68 +0,0 @@
-name: Build and Deploy Documentation
-
-on:
- push:
- branches: [main]
- paths:
- - 'docs/**'
- - 'docs_check.py'
- - 'CHANGELOG.md'
- - 'RELEASE.md'
- pull_request:
- branches: [main]
- paths:
- - 'docs/**'
- - 'docs_check.py'
- workflow_dispatch:
-
-permissions:
- contents: read
- pages: write
- id-token: write
-
-concurrency:
- group: "pages"
- cancel-in-progress: false
-
-jobs:
- validate:
- name: Validate Documentation
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
- with:
- python-version: '3.11'
- - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
- with:
- node-version: '20'
- - run: python docs_check.py
-
- deploy:
- name: Build and Deploy to GitHub Pages
- if: github.event_name != 'pull_request'
- runs-on: ubuntu-latest
- needs: validate
- steps:
- - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
-
- - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
- with:
- node-version: '20'
-
- - name: Export static site
- run: |
- cd docs
- npx mintlify export --output ../export.zip
- cd ..
- unzip -q export.zip -d site
-
- - uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6
-
- - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5
- with:
- path: ./site
-
- - name: Deploy to GitHub Pages
- id: deployment
- uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
deleted file mode 100644
index bbc8770c..00000000
--- a/.github/workflows/release.yml
+++ /dev/null
@@ -1,73 +0,0 @@
-name: Release
-
-on:
- push:
- tags: ['v*']
-
-permissions:
- contents: read
-
-jobs:
- release:
- runs-on: ubuntu-latest
- environment: pypi
- concurrency:
- group: release-${{ github.ref }}
- cancel-in-progress: false
- permissions:
- contents: write # for the GitHub Release
- id-token: write # for PyPI Trusted Publishing (OIDC) and attestation signing
- attestations: write # for SLSA build provenance
- # If you add another job to this workflow, give it its own explicit
- # `permissions:` block rather than relying on the workflow-level default
- # above (contents: read) - do not widen the workflow-level default.
- steps:
- - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
- with:
- python-version: '3.11'
- - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
- with:
- node-version: '20'
- cache: 'npm'
- cache-dependency-path: explorer/package-lock.json
- - name: Build Explorer frontend
- working-directory: explorer
- run: |
- npm ci
- npm run build
- # Install the pinned dependency set (with hashes) so the sdist/wheel
- # build runs against the same versions CI tests against.
- - name: Install pinned build dependencies
- run: pip install -r requirements-ci.txt
- - run: pip install build
- # wheel is build-time only (not in requirements-ci.txt) — install the
- # same pinned version [build-system] declares so --no-isolation works.
- - run: pip install wheel==0.48.0
- - name: Build package (no isolation — pinned deps)
- run: python -m build --no-isolation
- - name: Verify Explorer frontend is packaged
- run: |
- python - <<'PY'
- import zipfile
- from pathlib import Path
-
- wheels = list(Path("dist").glob("*.whl"))
- assert wheels, "No wheel was built"
-
- with zipfile.ZipFile(wheels[0]) as wheel:
- names = set(wheel.namelist())
-
- assert "semantica/static/index.html" in names, "Explorer index.html missing from wheel"
- assert any(name.startswith("semantica/static/assets/") for name in names), "Explorer assets missing from wheel"
-
- print("Explorer frontend is packaged")
- PY
- - name: Attest build provenance
- uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4
- with:
- subject-path: 'dist/*'
- - uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3
- with:
- files: dist/*
- - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1
diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml
deleted file mode 100644
index 5b4461af..00000000
--- a/.github/workflows/security-scan.yml
+++ /dev/null
@@ -1,239 +0,0 @@
-name: Security Scan
-
-on:
- schedule:
- - cron: '30 1 * * 1,4' # Mon/Thu 7 AM IST
- push:
- branches: [main]
- paths-ignore:
- - 'docs/**'
- - 'mkdocs.yml'
- - 'requirements-docs.txt'
- - '**/*.md'
- pull_request:
- branches: [main]
- paths-ignore:
- - 'docs/**'
- - 'mkdocs.yml'
- - 'requirements-docs.txt'
- - '**/*.md'
-
-permissions:
- contents: read
-
-jobs:
- security-scan:
- runs-on: ubuntu-latest
- permissions:
- contents: read
- security-events: write
- actions: read
- # Needed for the "Comment PR with Security Results" step below. Safe on
- # pull_request (not pull_request_target): GitHub always forces a
- # read-only token for PRs from forks regardless of this permission.
- pull-requests: write
-
- steps:
- - name: Checkout repository
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
-
- - name: Set up Python
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
- with:
- python-version: '3.11'
-
- - name: Install dependencies
- run: |
- python -m pip install --upgrade pip
- # Install the pinned dependency set FIRST so Safety scans Semantica's
- # exact CI/release dependency tree (requirements-ci.txt is generated
- # from pyproject.toml extras, so this covers the project's real deps).
- pip install -r requirements-ci.txt
- # Tooling AFTER the pinned set: installing safety/bandit/semgrep/jq
- # first lets the pinned requirements overwrite their transitive deps
- # (e.g. rich), which breaks the safety CLI at runtime.
- pip install safety bandit semgrep jq
-
- - name: Run Safety Check (Package Vulnerabilities)
- run: |
- # NOTE: Safety 3.x repurposed --output to select a console format
- # (json/text/screen/...), not a file path. Writing JSON to a file
- # now requires --save-json; the previous `--output safety-report.json`
- # usage was silently invalid and never produced a report.
- safety check --save-json safety-report.json || true
-
- # Guard 1: fail loudly if Safety exited before writing a report at all
- # (network error, API auth failure, tool crash). Without this check a
- # missing or empty file causes jq to fall back to "0", making a broken
- # scanner indistinguishable from a clean scan.
- if [ ! -s safety-report.json ]; then
- echo "::error::Safety scan produced no report (safety-report.json is missing or empty). Treating as failure — check for network errors, API auth failures, or Safety crashes in the logs above."
- exit 1
- fi
-
- echo "Checking for package vulnerabilities..."
-
- # No || echo "0" fallback: if jq fails (malformed JSON, missing key,
- # vulnerabilities:null) VULNS will be empty or "null" so guard 2 below
- # catches it rather than silently treating the broken report as zero.
- VULNS=$(jq '.vulnerabilities | length' safety-report.json 2>/dev/null)
-
- # Guard 2: ensure VULNS is a non-negative integer before the -gt
- # comparison. "null" (missing/null key) or "" (jq parse failure) would
- # cause bash's -gt to throw an arithmetic error and fall through to the
- # success branch — the same silent-pass bug as a missing file.
- if ! [[ "$VULNS" =~ ^[0-9]+$ ]]; then
- echo "::error::Safety report exists but 'vulnerabilities' is missing or non-numeric (got: '${VULNS}'). The report may be malformed or Safety may have written an error-only JSON. Treating as failure."
- exit 1
- fi
-
- if [ "$VULNS" -gt 0 ]; then
- echo "❌ Security vulnerabilities found: $VULNS"
- echo "CI will fail to prevent merging of vulnerable dependencies"
- echo ""
- echo "Vulnerability details:"
- jq -r '.vulnerabilities[] | "- \(.package_name)==\(.analyzed_version): \(.vulnerability_id) (\(.CVE // "no CVE assigned"))"' safety-report.json || true
- exit 1
- else
- echo "✅ No security vulnerabilities found"
- fi
-
- - name: Run Bandit (Code Security Linter)
- run: |
- bandit -r semantica/ -f json -o bandit-report.json || true
- echo "Checking for HIGH severity security issues..."
-
- # Count HIGH severity issues
- HIGH_ISSUES=$(bandit -r semantica/ -f json -ll 2>/dev/null | jq -r '.results[]? | select(.issue_severity == "HIGH") | .test_name' 2>/dev/null | wc -l || echo "0")
-
- if [ "$HIGH_ISSUES" -gt 0 ]; then
- echo "❌ HIGH severity security issues found: $HIGH_ISSUES"
- echo "CI will fail to prevent merging of high-risk code"
- echo ""
- echo "High severity issues:"
- bandit -r semantica/ -ll | grep "Severity: High" -A 5 -B 1 || true
- exit 1
- else
- echo "✅ No HIGH severity security issues found"
- fi
-
- - name: Run Semgrep (Static Analysis)
- run: |
- echo "Running Semgrep static analysis..."
- semgrep --config=auto --json --output=semgrep-report.json semantica/ || true
-
- # Run security-focused rules
- echo "Checking for security patterns..."
- SECURITY_ISSUES=$(semgrep --config=p/security --json semantica/ 2>/dev/null | jq '.results | length' 2>/dev/null || echo "0")
-
- if [ "$SECURITY_ISSUES" -gt 0 ]; then
- echo "⚠️ Security patterns found: $SECURITY_ISSUES"
- echo "Review these findings for potential improvements"
- semgrep --config=p/security semantica/ || true
- else
- echo "✅ No security patterns found"
- fi
-
- - name: Upload Security Reports
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
- with:
- name: security-reports
- retention-days: 14
- path: |
- safety-report.json
- bandit-report.json
- semgrep-report.json
-
- - name: Comment PR with Security Results
- if: github.event_name == 'pull_request'
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
- with:
- script: |
- const fs = require('fs');
-
- // Renders one tool's findings as a section. `items` is already
- // the list of pre-formatted "- `thing` in `where`" strings; this
- // just handles the found/not-found/report-missing framing and
- // collapses long lists into a <details> block so the comment
- // doesn't turn into a wall of text.
- function renderSection(title, reportPath, parse) {
- let data;
- try {
- data = JSON.parse(fs.readFileSync(reportPath, 'utf8'));
- } catch (e) {
- return [
- `### ${title}`,
- `⚠️ No report found at \`${reportPath}\` — the scan may have failed before producing output. Check the job logs.`,
- ].join('\n');
- }
-
- const items = parse(data);
- if (items.length === 0) {
- return [`### ${title}`, `✅ No findings.`].join('\n');
- }
-
- const lines = [`### ${title}`, `Found **${items.length}**.`, ''];
- const shown = items.slice(0, 15);
- if (items.length > 15) {
- lines.push('<details>', '<summary>Show all findings</summary>', '');
- lines.push(...items);
- lines.push('', '</details>');
- } else {
- lines.push(...shown);
- }
- return lines.join('\n');
- }
-
- const safetySection = renderSection(
- 'Safety — dependency vulnerabilities',
- 'safety-report.json',
- (data) => (data.vulnerabilities || []).map(
- (v) => `- \`${v.package_name}==${v.analyzed_version}\`: ${v.vulnerability_id}` +
- (v.CVE ? ` (${v.CVE})` : '') + ` — ${v.advisory || 'no advisory text'}`
- )
- );
-
- const banditSection = renderSection(
- 'Bandit — HIGH-severity code issues',
- 'bandit-report.json',
- (data) => (data.results || [])
- .filter((issue) => issue.issue_severity === 'HIGH')
- .map((issue) => `- \`${issue.test_name}\` in \`${issue.filename}:${issue.line_number}\``)
- );
-
- const semgrepSection = renderSection(
- 'Semgrep — static analysis patterns',
- 'semgrep-report.json',
- (data) => (data.results || []).map(
- (issue) => `- \`${issue.check_id}\` in \`${issue.path}:${issue.start?.line ?? '?'}\``
- )
- );
-
- const comment = [
- '# 🔒 Security Scan Results',
- '',
- safetySection,
- '',
- banditSection,
- '',
- semgrepSection,
- '',
- '---',
- '',
- '*This security scan runs automatically on source-code PRs and bi-weekly (skipped for doc/markdown-only changes).*',
- '',
- '📊 **Security Policy**: CI fails on Safety vulnerabilities and Bandit HIGH-severity findings. Semgrep findings above are informational and do not block merge.',
- ].join('\n');
-
- try {
- await github.rest.issues.createComment({
- issue_number: context.issue.number,
- owner: context.repo.owner,
- repo: context.repo.repo,
- body: comment,
- });
- console.log('✅ Security comment posted successfully');
- } catch (error) {
- console.log('⚠️ Could not post security comment:', error.message);
- console.log('📋 Security scan results saved to artifacts');
- }
diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml
deleted file mode 100644
index 412e7eaa..00000000
--- a/.github/workflows/security.yml
+++ /dev/null
@@ -1,42 +0,0 @@
-name: Security
-
-on:
- schedule:
- - cron: '0 0 * * 1'
- workflow_dispatch:
- pull_request:
- branches: [main]
- paths:
- - 'pyproject.toml'
- - 'requirements-ci.txt'
- - '.github/workflows/security.yml'
-
-permissions:
- contents: read
-
-jobs:
- audit:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
- with:
- python-version: '3.11'
- # Upgrade first: actions/setup-python's baked-in setuptools has been
- # behind known-vulnerable floors before (e.g. PYSEC-2026-3447 /
- # setuptools 75.1.0), so don't trust the preinstalled one.
- - run: python -m pip install --upgrade pip setuptools
- # Audit the pinned dependency set (requirements-ci.txt is compiled from
- # pyproject.toml with --extra all — the same coverage as the [all]
- # extra, minus the Linux-only gpu set — so this keeps scan parity with
- # CI/release builds without a time-dependent resolution). This is the
- # fix for PYSEC-2024-38 (#869): the bare-env job never had fastapi or
- # python-multipart installed to look at.
- - run: pip install -r requirements-ci.txt
- # PR runs gate on findings, since they're scoped to actual
- # pyproject.toml changes under review. The schedule/workflow_dispatch
- # runs stay non-blocking until a full pass over pre-existing findings
- # across the whole [all] tree has been done.
- - run: pip install pip-audit
- - run: pip-audit -r requirements-ci.txt
- continue-on-error: ${{ github.event_name != 'pull_request' }}
diff --git a/.github/workflows/verify-action-pins.yml b/.github/workflows/verify-action-pins.yml
deleted file mode 100644
index 6e7dada9..00000000
--- a/.github/workflows/verify-action-pins.yml
+++ /dev/null
@@ -1,28 +0,0 @@
-name: Verify Action Pins
-
-on:
- pull_request:
- paths:
- - '.github/workflows/**'
- - '.github/scripts/verify-action-pins.sh'
- push:
- branches: [main]
- paths:
- - '.github/workflows/**'
- - '.github/scripts/verify-action-pins.sh'
- schedule:
- - cron: '0 3 * * 1' # weekly, in case an upstream tag is deliberately moved
- workflow_dispatch:
-
-permissions:
- contents: read
-
-jobs:
- verify:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- - name: Verify pinned action SHAs match their tag comments
- env:
- GH_TOKEN: ${{ github.token }}
- run: bash .github/scripts/verify-action-pins.sh
diff --git a/pyproject.toml b/pyproject.toml
index 03949d4e..738e68ef 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -85,7 +85,8 @@ dependencies = [
"loguru>=0.7.3",
"structlog>=22.1.0",
"gensim>=4.4.0",
- "httpx<0.29.0"
+ "httpx<0.29.0",
+ "pyarrow>=14.0.0"
]
[project.urls]
diff --git a/semantica/cli.py b/semantica/cli.py
index 7b944dc6..b5e14ebd 100644
--- a/semantica/cli.py
+++ b/semantica/cli.py
@@ -1708,7 +1708,28 @@ def embed_generate(cli_ctx: CLIContext, input_path: str, model: str,
except ImportError as exc:
raise click.ClickException(f"Embeddings module not available: {exc}") from exc
if output:
- Path(output).write_text(json.dumps(result, default=str), encoding="utf-8")
+ output_path = Path(output)
+ try:
+ import numpy as np
+ import pandas as pd
+ if output_path.suffix.lower() == ".parquet":
+ arr = np.asarray(result)
+ if arr.ndim == 1:
+ arr = arr[np.newaxis, :]
+ # schema: one column per embedding plus an id column
+ columns = [f"dim_{i}" for i in range(arr.shape[1])]
+ df = pd.DataFrame(arr, columns=columns)
+ df.index.name = "id"
+ df.to_parquet(output_path, index=True)
+ else:
+ output_path.write_text(
+ json.dumps(result, default=str), encoding="utf-8"
+ )
+ except ImportError as exc:
+ raise click.ClickException(
+ f"Missing dependency for --output: {exc}. "
+ f"Install pyarrow/pandas with: pip install semantica[ingest-parquet]"
+ ) from exc
_ok(cli_ctx, f"Wrote {output}")
elif _is_json(cli_ctx, local_json):
_jecho(result if isinstance(result, dict) else {"status": "ok"})
diff --git a/semantica/embeddings/methods.py b/semantica/embeddings/methods.py
index 30c47279..d2e8848e 100644
--- a/semantica/embeddings/methods.py
+++ b/semantica/embeddings/methods.py
@@ -116,9 +116,9 @@ def generate_embeddings(
>>> emb = generate_embeddings("Hello world", method="default")
>>> embs = generate_embeddings(["text1", "text2"], method="text")
"""
- # Check for custom method in registry
+ # Check for custom method in registry, skip self-reference
custom_method = method_registry.get("generation", method)
- if custom_method:
+ if custom_method and custom_method is not generate_embeddings:
try:
return custom_method(data, data_type=data_type, **kwargs)
except Exception as e:
@@ -164,9 +164,9 @@ def embed_text(
>>> emb = embed_text("Hello world", method="sentence_transformers")
>>> embs = embed_text(["text1", "text2"], method="sentence_transformers")
"""
- # Check for custom method in registry
+ # Check for custom method in registry, skip self-reference
custom_method = method_registry.get("text", method)
- if custom_method:
+ if custom_method and custom_method is not embed_text:
try:
return custom_method(text, **kwargs)
except Exception as e:
@@ -224,9 +224,9 @@ def calculate_similarity(
>>> similarity = calculate_similarity(emb1, emb2, method="cosine")
>>> print(f"Similarity: {similarity:.3f}")
"""
- # Check for custom method in registry
+ # Check for custom method in registry, skip self-reference
custom_method = method_registry.get("similarity", method)
- if custom_method:
+ if custom_method and custom_method is not calculate_similarity:
try:
return custom_method(embedding1, embedding2, **kwargs)
except Exception as e:
@@ -271,9 +271,9 @@ def pool_embeddings(
>>> pooled = pool_embeddings(embeddings, method="mean")
>>> attention_pooled = pool_embeddings(embeddings, method="attention")
"""
- # Check for custom method in registry
+ # Check for custom method in registry, skip self-reference
custom_method = method_registry.get("pooling", method)
- if custom_method:
+ if custom_method and custom_method is not pool_embeddings:
try:
return custom_method(embeddings, **kwargs)
except Exception as e: