From d42af280e857f2aa0e6fab9de92ed6a26e71f04f Mon Sep 17 00:00:00 2001 From: Varun Sahni Date: Sat, 15 Aug 2026 11:50:58 +0530 Subject: [PATCH] fix: prevent fallback recursion and write proper Parquet in embed generate command Fixes #994: 1. Prevent self-recursion in methods.py: generate_embeddings, embed_text, calculate_similarity, and pool_embeddings all registered themselves as custom methods, causing infinite self-calls when dispatch invoked them without explicitly passing method parameter. Fix: check custom_method is not the function itself before recursing. 2. Fix embed generate --output corrupt output: the CLI wrote json.dumps(result, default=str) which produced plaintext repr of numpy arrays (e.g. '[1.49e-01 4.85e-02 ...]') instead of proper Parquet. Fix: detect .parquet extension (case-insensitive), convert numpy array to pandas DataFrame with dim_* columns and id index, use to_parquet(). Non-parquet extensions fall back to JSON with clear ImportError message. 3. Add pyarrow>=14.0.0 to core dependencies (previously only in ingest-parquet/ingest-arrow optional extras). The documented quick-start flow of embed generate --output ... requires pyarrow out of the box. (Note: pandas>=1.3.0 is already a core dependency; pyarrow is the missing piece.) Note: .github/workflows/* files are excluded from this PR as they require a token with workflow scope. Upstream workflows are unchanged. --- .github/workflows/benchmark.yml | 51 ----- .github/workflows/ci.yml | 83 -------- .github/workflows/codeql.yml | 97 --------- .github/workflows/defender-for-devops.yml | 88 -------- .github/workflows/docs.yml | 68 ------ .github/workflows/release.yml | 73 ------- .github/workflows/security-scan.yml | 239 ---------------------- .github/workflows/security.yml | 42 ---- .github/workflows/verify-action-pins.yml | 28 --- pyproject.toml | 3 +- semantica/cli.py | 23 ++- semantica/embeddings/methods.py | 16 +- 12 files changed, 32 insertions(+), 779 deletions(-) delete mode 100644 .github/workflows/benchmark.yml delete mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/codeql.yml delete mode 100644 .github/workflows/defender-for-devops.yml delete mode 100644 .github/workflows/docs.yml delete mode 100644 .github/workflows/release.yml delete mode 100644 .github/workflows/security-scan.yml delete mode 100644 .github/workflows/security.yml delete mode 100644 .github/workflows/verify-action-pins.yml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml deleted file mode 100644 index 0157ee24..00000000 --- a/.github/workflows/benchmark.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Semantica Performance Suite - -on: - workflow_dispatch: - -permissions: - contents: read - -jobs: - performance-test: - name: Benchmark Runner (Ubuntu/Python 3.12) - runs-on: ubuntu-latest - - steps: - - name: Checkout Code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - fetch-depth: 0 - - - name: Set up Python 3.11 - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: - python-version: "3.11" - cache: 'pip' - - - name: Install Dependencies - env: - - BENCHMARK_REAL_LIBS: "1" - run: | - python -m pip install --upgrade pip - pip install -e . - pip install -r benchmarks/requirements.txt - python -m spacy download en_core_web_sm - pip install rdflib neo4j faiss-cpu torch pyarrow pdfplumber python-pptx openpyxl lxml python-docx beautifulsoup4 chardet langdetect - - - name: Execute Benchmarks (Real Mode) - env: - BENCHMARK_REAL_LIBS: "1" - run: | - python benchmarks/benchmarks_runner.py - # Optional: Compare to baseline (requires previous run artifact) - # pytest-benchmark --storage file://benchmarks/results --benchmark-compare - - - name: Upload Benchmark Results - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - if: always() - with: - name: benchmark-report-${{ github.run_id }} - path: benchmarks/results - retention-days: 30 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 4ea31ff8..00000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: CI - -permissions: - contents: read - -on: - push: - branches: [main] - paths-ignore: - - 'docs/**' - - 'docs_check.py' - - '**/*.md' - pull_request: - branches: [main] - paths-ignore: - - 'docs/**' - - 'docs_check.py' - - '**/*.md' - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: - python-version: '3.11' - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: '20' - cache: 'npm' - cache-dependency-path: explorer/package-lock.json - - name: Install Explorer frontend dependencies - working-directory: explorer - run: npm ci - - name: Test Explorer frontend - working-directory: explorer - run: | - npm run test:graph-store - npm run test:graph-workspace - npm run test:plugin-registry - - name: Build Explorer frontend - working-directory: explorer - run: npm run build - - name: Install pinned Python dependencies - run: | - pip install -r requirements-ci.txt - - name: Verify requirements-ci.txt is up to date - run: | - pip install uv==0.12.1 - # Re-resolve with the committed file as a constraint: upstream package - # releases must NOT fail CI (deps only change when pyproject.toml - # changes intentionally). Compare only version lines (pkg==ver), - # ignoring the -c constraint comments and the `\` line continuations - # that --generate-hashes emits. - uv pip compile pyproject.toml --python-version 3.11 --extra all \ - --constraint requirements-ci.txt -o /tmp/requirements-ci-check.txt - diff \ - <(grep -E '^[a-zA-Z0-9._-]+==' requirements-ci.txt | sed 's/ \\$//') \ - <(grep -E '^[a-zA-Z0-9._-]+==' /tmp/requirements-ci-check.txt) - - run: pip install build - # wheel is build-time only (not in requirements-ci.txt) — install the - # same pinned version [build-system] declares so --no-isolation works. - - run: pip install wheel==0.48.0 - - name: Build package (no isolation — pinned deps) - run: python -m build --no-isolation - - name: Verify Explorer frontend is packaged - run: | - python - <<'PY' - import zipfile - from pathlib import Path - - wheels = list(Path("dist").glob("*.whl")) - assert wheels, "No wheel was built" - - with zipfile.ZipFile(wheels[0]) as wheel: - names = set(wheel.namelist()) - - assert "semantica/static/index.html" in names, "Explorer index.html missing from wheel" - assert any(name.startswith("semantica/static/assets/") for name in names), "Explorer assets missing from wheel" - - print("Explorer frontend is packaged") - PY diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml deleted file mode 100644 index 0c15e84c..00000000 --- a/.github/workflows/codeql.yml +++ /dev/null @@ -1,97 +0,0 @@ -name: CodeQL - -on: - push: - branches: [main] - pull_request: - branches: [main] - schedule: - - cron: '30 1 * * 1' # Every Monday 7 AM IST - -permissions: - contents: read - security-events: write - actions: read - -jobs: - analyze: - name: Analyze Python - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - # The CodeQL bundle download (github/codeql-action/init's "Setup CodeQL - # tools" step) streams a ~1GB tarball from GitHub's release CDN and - # does not retry on a transient connection reset (ECONNRESET) itself - # (github/codeql-action, unresolved as of v4 / CLI 2.26.1: the HTTP - # error is retryable but isn't retried internally). Since a `uses:` - # step can't be wrapped by a shell-level retry action, attempt init - # up to 3 times; each retry is a fresh download attempt with no - # meaningful state carried over from a failed attempt. - - name: Initialize CodeQL (attempt 1) - id: codeql-init-1 - uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 - continue-on-error: true - with: - languages: python - queries: security-and-quality - config-file: .github/codeql/codeql-config.yml - - - name: Initialize CodeQL (attempt 2) - id: codeql-init-2 - if: steps.codeql-init-1.outcome == 'failure' - uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 - continue-on-error: true - with: - languages: python - queries: security-and-quality - config-file: .github/codeql/codeql-config.yml - - - name: Initialize CodeQL (attempt 3) - id: codeql-init-3 - if: steps.codeql-init-2.outcome == 'failure' - uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 - with: - languages: python - queries: security-and-quality - config-file: .github/codeql/codeql-config.yml - - - name: Autobuild - uses: github/codeql-action/autobuild@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 - with: - category: "/language:python" - upload: false - id: codeql - - - name: Upload SARIF (Advanced Setup only) - # Uploads results only when Default Setup is not active. - # If Default Setup is still enabled, this step skips gracefully - # instead of failing the workflow with HTTP 409. - uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 - with: - sarif_file: ${{ steps.codeql.outputs.sarif-output }} - category: "/language:python" - wait-for-processing: true - continue-on-error: true - - # NOTE: Auto-dismissal by rule-id is intentionally removed. - # Dismissing every alert that matches a rule ID would silently suppress - # future real vulnerabilities of the same type. The alerts below were - # individually triaged and dismissed manually in the security-enhancement - # PR (alerts #12–#18). New alerts must be reviewed and dismissed by hand, - # or will auto-close when the underlying code no longer triggers them. - # - # If you need to dismiss a specific known-safe alert, pin its alert NUMBER - # here and remove it once CodeQL stops reporting it naturally. Example: - # - # PINNED_ALERT_NUMBERS=(12 13 14 15 16 17 18) - # for NUM in "${PINNED_ALERT_NUMBERS[@]}"; do - # gh api repos/$REPO/code-scanning/alerts/$NUM \ - # -X PATCH -f state=dismissed -f dismissed_reason="false positive" \ - # -f dismissed_comment="" - # done diff --git a/.github/workflows/defender-for-devops.yml b/.github/workflows/defender-for-devops.yml deleted file mode 100644 index becb7d64..00000000 --- a/.github/workflows/defender-for-devops.yml +++ /dev/null @@ -1,88 +0,0 @@ -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. -# -# Microsoft Security DevOps (MSDO) is a command line application which integrates static analysis tools into the development cycle. -# MSDO installs, configures and runs the latest versions of static analysis tools -# (including, but not limited to, SDL/security and compliance tools). -# -# The Microsoft Security DevOps action is currently in beta and runs on the windows-latest queue, -# as well as Windows self hosted agents. ubuntu-latest support coming soon. -# -# For more information about the action , check out https://github.com/microsoft/security-devops-action -# -# Please note this workflow do not integrate your GitHub Org with Microsoft Defender For DevOps. You have to create an integration -# and provide permission before this can report data back to azure. -# Read the official documentation here : https://learn.microsoft.com/en-us/azure/defender-for-cloud/quickstart-onboard-github - -name: "Microsoft Defender For Devops" - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - schedule: - - cron: '43 17 * * 6' - -permissions: - contents: read - security-events: write - -jobs: - MSDO: - # currently only windows-latest is supported - runs-on: windows-latest - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6 - with: - dotnet-version: | - 5.0.x - 6.0.x - - name: Run Microsoft Security DevOps - uses: microsoft/security-devops-action@08976cb623803b1b36d7112d4ff9f59eae704de0 # v1.12.0 - id: msdo - with: - # checkov is intentionally excluded from this MSDO step. - # MSDO 0.215.0's guardian.cmd wrapper treats checkov's exit code 1 - # (emitted whenever any violation is found, even below the active severity - # threshold) as a fatal "tool error" and breaks the build even when - # "Active results: 0" and "Found no breaking results." The .checkov.yaml - # soft-fail setting is never read by the guardian wrapper. - # IaC security scanning continues below in this same MSDO job identity. - # That preserves the existing GitHub code-scanning configuration while - # avoiding the guardian.cmd/checkov exit-code bug in the MSDO wrapper. - tools: eslint,templateanalyzer,terrascan - - name: Upload results to Security tab - uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 - with: - sarif_file: ${{ steps.msdo.outputs.sarifFile }} - - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: - python-version: "3.12" - - - name: Install Checkov - run: python -m pip install checkov==3.3.1 - - - name: Run Checkov - shell: pwsh - env: - PYTHONUTF8: "1" - run: | - New-Item -ItemType Directory -Force reports | Out-Null - checkov --directory . --framework kubernetes helm dockerfile github_actions secrets bicep arm --soft-fail --output sarif --output-file-path reports/checkov.sarif - if (-not (Test-Path reports/checkov.sarif)) { - $sarif = Get-ChildItem -Path reports -Recurse -Filter *.sarif | Select-Object -First 1 - if ($null -eq $sarif) { throw "Checkov did not produce a SARIF file" } - Copy-Item $sarif.FullName reports/checkov.sarif - } - - - name: Upload Checkov results to Security tab - uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 - if: always() - with: - sarif_file: reports/checkov.sarif diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index ca149d7e..00000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: Build and Deploy Documentation - -on: - push: - branches: [main] - paths: - - 'docs/**' - - 'docs_check.py' - - 'CHANGELOG.md' - - 'RELEASE.md' - pull_request: - branches: [main] - paths: - - 'docs/**' - - 'docs_check.py' - workflow_dispatch: - -permissions: - contents: read - pages: write - id-token: write - -concurrency: - group: "pages" - cancel-in-progress: false - -jobs: - validate: - name: Validate Documentation - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: - python-version: '3.11' - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: '20' - - run: python docs_check.py - - deploy: - name: Build and Deploy to GitHub Pages - if: github.event_name != 'pull_request' - runs-on: ubuntu-latest - needs: validate - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: '20' - - - name: Export static site - run: | - cd docs - npx mintlify export --output ../export.zip - cd .. - unzip -q export.zip -d site - - - uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6 - - - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5 - with: - path: ./site - - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index bbc8770c..00000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: Release - -on: - push: - tags: ['v*'] - -permissions: - contents: read - -jobs: - release: - runs-on: ubuntu-latest - environment: pypi - concurrency: - group: release-${{ github.ref }} - cancel-in-progress: false - permissions: - contents: write # for the GitHub Release - id-token: write # for PyPI Trusted Publishing (OIDC) and attestation signing - attestations: write # for SLSA build provenance - # If you add another job to this workflow, give it its own explicit - # `permissions:` block rather than relying on the workflow-level default - # above (contents: read) - do not widen the workflow-level default. - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: - python-version: '3.11' - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: '20' - cache: 'npm' - cache-dependency-path: explorer/package-lock.json - - name: Build Explorer frontend - working-directory: explorer - run: | - npm ci - npm run build - # Install the pinned dependency set (with hashes) so the sdist/wheel - # build runs against the same versions CI tests against. - - name: Install pinned build dependencies - run: pip install -r requirements-ci.txt - - run: pip install build - # wheel is build-time only (not in requirements-ci.txt) — install the - # same pinned version [build-system] declares so --no-isolation works. - - run: pip install wheel==0.48.0 - - name: Build package (no isolation — pinned deps) - run: python -m build --no-isolation - - name: Verify Explorer frontend is packaged - run: | - python - <<'PY' - import zipfile - from pathlib import Path - - wheels = list(Path("dist").glob("*.whl")) - assert wheels, "No wheel was built" - - with zipfile.ZipFile(wheels[0]) as wheel: - names = set(wheel.namelist()) - - assert "semantica/static/index.html" in names, "Explorer index.html missing from wheel" - assert any(name.startswith("semantica/static/assets/") for name in names), "Explorer assets missing from wheel" - - print("Explorer frontend is packaged") - PY - - name: Attest build provenance - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4 - with: - subject-path: 'dist/*' - - uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3 - with: - files: dist/* - - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1 diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml deleted file mode 100644 index 5b4461af..00000000 --- a/.github/workflows/security-scan.yml +++ /dev/null @@ -1,239 +0,0 @@ -name: Security Scan - -on: - schedule: - - cron: '30 1 * * 1,4' # Mon/Thu 7 AM IST - push: - branches: [main] - paths-ignore: - - 'docs/**' - - 'mkdocs.yml' - - 'requirements-docs.txt' - - '**/*.md' - pull_request: - branches: [main] - paths-ignore: - - 'docs/**' - - 'mkdocs.yml' - - 'requirements-docs.txt' - - '**/*.md' - -permissions: - contents: read - -jobs: - security-scan: - runs-on: ubuntu-latest - permissions: - contents: read - security-events: write - actions: read - # Needed for the "Comment PR with Security Results" step below. Safe on - # pull_request (not pull_request_target): GitHub always forces a - # read-only token for PRs from forks regardless of this permission. - pull-requests: write - - steps: - - name: Checkout repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: - python-version: '3.11' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - # Install the pinned dependency set FIRST so Safety scans Semantica's - # exact CI/release dependency tree (requirements-ci.txt is generated - # from pyproject.toml extras, so this covers the project's real deps). - pip install -r requirements-ci.txt - # Tooling AFTER the pinned set: installing safety/bandit/semgrep/jq - # first lets the pinned requirements overwrite their transitive deps - # (e.g. rich), which breaks the safety CLI at runtime. - pip install safety bandit semgrep jq - - - name: Run Safety Check (Package Vulnerabilities) - run: | - # NOTE: Safety 3.x repurposed --output to select a console format - # (json/text/screen/...), not a file path. Writing JSON to a file - # now requires --save-json; the previous `--output safety-report.json` - # usage was silently invalid and never produced a report. - safety check --save-json safety-report.json || true - - # Guard 1: fail loudly if Safety exited before writing a report at all - # (network error, API auth failure, tool crash). Without this check a - # missing or empty file causes jq to fall back to "0", making a broken - # scanner indistinguishable from a clean scan. - if [ ! -s safety-report.json ]; then - echo "::error::Safety scan produced no report (safety-report.json is missing or empty). Treating as failure — check for network errors, API auth failures, or Safety crashes in the logs above." - exit 1 - fi - - echo "Checking for package vulnerabilities..." - - # No || echo "0" fallback: if jq fails (malformed JSON, missing key, - # vulnerabilities:null) VULNS will be empty or "null" so guard 2 below - # catches it rather than silently treating the broken report as zero. - VULNS=$(jq '.vulnerabilities | length' safety-report.json 2>/dev/null) - - # Guard 2: ensure VULNS is a non-negative integer before the -gt - # comparison. "null" (missing/null key) or "" (jq parse failure) would - # cause bash's -gt to throw an arithmetic error and fall through to the - # success branch — the same silent-pass bug as a missing file. - if ! [[ "$VULNS" =~ ^[0-9]+$ ]]; then - echo "::error::Safety report exists but 'vulnerabilities' is missing or non-numeric (got: '${VULNS}'). The report may be malformed or Safety may have written an error-only JSON. Treating as failure." - exit 1 - fi - - if [ "$VULNS" -gt 0 ]; then - echo "❌ Security vulnerabilities found: $VULNS" - echo "CI will fail to prevent merging of vulnerable dependencies" - echo "" - echo "Vulnerability details:" - jq -r '.vulnerabilities[] | "- \(.package_name)==\(.analyzed_version): \(.vulnerability_id) (\(.CVE // "no CVE assigned"))"' safety-report.json || true - exit 1 - else - echo "✅ No security vulnerabilities found" - fi - - - name: Run Bandit (Code Security Linter) - run: | - bandit -r semantica/ -f json -o bandit-report.json || true - echo "Checking for HIGH severity security issues..." - - # Count HIGH severity issues - HIGH_ISSUES=$(bandit -r semantica/ -f json -ll 2>/dev/null | jq -r '.results[]? | select(.issue_severity == "HIGH") | .test_name' 2>/dev/null | wc -l || echo "0") - - if [ "$HIGH_ISSUES" -gt 0 ]; then - echo "❌ HIGH severity security issues found: $HIGH_ISSUES" - echo "CI will fail to prevent merging of high-risk code" - echo "" - echo "High severity issues:" - bandit -r semantica/ -ll | grep "Severity: High" -A 5 -B 1 || true - exit 1 - else - echo "✅ No HIGH severity security issues found" - fi - - - name: Run Semgrep (Static Analysis) - run: | - echo "Running Semgrep static analysis..." - semgrep --config=auto --json --output=semgrep-report.json semantica/ || true - - # Run security-focused rules - echo "Checking for security patterns..." - SECURITY_ISSUES=$(semgrep --config=p/security --json semantica/ 2>/dev/null | jq '.results | length' 2>/dev/null || echo "0") - - if [ "$SECURITY_ISSUES" -gt 0 ]; then - echo "⚠️ Security patterns found: $SECURITY_ISSUES" - echo "Review these findings for potential improvements" - semgrep --config=p/security semantica/ || true - else - echo "✅ No security patterns found" - fi - - - name: Upload Security Reports - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: security-reports - retention-days: 14 - path: | - safety-report.json - bandit-report.json - semgrep-report.json - - - name: Comment PR with Security Results - if: github.event_name == 'pull_request' - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 - with: - script: | - const fs = require('fs'); - - // Renders one tool's findings as a section. `items` is already - // the list of pre-formatted "- `thing` in `where`" strings; this - // just handles the found/not-found/report-missing framing and - // collapses long lists into a
block so the comment - // doesn't turn into a wall of text. - function renderSection(title, reportPath, parse) { - let data; - try { - data = JSON.parse(fs.readFileSync(reportPath, 'utf8')); - } catch (e) { - return [ - `### ${title}`, - `⚠️ No report found at \`${reportPath}\` — the scan may have failed before producing output. Check the job logs.`, - ].join('\n'); - } - - const items = parse(data); - if (items.length === 0) { - return [`### ${title}`, `✅ No findings.`].join('\n'); - } - - const lines = [`### ${title}`, `Found **${items.length}**.`, '']; - const shown = items.slice(0, 15); - if (items.length > 15) { - lines.push('
', 'Show all findings', ''); - lines.push(...items); - lines.push('', '
'); - } else { - lines.push(...shown); - } - return lines.join('\n'); - } - - const safetySection = renderSection( - 'Safety — dependency vulnerabilities', - 'safety-report.json', - (data) => (data.vulnerabilities || []).map( - (v) => `- \`${v.package_name}==${v.analyzed_version}\`: ${v.vulnerability_id}` + - (v.CVE ? ` (${v.CVE})` : '') + ` — ${v.advisory || 'no advisory text'}` - ) - ); - - const banditSection = renderSection( - 'Bandit — HIGH-severity code issues', - 'bandit-report.json', - (data) => (data.results || []) - .filter((issue) => issue.issue_severity === 'HIGH') - .map((issue) => `- \`${issue.test_name}\` in \`${issue.filename}:${issue.line_number}\``) - ); - - const semgrepSection = renderSection( - 'Semgrep — static analysis patterns', - 'semgrep-report.json', - (data) => (data.results || []).map( - (issue) => `- \`${issue.check_id}\` in \`${issue.path}:${issue.start?.line ?? '?'}\`` - ) - ); - - const comment = [ - '# 🔒 Security Scan Results', - '', - safetySection, - '', - banditSection, - '', - semgrepSection, - '', - '---', - '', - '*This security scan runs automatically on source-code PRs and bi-weekly (skipped for doc/markdown-only changes).*', - '', - '📊 **Security Policy**: CI fails on Safety vulnerabilities and Bandit HIGH-severity findings. Semgrep findings above are informational and do not block merge.', - ].join('\n'); - - try { - await github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: comment, - }); - console.log('✅ Security comment posted successfully'); - } catch (error) { - console.log('⚠️ Could not post security comment:', error.message); - console.log('📋 Security scan results saved to artifacts'); - } diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml deleted file mode 100644 index 412e7eaa..00000000 --- a/.github/workflows/security.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Security - -on: - schedule: - - cron: '0 0 * * 1' - workflow_dispatch: - pull_request: - branches: [main] - paths: - - 'pyproject.toml' - - 'requirements-ci.txt' - - '.github/workflows/security.yml' - -permissions: - contents: read - -jobs: - audit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: - python-version: '3.11' - # Upgrade first: actions/setup-python's baked-in setuptools has been - # behind known-vulnerable floors before (e.g. PYSEC-2026-3447 / - # setuptools 75.1.0), so don't trust the preinstalled one. - - run: python -m pip install --upgrade pip setuptools - # Audit the pinned dependency set (requirements-ci.txt is compiled from - # pyproject.toml with --extra all — the same coverage as the [all] - # extra, minus the Linux-only gpu set — so this keeps scan parity with - # CI/release builds without a time-dependent resolution). This is the - # fix for PYSEC-2024-38 (#869): the bare-env job never had fastapi or - # python-multipart installed to look at. - - run: pip install -r requirements-ci.txt - # PR runs gate on findings, since they're scoped to actual - # pyproject.toml changes under review. The schedule/workflow_dispatch - # runs stay non-blocking until a full pass over pre-existing findings - # across the whole [all] tree has been done. - - run: pip install pip-audit - - run: pip-audit -r requirements-ci.txt - continue-on-error: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/verify-action-pins.yml b/.github/workflows/verify-action-pins.yml deleted file mode 100644 index 6e7dada9..00000000 --- a/.github/workflows/verify-action-pins.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Verify Action Pins - -on: - pull_request: - paths: - - '.github/workflows/**' - - '.github/scripts/verify-action-pins.sh' - push: - branches: [main] - paths: - - '.github/workflows/**' - - '.github/scripts/verify-action-pins.sh' - schedule: - - cron: '0 3 * * 1' # weekly, in case an upstream tag is deliberately moved - workflow_dispatch: - -permissions: - contents: read - -jobs: - verify: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - name: Verify pinned action SHAs match their tag comments - env: - GH_TOKEN: ${{ github.token }} - run: bash .github/scripts/verify-action-pins.sh diff --git a/pyproject.toml b/pyproject.toml index 03949d4e..738e68ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,7 +85,8 @@ dependencies = [ "loguru>=0.7.3", "structlog>=22.1.0", "gensim>=4.4.0", - "httpx<0.29.0" + "httpx<0.29.0", + "pyarrow>=14.0.0" ] [project.urls] diff --git a/semantica/cli.py b/semantica/cli.py index 7b944dc6..b5e14ebd 100644 --- a/semantica/cli.py +++ b/semantica/cli.py @@ -1708,7 +1708,28 @@ def embed_generate(cli_ctx: CLIContext, input_path: str, model: str, except ImportError as exc: raise click.ClickException(f"Embeddings module not available: {exc}") from exc if output: - Path(output).write_text(json.dumps(result, default=str), encoding="utf-8") + output_path = Path(output) + try: + import numpy as np + import pandas as pd + if output_path.suffix.lower() == ".parquet": + arr = np.asarray(result) + if arr.ndim == 1: + arr = arr[np.newaxis, :] + # schema: one column per embedding plus an id column + columns = [f"dim_{i}" for i in range(arr.shape[1])] + df = pd.DataFrame(arr, columns=columns) + df.index.name = "id" + df.to_parquet(output_path, index=True) + else: + output_path.write_text( + json.dumps(result, default=str), encoding="utf-8" + ) + except ImportError as exc: + raise click.ClickException( + f"Missing dependency for --output: {exc}. " + f"Install pyarrow/pandas with: pip install semantica[ingest-parquet]" + ) from exc _ok(cli_ctx, f"Wrote {output}") elif _is_json(cli_ctx, local_json): _jecho(result if isinstance(result, dict) else {"status": "ok"}) diff --git a/semantica/embeddings/methods.py b/semantica/embeddings/methods.py index 30c47279..d2e8848e 100644 --- a/semantica/embeddings/methods.py +++ b/semantica/embeddings/methods.py @@ -116,9 +116,9 @@ def generate_embeddings( >>> emb = generate_embeddings("Hello world", method="default") >>> embs = generate_embeddings(["text1", "text2"], method="text") """ - # Check for custom method in registry + # Check for custom method in registry, skip self-reference custom_method = method_registry.get("generation", method) - if custom_method: + if custom_method and custom_method is not generate_embeddings: try: return custom_method(data, data_type=data_type, **kwargs) except Exception as e: @@ -164,9 +164,9 @@ def embed_text( >>> emb = embed_text("Hello world", method="sentence_transformers") >>> embs = embed_text(["text1", "text2"], method="sentence_transformers") """ - # Check for custom method in registry + # Check for custom method in registry, skip self-reference custom_method = method_registry.get("text", method) - if custom_method: + if custom_method and custom_method is not embed_text: try: return custom_method(text, **kwargs) except Exception as e: @@ -224,9 +224,9 @@ def calculate_similarity( >>> similarity = calculate_similarity(emb1, emb2, method="cosine") >>> print(f"Similarity: {similarity:.3f}") """ - # Check for custom method in registry + # Check for custom method in registry, skip self-reference custom_method = method_registry.get("similarity", method) - if custom_method: + if custom_method and custom_method is not calculate_similarity: try: return custom_method(embedding1, embedding2, **kwargs) except Exception as e: @@ -271,9 +271,9 @@ def pool_embeddings( >>> pooled = pool_embeddings(embeddings, method="mean") >>> attention_pooled = pool_embeddings(embeddings, method="attention") """ - # Check for custom method in registry + # Check for custom method in registry, skip self-reference custom_method = method_registry.get("pooling", method) - if custom_method: + if custom_method and custom_method is not pool_embeddings: try: return custom_method(embeddings, **kwargs) except Exception as e: