From d42af280e857f2aa0e6fab9de92ed6a26e71f04f Mon Sep 17 00:00:00 2001 From: Varun Sahni Date: Sat, 15 Aug 2026 11:50:58 +0530 Subject: [PATCH 01/20] fix: prevent fallback recursion and write proper Parquet in embed generate command Fixes #994: 1. Prevent self-recursion in methods.py: generate_embeddings, embed_text, calculate_similarity, and pool_embeddings all registered themselves as custom methods, causing infinite self-calls when dispatch invoked them without explicitly passing method parameter. Fix: check custom_method is not the function itself before recursing. 2. Fix embed generate --output corrupt output: the CLI wrote json.dumps(result, default=str) which produced plaintext repr of numpy arrays (e.g. '[1.49e-01 4.85e-02 ...]') instead of proper Parquet. Fix: detect .parquet extension (case-insensitive), convert numpy array to pandas DataFrame with dim_* columns and id index, use to_parquet(). Non-parquet extensions fall back to JSON with clear ImportError message. 3. Add pyarrow>=14.0.0 to core dependencies (previously only in ingest-parquet/ingest-arrow optional extras). The documented quick-start flow of embed generate --output ... requires pyarrow out of the box. (Note: pandas>=1.3.0 is already a core dependency; pyarrow is the missing piece.) Note: .github/workflows/* files are excluded from this PR as they require a token with workflow scope. Upstream workflows are unchanged. --- .github/workflows/benchmark.yml | 51 ----- .github/workflows/ci.yml | 83 -------- .github/workflows/codeql.yml | 97 --------- .github/workflows/defender-for-devops.yml | 88 -------- .github/workflows/docs.yml | 68 ------ .github/workflows/release.yml | 73 ------- .github/workflows/security-scan.yml | 239 ---------------------- .github/workflows/security.yml | 42 ---- .github/workflows/verify-action-pins.yml | 28 --- pyproject.toml | 3 +- semantica/cli.py | 23 ++- semantica/embeddings/methods.py | 16 +- 12 files changed, 32 insertions(+), 779 deletions(-) delete mode 100644 .github/workflows/benchmark.yml delete mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/codeql.yml delete mode 100644 .github/workflows/defender-for-devops.yml delete mode 100644 .github/workflows/docs.yml delete mode 100644 .github/workflows/release.yml delete mode 100644 .github/workflows/security-scan.yml delete mode 100644 .github/workflows/security.yml delete mode 100644 .github/workflows/verify-action-pins.yml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml deleted file mode 100644 index 0157ee24..00000000 --- a/.github/workflows/benchmark.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Semantica Performance Suite - -on: - workflow_dispatch: - -permissions: - contents: read - -jobs: - performance-test: - name: Benchmark Runner (Ubuntu/Python 3.12) - runs-on: ubuntu-latest - - steps: - - name: Checkout Code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - fetch-depth: 0 - - - name: Set up Python 3.11 - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: - python-version: "3.11" - cache: 'pip' - - - name: Install Dependencies - env: - - BENCHMARK_REAL_LIBS: "1" - run: | - python -m pip install --upgrade pip - pip install -e . - pip install -r benchmarks/requirements.txt - python -m spacy download en_core_web_sm - pip install rdflib neo4j faiss-cpu torch pyarrow pdfplumber python-pptx openpyxl lxml python-docx beautifulsoup4 chardet langdetect - - - name: Execute Benchmarks (Real Mode) - env: - BENCHMARK_REAL_LIBS: "1" - run: | - python benchmarks/benchmarks_runner.py - # Optional: Compare to baseline (requires previous run artifact) - # pytest-benchmark --storage file://benchmarks/results --benchmark-compare - - - name: Upload Benchmark Results - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - if: always() - with: - name: benchmark-report-${{ github.run_id }} - path: benchmarks/results - retention-days: 30 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 4ea31ff8..00000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: CI - -permissions: - contents: read - -on: - push: - branches: [main] - paths-ignore: - - 'docs/**' - - 'docs_check.py' - - '**/*.md' - pull_request: - branches: [main] - paths-ignore: - - 'docs/**' - - 'docs_check.py' - - '**/*.md' - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: - python-version: '3.11' - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: '20' - cache: 'npm' - cache-dependency-path: explorer/package-lock.json - - name: Install Explorer frontend dependencies - working-directory: explorer - run: npm ci - - name: Test Explorer frontend - working-directory: explorer - run: | - npm run test:graph-store - npm run test:graph-workspace - npm run test:plugin-registry - - name: Build Explorer frontend - working-directory: explorer - run: npm run build - - name: Install pinned Python dependencies - run: | - pip install -r requirements-ci.txt - - name: Verify requirements-ci.txt is up to date - run: | - pip install uv==0.12.1 - # Re-resolve with the committed file as a constraint: upstream package - # releases must NOT fail CI (deps only change when pyproject.toml - # changes intentionally). Compare only version lines (pkg==ver), - # ignoring the -c constraint comments and the `\` line continuations - # that --generate-hashes emits. - uv pip compile pyproject.toml --python-version 3.11 --extra all \ - --constraint requirements-ci.txt -o /tmp/requirements-ci-check.txt - diff \ - <(grep -E '^[a-zA-Z0-9._-]+==' requirements-ci.txt | sed 's/ \\$//') \ - <(grep -E '^[a-zA-Z0-9._-]+==' /tmp/requirements-ci-check.txt) - - run: pip install build - # wheel is build-time only (not in requirements-ci.txt) — install the - # same pinned version [build-system] declares so --no-isolation works. - - run: pip install wheel==0.48.0 - - name: Build package (no isolation — pinned deps) - run: python -m build --no-isolation - - name: Verify Explorer frontend is packaged - run: | - python - <<'PY' - import zipfile - from pathlib import Path - - wheels = list(Path("dist").glob("*.whl")) - assert wheels, "No wheel was built" - - with zipfile.ZipFile(wheels[0]) as wheel: - names = set(wheel.namelist()) - - assert "semantica/static/index.html" in names, "Explorer index.html missing from wheel" - assert any(name.startswith("semantica/static/assets/") for name in names), "Explorer assets missing from wheel" - - print("Explorer frontend is packaged") - PY diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml deleted file mode 100644 index 0c15e84c..00000000 --- a/.github/workflows/codeql.yml +++ /dev/null @@ -1,97 +0,0 @@ -name: CodeQL - -on: - push: - branches: [main] - pull_request: - branches: [main] - schedule: - - cron: '30 1 * * 1' # Every Monday 7 AM IST - -permissions: - contents: read - security-events: write - actions: read - -jobs: - analyze: - name: Analyze Python - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - # The CodeQL bundle download (github/codeql-action/init's "Setup CodeQL - # tools" step) streams a ~1GB tarball from GitHub's release CDN and - # does not retry on a transient connection reset (ECONNRESET) itself - # (github/codeql-action, unresolved as of v4 / CLI 2.26.1: the HTTP - # error is retryable but isn't retried internally). Since a `uses:` - # step can't be wrapped by a shell-level retry action, attempt init - # up to 3 times; each retry is a fresh download attempt with no - # meaningful state carried over from a failed attempt. - - name: Initialize CodeQL (attempt 1) - id: codeql-init-1 - uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 - continue-on-error: true - with: - languages: python - queries: security-and-quality - config-file: .github/codeql/codeql-config.yml - - - name: Initialize CodeQL (attempt 2) - id: codeql-init-2 - if: steps.codeql-init-1.outcome == 'failure' - uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 - continue-on-error: true - with: - languages: python - queries: security-and-quality - config-file: .github/codeql/codeql-config.yml - - - name: Initialize CodeQL (attempt 3) - id: codeql-init-3 - if: steps.codeql-init-2.outcome == 'failure' - uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 - with: - languages: python - queries: security-and-quality - config-file: .github/codeql/codeql-config.yml - - - name: Autobuild - uses: github/codeql-action/autobuild@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 - with: - category: "/language:python" - upload: false - id: codeql - - - name: Upload SARIF (Advanced Setup only) - # Uploads results only when Default Setup is not active. - # If Default Setup is still enabled, this step skips gracefully - # instead of failing the workflow with HTTP 409. - uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 - with: - sarif_file: ${{ steps.codeql.outputs.sarif-output }} - category: "/language:python" - wait-for-processing: true - continue-on-error: true - - # NOTE: Auto-dismissal by rule-id is intentionally removed. - # Dismissing every alert that matches a rule ID would silently suppress - # future real vulnerabilities of the same type. The alerts below were - # individually triaged and dismissed manually in the security-enhancement - # PR (alerts #12–#18). New alerts must be reviewed and dismissed by hand, - # or will auto-close when the underlying code no longer triggers them. - # - # If you need to dismiss a specific known-safe alert, pin its alert NUMBER - # here and remove it once CodeQL stops reporting it naturally. Example: - # - # PINNED_ALERT_NUMBERS=(12 13 14 15 16 17 18) - # for NUM in "${PINNED_ALERT_NUMBERS[@]}"; do - # gh api repos/$REPO/code-scanning/alerts/$NUM \ - # -X PATCH -f state=dismissed -f dismissed_reason="false positive" \ - # -f dismissed_comment="" - # done diff --git a/.github/workflows/defender-for-devops.yml b/.github/workflows/defender-for-devops.yml deleted file mode 100644 index becb7d64..00000000 --- a/.github/workflows/defender-for-devops.yml +++ /dev/null @@ -1,88 +0,0 @@ -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. -# -# Microsoft Security DevOps (MSDO) is a command line application which integrates static analysis tools into the development cycle. -# MSDO installs, configures and runs the latest versions of static analysis tools -# (including, but not limited to, SDL/security and compliance tools). -# -# The Microsoft Security DevOps action is currently in beta and runs on the windows-latest queue, -# as well as Windows self hosted agents. ubuntu-latest support coming soon. -# -# For more information about the action , check out https://github.com/microsoft/security-devops-action -# -# Please note this workflow do not integrate your GitHub Org with Microsoft Defender For DevOps. You have to create an integration -# and provide permission before this can report data back to azure. -# Read the official documentation here : https://learn.microsoft.com/en-us/azure/defender-for-cloud/quickstart-onboard-github - -name: "Microsoft Defender For Devops" - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - schedule: - - cron: '43 17 * * 6' - -permissions: - contents: read - security-events: write - -jobs: - MSDO: - # currently only windows-latest is supported - runs-on: windows-latest - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6 - with: - dotnet-version: | - 5.0.x - 6.0.x - - name: Run Microsoft Security DevOps - uses: microsoft/security-devops-action@08976cb623803b1b36d7112d4ff9f59eae704de0 # v1.12.0 - id: msdo - with: - # checkov is intentionally excluded from this MSDO step. - # MSDO 0.215.0's guardian.cmd wrapper treats checkov's exit code 1 - # (emitted whenever any violation is found, even below the active severity - # threshold) as a fatal "tool error" and breaks the build even when - # "Active results: 0" and "Found no breaking results." The .checkov.yaml - # soft-fail setting is never read by the guardian wrapper. - # IaC security scanning continues below in this same MSDO job identity. - # That preserves the existing GitHub code-scanning configuration while - # avoiding the guardian.cmd/checkov exit-code bug in the MSDO wrapper. - tools: eslint,templateanalyzer,terrascan - - name: Upload results to Security tab - uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 - with: - sarif_file: ${{ steps.msdo.outputs.sarifFile }} - - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: - python-version: "3.12" - - - name: Install Checkov - run: python -m pip install checkov==3.3.1 - - - name: Run Checkov - shell: pwsh - env: - PYTHONUTF8: "1" - run: | - New-Item -ItemType Directory -Force reports | Out-Null - checkov --directory . --framework kubernetes helm dockerfile github_actions secrets bicep arm --soft-fail --output sarif --output-file-path reports/checkov.sarif - if (-not (Test-Path reports/checkov.sarif)) { - $sarif = Get-ChildItem -Path reports -Recurse -Filter *.sarif | Select-Object -First 1 - if ($null -eq $sarif) { throw "Checkov did not produce a SARIF file" } - Copy-Item $sarif.FullName reports/checkov.sarif - } - - - name: Upload Checkov results to Security tab - uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 - if: always() - with: - sarif_file: reports/checkov.sarif diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index ca149d7e..00000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: Build and Deploy Documentation - -on: - push: - branches: [main] - paths: - - 'docs/**' - - 'docs_check.py' - - 'CHANGELOG.md' - - 'RELEASE.md' - pull_request: - branches: [main] - paths: - - 'docs/**' - - 'docs_check.py' - workflow_dispatch: - -permissions: - contents: read - pages: write - id-token: write - -concurrency: - group: "pages" - cancel-in-progress: false - -jobs: - validate: - name: Validate Documentation - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: - python-version: '3.11' - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: '20' - - run: python docs_check.py - - deploy: - name: Build and Deploy to GitHub Pages - if: github.event_name != 'pull_request' - runs-on: ubuntu-latest - needs: validate - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: '20' - - - name: Export static site - run: | - cd docs - npx mintlify export --output ../export.zip - cd .. - unzip -q export.zip -d site - - - uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6 - - - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5 - with: - path: ./site - - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index bbc8770c..00000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: Release - -on: - push: - tags: ['v*'] - -permissions: - contents: read - -jobs: - release: - runs-on: ubuntu-latest - environment: pypi - concurrency: - group: release-${{ github.ref }} - cancel-in-progress: false - permissions: - contents: write # for the GitHub Release - id-token: write # for PyPI Trusted Publishing (OIDC) and attestation signing - attestations: write # for SLSA build provenance - # If you add another job to this workflow, give it its own explicit - # `permissions:` block rather than relying on the workflow-level default - # above (contents: read) - do not widen the workflow-level default. - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: - python-version: '3.11' - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: '20' - cache: 'npm' - cache-dependency-path: explorer/package-lock.json - - name: Build Explorer frontend - working-directory: explorer - run: | - npm ci - npm run build - # Install the pinned dependency set (with hashes) so the sdist/wheel - # build runs against the same versions CI tests against. - - name: Install pinned build dependencies - run: pip install -r requirements-ci.txt - - run: pip install build - # wheel is build-time only (not in requirements-ci.txt) — install the - # same pinned version [build-system] declares so --no-isolation works. - - run: pip install wheel==0.48.0 - - name: Build package (no isolation — pinned deps) - run: python -m build --no-isolation - - name: Verify Explorer frontend is packaged - run: | - python - <<'PY' - import zipfile - from pathlib import Path - - wheels = list(Path("dist").glob("*.whl")) - assert wheels, "No wheel was built" - - with zipfile.ZipFile(wheels[0]) as wheel: - names = set(wheel.namelist()) - - assert "semantica/static/index.html" in names, "Explorer index.html missing from wheel" - assert any(name.startswith("semantica/static/assets/") for name in names), "Explorer assets missing from wheel" - - print("Explorer frontend is packaged") - PY - - name: Attest build provenance - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4 - with: - subject-path: 'dist/*' - - uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3 - with: - files: dist/* - - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1 diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml deleted file mode 100644 index 5b4461af..00000000 --- a/.github/workflows/security-scan.yml +++ /dev/null @@ -1,239 +0,0 @@ -name: Security Scan - -on: - schedule: - - cron: '30 1 * * 1,4' # Mon/Thu 7 AM IST - push: - branches: [main] - paths-ignore: - - 'docs/**' - - 'mkdocs.yml' - - 'requirements-docs.txt' - - '**/*.md' - pull_request: - branches: [main] - paths-ignore: - - 'docs/**' - - 'mkdocs.yml' - - 'requirements-docs.txt' - - '**/*.md' - -permissions: - contents: read - -jobs: - security-scan: - runs-on: ubuntu-latest - permissions: - contents: read - security-events: write - actions: read - # Needed for the "Comment PR with Security Results" step below. Safe on - # pull_request (not pull_request_target): GitHub always forces a - # read-only token for PRs from forks regardless of this permission. - pull-requests: write - - steps: - - name: Checkout repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: - python-version: '3.11' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - # Install the pinned dependency set FIRST so Safety scans Semantica's - # exact CI/release dependency tree (requirements-ci.txt is generated - # from pyproject.toml extras, so this covers the project's real deps). - pip install -r requirements-ci.txt - # Tooling AFTER the pinned set: installing safety/bandit/semgrep/jq - # first lets the pinned requirements overwrite their transitive deps - # (e.g. rich), which breaks the safety CLI at runtime. - pip install safety bandit semgrep jq - - - name: Run Safety Check (Package Vulnerabilities) - run: | - # NOTE: Safety 3.x repurposed --output to select a console format - # (json/text/screen/...), not a file path. Writing JSON to a file - # now requires --save-json; the previous `--output safety-report.json` - # usage was silently invalid and never produced a report. - safety check --save-json safety-report.json || true - - # Guard 1: fail loudly if Safety exited before writing a report at all - # (network error, API auth failure, tool crash). Without this check a - # missing or empty file causes jq to fall back to "0", making a broken - # scanner indistinguishable from a clean scan. - if [ ! -s safety-report.json ]; then - echo "::error::Safety scan produced no report (safety-report.json is missing or empty). Treating as failure — check for network errors, API auth failures, or Safety crashes in the logs above." - exit 1 - fi - - echo "Checking for package vulnerabilities..." - - # No || echo "0" fallback: if jq fails (malformed JSON, missing key, - # vulnerabilities:null) VULNS will be empty or "null" so guard 2 below - # catches it rather than silently treating the broken report as zero. - VULNS=$(jq '.vulnerabilities | length' safety-report.json 2>/dev/null) - - # Guard 2: ensure VULNS is a non-negative integer before the -gt - # comparison. "null" (missing/null key) or "" (jq parse failure) would - # cause bash's -gt to throw an arithmetic error and fall through to the - # success branch — the same silent-pass bug as a missing file. - if ! [[ "$VULNS" =~ ^[0-9]+$ ]]; then - echo "::error::Safety report exists but 'vulnerabilities' is missing or non-numeric (got: '${VULNS}'). The report may be malformed or Safety may have written an error-only JSON. Treating as failure." - exit 1 - fi - - if [ "$VULNS" -gt 0 ]; then - echo "❌ Security vulnerabilities found: $VULNS" - echo "CI will fail to prevent merging of vulnerable dependencies" - echo "" - echo "Vulnerability details:" - jq -r '.vulnerabilities[] | "- \(.package_name)==\(.analyzed_version): \(.vulnerability_id) (\(.CVE // "no CVE assigned"))"' safety-report.json || true - exit 1 - else - echo "✅ No security vulnerabilities found" - fi - - - name: Run Bandit (Code Security Linter) - run: | - bandit -r semantica/ -f json -o bandit-report.json || true - echo "Checking for HIGH severity security issues..." - - # Count HIGH severity issues - HIGH_ISSUES=$(bandit -r semantica/ -f json -ll 2>/dev/null | jq -r '.results[]? | select(.issue_severity == "HIGH") | .test_name' 2>/dev/null | wc -l || echo "0") - - if [ "$HIGH_ISSUES" -gt 0 ]; then - echo "❌ HIGH severity security issues found: $HIGH_ISSUES" - echo "CI will fail to prevent merging of high-risk code" - echo "" - echo "High severity issues:" - bandit -r semantica/ -ll | grep "Severity: High" -A 5 -B 1 || true - exit 1 - else - echo "✅ No HIGH severity security issues found" - fi - - - name: Run Semgrep (Static Analysis) - run: | - echo "Running Semgrep static analysis..." - semgrep --config=auto --json --output=semgrep-report.json semantica/ || true - - # Run security-focused rules - echo "Checking for security patterns..." - SECURITY_ISSUES=$(semgrep --config=p/security --json semantica/ 2>/dev/null | jq '.results | length' 2>/dev/null || echo "0") - - if [ "$SECURITY_ISSUES" -gt 0 ]; then - echo "⚠️ Security patterns found: $SECURITY_ISSUES" - echo "Review these findings for potential improvements" - semgrep --config=p/security semantica/ || true - else - echo "✅ No security patterns found" - fi - - - name: Upload Security Reports - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: security-reports - retention-days: 14 - path: | - safety-report.json - bandit-report.json - semgrep-report.json - - - name: Comment PR with Security Results - if: github.event_name == 'pull_request' - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 - with: - script: | - const fs = require('fs'); - - // Renders one tool's findings as a section. `items` is already - // the list of pre-formatted "- `thing` in `where`" strings; this - // just handles the found/not-found/report-missing framing and - // collapses long lists into a
block so the comment - // doesn't turn into a wall of text. - function renderSection(title, reportPath, parse) { - let data; - try { - data = JSON.parse(fs.readFileSync(reportPath, 'utf8')); - } catch (e) { - return [ - `### ${title}`, - `⚠️ No report found at \`${reportPath}\` — the scan may have failed before producing output. Check the job logs.`, - ].join('\n'); - } - - const items = parse(data); - if (items.length === 0) { - return [`### ${title}`, `✅ No findings.`].join('\n'); - } - - const lines = [`### ${title}`, `Found **${items.length}**.`, '']; - const shown = items.slice(0, 15); - if (items.length > 15) { - lines.push('
', 'Show all findings', ''); - lines.push(...items); - lines.push('', '
'); - } else { - lines.push(...shown); - } - return lines.join('\n'); - } - - const safetySection = renderSection( - 'Safety — dependency vulnerabilities', - 'safety-report.json', - (data) => (data.vulnerabilities || []).map( - (v) => `- \`${v.package_name}==${v.analyzed_version}\`: ${v.vulnerability_id}` + - (v.CVE ? ` (${v.CVE})` : '') + ` — ${v.advisory || 'no advisory text'}` - ) - ); - - const banditSection = renderSection( - 'Bandit — HIGH-severity code issues', - 'bandit-report.json', - (data) => (data.results || []) - .filter((issue) => issue.issue_severity === 'HIGH') - .map((issue) => `- \`${issue.test_name}\` in \`${issue.filename}:${issue.line_number}\``) - ); - - const semgrepSection = renderSection( - 'Semgrep — static analysis patterns', - 'semgrep-report.json', - (data) => (data.results || []).map( - (issue) => `- \`${issue.check_id}\` in \`${issue.path}:${issue.start?.line ?? '?'}\`` - ) - ); - - const comment = [ - '# 🔒 Security Scan Results', - '', - safetySection, - '', - banditSection, - '', - semgrepSection, - '', - '---', - '', - '*This security scan runs automatically on source-code PRs and bi-weekly (skipped for doc/markdown-only changes).*', - '', - '📊 **Security Policy**: CI fails on Safety vulnerabilities and Bandit HIGH-severity findings. Semgrep findings above are informational and do not block merge.', - ].join('\n'); - - try { - await github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: comment, - }); - console.log('✅ Security comment posted successfully'); - } catch (error) { - console.log('⚠️ Could not post security comment:', error.message); - console.log('📋 Security scan results saved to artifacts'); - } diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml deleted file mode 100644 index 412e7eaa..00000000 --- a/.github/workflows/security.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Security - -on: - schedule: - - cron: '0 0 * * 1' - workflow_dispatch: - pull_request: - branches: [main] - paths: - - 'pyproject.toml' - - 'requirements-ci.txt' - - '.github/workflows/security.yml' - -permissions: - contents: read - -jobs: - audit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 - with: - python-version: '3.11' - # Upgrade first: actions/setup-python's baked-in setuptools has been - # behind known-vulnerable floors before (e.g. PYSEC-2026-3447 / - # setuptools 75.1.0), so don't trust the preinstalled one. - - run: python -m pip install --upgrade pip setuptools - # Audit the pinned dependency set (requirements-ci.txt is compiled from - # pyproject.toml with --extra all — the same coverage as the [all] - # extra, minus the Linux-only gpu set — so this keeps scan parity with - # CI/release builds without a time-dependent resolution). This is the - # fix for PYSEC-2024-38 (#869): the bare-env job never had fastapi or - # python-multipart installed to look at. - - run: pip install -r requirements-ci.txt - # PR runs gate on findings, since they're scoped to actual - # pyproject.toml changes under review. The schedule/workflow_dispatch - # runs stay non-blocking until a full pass over pre-existing findings - # across the whole [all] tree has been done. - - run: pip install pip-audit - - run: pip-audit -r requirements-ci.txt - continue-on-error: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/verify-action-pins.yml b/.github/workflows/verify-action-pins.yml deleted file mode 100644 index 6e7dada9..00000000 --- a/.github/workflows/verify-action-pins.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Verify Action Pins - -on: - pull_request: - paths: - - '.github/workflows/**' - - '.github/scripts/verify-action-pins.sh' - push: - branches: [main] - paths: - - '.github/workflows/**' - - '.github/scripts/verify-action-pins.sh' - schedule: - - cron: '0 3 * * 1' # weekly, in case an upstream tag is deliberately moved - workflow_dispatch: - -permissions: - contents: read - -jobs: - verify: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - name: Verify pinned action SHAs match their tag comments - env: - GH_TOKEN: ${{ github.token }} - run: bash .github/scripts/verify-action-pins.sh diff --git a/pyproject.toml b/pyproject.toml index 03949d4e..738e68ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,7 +85,8 @@ dependencies = [ "loguru>=0.7.3", "structlog>=22.1.0", "gensim>=4.4.0", - "httpx<0.29.0" + "httpx<0.29.0", + "pyarrow>=14.0.0" ] [project.urls] diff --git a/semantica/cli.py b/semantica/cli.py index 7b944dc6..b5e14ebd 100644 --- a/semantica/cli.py +++ b/semantica/cli.py @@ -1708,7 +1708,28 @@ def embed_generate(cli_ctx: CLIContext, input_path: str, model: str, except ImportError as exc: raise click.ClickException(f"Embeddings module not available: {exc}") from exc if output: - Path(output).write_text(json.dumps(result, default=str), encoding="utf-8") + output_path = Path(output) + try: + import numpy as np + import pandas as pd + if output_path.suffix.lower() == ".parquet": + arr = np.asarray(result) + if arr.ndim == 1: + arr = arr[np.newaxis, :] + # schema: one column per embedding plus an id column + columns = [f"dim_{i}" for i in range(arr.shape[1])] + df = pd.DataFrame(arr, columns=columns) + df.index.name = "id" + df.to_parquet(output_path, index=True) + else: + output_path.write_text( + json.dumps(result, default=str), encoding="utf-8" + ) + except ImportError as exc: + raise click.ClickException( + f"Missing dependency for --output: {exc}. " + f"Install pyarrow/pandas with: pip install semantica[ingest-parquet]" + ) from exc _ok(cli_ctx, f"Wrote {output}") elif _is_json(cli_ctx, local_json): _jecho(result if isinstance(result, dict) else {"status": "ok"}) diff --git a/semantica/embeddings/methods.py b/semantica/embeddings/methods.py index 30c47279..d2e8848e 100644 --- a/semantica/embeddings/methods.py +++ b/semantica/embeddings/methods.py @@ -116,9 +116,9 @@ def generate_embeddings( >>> emb = generate_embeddings("Hello world", method="default") >>> embs = generate_embeddings(["text1", "text2"], method="text") """ - # Check for custom method in registry + # Check for custom method in registry, skip self-reference custom_method = method_registry.get("generation", method) - if custom_method: + if custom_method and custom_method is not generate_embeddings: try: return custom_method(data, data_type=data_type, **kwargs) except Exception as e: @@ -164,9 +164,9 @@ def embed_text( >>> emb = embed_text("Hello world", method="sentence_transformers") >>> embs = embed_text(["text1", "text2"], method="sentence_transformers") """ - # Check for custom method in registry + # Check for custom method in registry, skip self-reference custom_method = method_registry.get("text", method) - if custom_method: + if custom_method and custom_method is not embed_text: try: return custom_method(text, **kwargs) except Exception as e: @@ -224,9 +224,9 @@ def calculate_similarity( >>> similarity = calculate_similarity(emb1, emb2, method="cosine") >>> print(f"Similarity: {similarity:.3f}") """ - # Check for custom method in registry + # Check for custom method in registry, skip self-reference custom_method = method_registry.get("similarity", method) - if custom_method: + if custom_method and custom_method is not calculate_similarity: try: return custom_method(embedding1, embedding2, **kwargs) except Exception as e: @@ -271,9 +271,9 @@ def pool_embeddings( >>> pooled = pool_embeddings(embeddings, method="mean") >>> attention_pooled = pool_embeddings(embeddings, method="attention") """ - # Check for custom method in registry + # Check for custom method in registry, skip self-reference custom_method = method_registry.get("pooling", method) - if custom_method: + if custom_method and custom_method is not pool_embeddings: try: return custom_method(embeddings, **kwargs) except Exception as e: From 616f5ca9b9a6c5f6f3b7bb2ee878021f3d831416 Mon Sep 17 00:00:00 2001 From: Varun Sahni Date: Sat, 15 Aug 2026 14:04:15 +0530 Subject: [PATCH 02/20] fix: use list[float] vector column in embed generate Parquet output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Qodo finding: embed generate wrote scalar dim_* columns, but embed index only detects embeddings when a column's values are list/np.ndarray. This broke the generate→index pipeline with 'No vector column found'. Fix: write a single 'embedding' column where each value is a list[float], matching what embed index's isinstance(df[c].iloc[0], (list, np.ndarray)) check expects. Row indices serve as ids (embed index will pass ids=None to create_index, which is acceptable — vectors index correctly regardless). Also addressed from Qodo review: - .parquet suffix check is now case-insensitive (.lower()) - pandas already a core dependency (bot was wrong) - pyarrow dependency remains added --- semantica/cli.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/semantica/cli.py b/semantica/cli.py index b5e14ebd..f55d8939 100644 --- a/semantica/cli.py +++ b/semantica/cli.py @@ -1716,11 +1716,17 @@ def embed_generate(cli_ctx: CLIContext, input_path: str, model: str, arr = np.asarray(result) if arr.ndim == 1: arr = arr[np.newaxis, :] - # schema: one column per embedding plus an id column - columns = [f"dim_{i}" for i in range(arr.shape[1])] - df = pd.DataFrame(arr, columns=columns) + # Schema: single 'embedding' column (list[float] per row). + # embed index detects vector columns via + # isinstance(df[c].iloc[0], (list, np.ndarray)). + # Row indices serve as ids: embed index will see ids=None + # but vectors will index correctly regardless. + df = pd.DataFrame({ + "embedding": [list(row) for row in arr], + }) df.index.name = "id" - df.to_parquet(output_path, index=True) + df.index = [str(i) for i in range(len(arr))] + df.to_parquet(output_path, index=False) else: output_path.write_text( json.dumps(result, default=str), encoding="utf-8" From 4b6cc095850e4ed692d600c8ac088faf6de7f07a Mon Sep 17 00:00:00 2001 From: Varun Sahni Date: Sun, 16 Aug 2026 15:49:39 +0530 Subject: [PATCH 03/20] fix: write JSON/JSONL output as real lists, reject unsupported formats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The non-Parquet branch still used json.dumps(result, default=str), which stringifies numpy arrays to their repr() — the same corrupt-output bug #994 reports, just for .json/.jsonl extensions instead of .parquet. embed index reads .json/.jsonl via pd.read_json(lines=...) and detects a vector column by isinstance(val, (list, np.ndarray)); a repr() string fails that check, so generate→index still breaks for JSON outputs. - .json/.jsonl now use pandas to_json(orient='records') with real lists - Unsupported extensions (.txt, .csv, etc.) now raise ClickException instead of silently writing JSON text, matching embed index behavior - Error message corrected: pyarrow is now a core dep, not an extra --- semantica/cli.py | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/semantica/cli.py b/semantica/cli.py index f55d8939..37a3a978 100644 --- a/semantica/cli.py +++ b/semantica/cli.py @@ -1709,32 +1709,41 @@ def embed_generate(cli_ctx: CLIContext, input_path: str, model: str, raise click.ClickException(f"Embeddings module not available: {exc}") from exc if output: output_path = Path(output) + suffix = output_path.suffix.lower() try: import numpy as np import pandas as pd - if output_path.suffix.lower() == ".parquet": - arr = np.asarray(result) - if arr.ndim == 1: - arr = arr[np.newaxis, :] + arr = np.asarray(result) + if arr.ndim == 1: + arr = arr[np.newaxis, :] + if arr.ndim != 2: + raise click.ClickException( + f"embed generate --output expects a 1-D or 2-D array, " + f"got {arr.ndim}-D (shape {arr.shape})" + ) + rows = [list(row) for row in arr] + if suffix == ".parquet": # Schema: single 'embedding' column (list[float] per row). # embed index detects vector columns via # isinstance(df[c].iloc[0], (list, np.ndarray)). - # Row indices serve as ids: embed index will see ids=None - # but vectors will index correctly regardless. - df = pd.DataFrame({ - "embedding": [list(row) for row in arr], - }) - df.index.name = "id" - df.index = [str(i) for i in range(len(arr))] + df = pd.DataFrame({"embedding": rows}) df.to_parquet(output_path, index=False) + elif suffix in (".json", ".jsonl"): + df = pd.DataFrame({"embedding": rows}) + df.to_json( + output_path, + orient="records", + lines=(suffix == ".jsonl"), + ) else: - output_path.write_text( - json.dumps(result, default=str), encoding="utf-8" + raise click.ClickException( + f"Unsupported output format '{suffix}'. " + "Use .parquet, .json, or .jsonl" ) except ImportError as exc: raise click.ClickException( f"Missing dependency for --output: {exc}. " - f"Install pyarrow/pandas with: pip install semantica[ingest-parquet]" + "Install pyarrow with: pip install pyarrow" ) from exc _ok(cli_ctx, f"Wrote {output}") elif _is_json(cli_ctx, local_json): From 5d554ec58614224447eb35d1726df6a12f8be1c6 Mon Sep 17 00:00:00 2001 From: Varun Sahni Date: Thu, 20 Aug 2026 08:17:46 +0530 Subject: [PATCH 04/20] fix: cherry-pick recursion guard and doctor embedding checks from #1005, #1006 Consolidates the remaining #994 fixes into this PR so it can fully close the issue, per maintainer request. From #1005 (yzxcj797): - EmbeddingGeneratorWithProvenance.__getattr__ self-recursion guard: accessing self._generator via attribute syntax re-entered __getattr__ forever when _generator was absent (failed __init__, pickle/copy probes like __deepcopy__). Private-name lookups now raise AttributeError. - 4 regression tests in TestMethodDispatchRecursion: default dispatch no longer self-recurses for generation/text, a user-registered custom method still takes precedence, and a bare provenance wrapper raises AttributeError instead of RecursionError. (The methods.py identity guards from #1005 are already present here.) From #1006 (yzxcj797): - doctor gains two embedding backend checks, "Embeddings (sentence-transformers)" and "Embeddings (fastembed)". Default is a cheap import+version check (uninstalled backend now reports fail with a pip hint instead of invisible). --deep-embeddings (or SEMANTICA_DOCTOR_DEEP_EMBEDDINGS=1) instantiates via TextEmbedder and embeds a probe, catching backends that import cleanly but cannot load (the #994 failure mode) via the hash-fallback-active signal. _DeepEmbeddingFailure marks post-import runtime/model-load failures so they get a remediation hint instead of a misleading pip-install hint. - 7 tests in TestDoctorEmbeddings and TestDoctorEmbeddingHintsAndEnv. Validation: - tests/test_cli_commands.py: 237 passed (7 new) - tests/test_embedding_providers.py: 9 passed (4 new) - AST parse + import of all four modules OK --- semantica/cli.py | 68 ++++++++++- semantica/embeddings/embeddings_provenance.py | 9 ++ tests/test_cli_commands.py | 115 ++++++++++++++++++ tests/test_embedding_providers.py | 46 +++++++ 4 files changed, 237 insertions(+), 1 deletion(-) diff --git a/semantica/cli.py b/semantica/cli.py index 7f886a82..805bccdc 100644 --- a/semantica/cli.py +++ b/semantica/cli.py @@ -773,10 +773,22 @@ def changelog(cli_ctx: CLIContext, local_json: bool) -> None: _run_with_error_handling(_action) +class _DeepEmbeddingFailure(Exception): + """A deep-probe failure from doctor's embedding checks. + + Marks failures that happened AFTER the backend imported cleanly — model + load, probe, or runtime problems — so the check's hint can point at the + real remediation instead of `pip install`. + """ + + @main.command() @click.option("--json", "local_json", is_flag=True, default=False) +@click.option("--deep-embeddings", "deep_embeddings", is_flag=True, default=False, + help="Also instantiate the local embedding backends and embed a probe " + "text (catches backends that import cleanly but cannot load).") @click.pass_obj -def doctor(cli_ctx: CLIContext, local_json: bool) -> None: +def doctor(cli_ctx: CLIContext, local_json: bool, deep_embeddings: bool) -> None: """Run a health check on all Semantica components and backends.""" import importlib.metadata cli_ctx = _require_ctx(cli_ctx) @@ -787,6 +799,16 @@ def doctor(cli_ctx: CLIContext, local_json: bool) -> None: try: note = fn() return label, "ok", note, None + except _DeepEmbeddingFailure as exc: + # A deep-probe failure means the package IMPORTED fine: the pip + # hint would be the wrong remediation for what is actually a + # runtime/model-load problem (broken torch, failed model + # download, missing shared libs). + return label, "fail", str(exc), ( + "runtime/model-load failure — reinstalling the package usually " + "does not help; check the warnings above (torch install, model " + "download, disk space)" + ) except Exception as exc: return label, "fail", str(exc), hint @@ -827,6 +849,50 @@ def doctor(cli_ctx: CLIContext, local_json: bool) -> None: return f"{backend} importable" checks.append(_check("Vector store", _vector, hint="pip install semantica[vectorstore-…]")) + # Embedding backends (#994): `doctor` used to report all green while + # every local embedding backend was non-functional — import success + # says nothing about model loading. Default checks stay cheap + # (import + version); --deep-embeddings (or + # SEMANTICA_DOCTOR_DEEP_EMBEDDINGS=1) instantiates the backend through + # TextEmbedder and embeds a probe, which is the only level that + # catches a backend that imports cleanly but cannot actually load. + deep = deep_embeddings or os.environ.get("SEMANTICA_DOCTOR_DEEP_EMBEDDINGS", "").strip().lower() in ("1", "true", "yes", "on") + + def _embedding_backend(method: str) -> str: + if method == "sentence_transformers": + import sentence_transformers # noqa: F401 + note = f"importable ({importlib.metadata.version('sentence-transformers')})" + else: + import fastembed # noqa: F401 + note = f"importable ({importlib.metadata.version('fastembed')})" + if not deep: + return note + try: + from .embeddings import TextEmbedder + embedder = TextEmbedder(method=method) + if embedder.model is None and embedder.fastembed_model is None: + raise RuntimeError( + "model failed to load — the hash fallback is active " + "(see warnings above); embedding quality is degraded" + ) + probe = embedder.embed_text("semantica doctor embedding probe") + except _DeepEmbeddingFailure: + raise + except Exception as exc: + raise _DeepEmbeddingFailure(str(exc)) from exc + return f"{note}; deep probe ok ({len(probe)}-dim)" + + checks.append(_check( + "Embeddings (sentence-transformers)", + lambda: _embedding_backend("sentence_transformers"), + hint="pip install sentence-transformers", + )) + checks.append(_check( + "Embeddings (fastembed)", + lambda: _embedding_backend("fastembed"), + hint="pip install fastembed", + )) + # LLM provider keys for provider, var in [("OpenAI", "OPENAI_API_KEY"), ("Anthropic", "ANTHROPIC_API_KEY"), ("Groq", "GROQ_API_KEY")]: diff --git a/semantica/embeddings/embeddings_provenance.py b/semantica/embeddings/embeddings_provenance.py index 03d7adbd..886fd8c1 100644 --- a/semantica/embeddings/embeddings_provenance.py +++ b/semantica/embeddings/embeddings_provenance.py @@ -69,6 +69,15 @@ class EmbeddingGeneratorWithProvenance: return embeddings def __getattr__(self, name): + # __getattr__ only runs when normal lookup fails. Accessing + # self._generator by attribute syntax HERE would re-enter + # __getattr__ for ever when _generator itself is missing — the shape + # pickle/copy protocol probes hit when __init__ never completed + # (#994's RecursionError family). Fail fast on private probes. + if name.startswith("_"): + raise AttributeError( + f"{type(self).__name__!r} object has no attribute {name!r}" + ) return getattr(self._generator, name) diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index 44e6d2a9..f2b626f7 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -1851,3 +1851,118 @@ class TestExitCodes: assert "Traceback" not in result.output, ( f"Traceback found for {argv}: {result.output}" ) + + +class TestDoctorEmbeddings: + """#994: doctor must surface non-functional embedding backends instead of + reporting all green. Default = import-level check; --deep-embeddings (or + SEMANTICA_DOCTOR_DEEP_EMBEDDINGS=1) instantiates via TextEmbedder.""" + + def _doctor_checks(self, runner, *extra): + result = runner.invoke(cli_module.main, ["doctor", "--json", *extra]) + _ok(result) + import json as _json + return {c["check"]: c for c in _json.loads(result.output)} + + def _with_fake_st(self, monkeypatch, **embedder_attrs): + fake_st = _fake_module( + __version__="9.9.9", + SentenceTransformer=object, + ) + monkeypatch.setitem(__import__("sys").modules, "sentence_transformers", fake_st) + + def test_doctor_reports_embedding_checks(self, runner): + checks = self._doctor_checks(runner) + assert "Embeddings (sentence-transformers)" in checks + assert "Embeddings (fastembed)" in checks + + def test_import_failure_is_fail_status_with_hint(self, runner): + checks = self._doctor_checks(runner) + st = checks["Embeddings (sentence-transformers)"] + if st["status"] == "fail": + assert st["hint"] == "pip install sentence-transformers" + + def test_deep_probe_detects_fallback_active(self, runner, monkeypatch): + self._with_fake_st(monkeypatch) + fake_embedder = types.SimpleNamespace(model=None, fastembed_model=None) + + fake_emb_mod = _fake_module(TextEmbedder=lambda **k: fake_embedder) + monkeypatch.setitem(__import__("sys").modules, "semantica.embeddings", fake_emb_mod) + + checks = self._doctor_checks(runner, "--deep-embeddings") + st = checks["Embeddings (sentence-transformers)"] + assert st["status"] == "fail" + assert "hash fallback" in st["note"] + + def test_deep_probe_ok_when_model_loads(self, runner, monkeypatch): + self._with_fake_st(monkeypatch) + import numpy as np + fake_embedder = types.SimpleNamespace( + model=object(), + fastembed_model=None, + embed_text=lambda text: np.zeros(384, dtype=np.float32), + ) + fake_emb_mod = _fake_module(TextEmbedder=lambda **k: fake_embedder) + monkeypatch.setitem(__import__("sys").modules, "semantica.embeddings", fake_emb_mod) + + checks = self._doctor_checks(runner, "--deep-embeddings") + st = checks["Embeddings (sentence-transformers)"] + assert st["status"] == "ok" + assert "384-dim" in st["note"] + + def test_env_var_enables_deep_mode(self, runner, monkeypatch): + monkeypatch.setenv("SEMANTICA_DOCTOR_DEEP_EMBEDDINGS", "1") + self._with_fake_st(monkeypatch) + fake_embedder = types.SimpleNamespace(model=None, fastembed_model=None) + fake_emb_mod = _fake_module(TextEmbedder=lambda **k: fake_embedder) + monkeypatch.setitem(__import__("sys").modules, "semantica.embeddings", fake_emb_mod) + + checks = self._doctor_checks(runner) + st = checks["Embeddings (sentence-transformers)"] + assert st["status"] == "fail" + assert "hash fallback" in st["note"] + + +class TestDoctorEmbeddingHintsAndEnv: + """Review follow-ups: deep failures must not carry the pip-install hint, + and the env toggle tolerates case/whitespace variants.""" + + def _doctor_checks(self, runner, *extra): + result = runner.invoke(cli_module.main, ["doctor", "--json", *extra]) + _ok(result) + import json as _json + return {c["check"]: c for c in _json.loads(result.output)} + + def _with_fake_st(self, monkeypatch): + fake_st = _fake_module( + __version__="9.9.9", + SentenceTransformer=object, + ) + monkeypatch.setitem(__import__("sys").modules, "sentence_transformers", fake_st) + + def test_deep_failure_hint_is_not_pip_install(self, runner, monkeypatch): + self._with_fake_st(monkeypatch) + fake_embedder = types.SimpleNamespace(model=None, fastembed_model=None) + fake_emb_mod = _fake_module(TextEmbedder=lambda **k: fake_embedder) + monkeypatch.setitem(__import__("sys").modules, "semantica.embeddings", fake_emb_mod) + + checks = self._doctor_checks(runner, "--deep-embeddings") + st = checks["Embeddings (sentence-transformers)"] + assert st["status"] == "fail" + assert "pip install" not in (st["hint"] or ""), ( + "a deep probe failure means the package imported fine — pointing " + "users at pip sends them to reinstall for a runtime/model problem" + ) + assert "runtime/model-load" in st["hint"] + + def test_env_var_tolerates_case_and_whitespace(self, runner, monkeypatch): + monkeypatch.setenv("SEMANTICA_DOCTOR_DEEP_EMBEDDINGS", " TRUE ") + self._with_fake_st(monkeypatch) + fake_embedder = types.SimpleNamespace(model=None, fastembed_model=None) + fake_emb_mod = _fake_module(TextEmbedder=lambda **k: fake_embedder) + monkeypatch.setitem(__import__("sys").modules, "semantica.embeddings", fake_emb_mod) + + checks = self._doctor_checks(runner) + st = checks["Embeddings (sentence-transformers)"] + assert st["status"] == "fail" + assert "hash fallback" in st["note"], "padded/caps env value must enable deep mode" diff --git a/tests/test_embedding_providers.py b/tests/test_embedding_providers.py index e41de0db..a3e02576 100644 --- a/tests/test_embedding_providers.py +++ b/tests/test_embedding_providers.py @@ -85,3 +85,49 @@ if __name__ == '__main__': runner = unittest.TextTestRunner(stream=f, verbosity=2) unittest.main(testRunner=runner, exit=False) + +class TestMethodDispatchRecursion(unittest.TestCase): + """#994: built-in aliases are registered in the method registry onto the + wrapper functions themselves, so dispatching through the registry called a + wrapper back into itself with the same default method — a recursion storm + that surfaced as `maximum recursion depth exceeded` during model loading.""" + + def test_generate_embeddings_default_does_not_self_recurse(self): + from semantica.embeddings.methods import generate_embeddings + emb = generate_embeddings("recursion probe") + self.assertIsNotNone(emb) + + def test_embed_text_default_does_not_self_recurse(self): + from semantica.embeddings.methods import embed_text + emb = embed_text("recursion probe", method="sentence_transformers") + self.assertIsNotNone(emb) + + def test_custom_registered_method_still_wins(self): + from semantica.embeddings.methods import method_registry + calls = [] + + def spy(data, *a, **k): + calls.append(data) + return {"custom": True} + + method_registry.register("generation", "my_custom_gen", spy) + try: + from semantica.embeddings.methods import generate_embeddings + out = generate_embeddings("payload", method="my_custom_gen") + self.assertEqual(out, {"custom": True}) + self.assertEqual(calls, ["payload"]) + finally: + method_registry.unregister("generation", "my_custom_gen") + + def test_provenance_wrapper_missing_generator_raises_attribute_error(self): + # Partially-initialised wrappers (failed __init__, pickle/copy probes) + # must raise AttributeError, not RecursionError via __getattr__. + from semantica.embeddings.embeddings_provenance import ( + EmbeddingGeneratorWithProvenance, + ) + bare = EmbeddingGeneratorWithProvenance.__new__( + EmbeddingGeneratorWithProvenance + ) + with self.assertRaises(AttributeError): + getattr(bare, "model") + From 556e786fd5c7c6cbe52d05cb714692b6725b7dfa Mon Sep 17 00:00:00 2001 From: Sameer6305 Date: Thu, 20 Aug 2026 15:49:46 +0530 Subject: [PATCH 05/20] test: make doctor import failure assertion deterministic --- tests/test_cli_commands.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index f2b626f7..3a6b91fd 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -1876,11 +1876,19 @@ class TestDoctorEmbeddings: assert "Embeddings (sentence-transformers)" in checks assert "Embeddings (fastembed)" in checks - def test_import_failure_is_fail_status_with_hint(self, runner): + def test_import_failure_is_fail_status_with_hint(self, runner, monkeypatch): + # Force the 'import sentence_transformers' inside _embedding_backend to + # raise ImportError regardless of whether the package is installed on + # this machine. Setting a module entry to None is the standard Python + # mechanism: any subsequent 'import ' raises + # "import of halted; None in sys.modules". + monkeypatch.setitem( + __import__("sys").modules, "sentence_transformers", None + ) checks = self._doctor_checks(runner) st = checks["Embeddings (sentence-transformers)"] - if st["status"] == "fail": - assert st["hint"] == "pip install sentence-transformers" + assert st["status"] == "fail" + assert st["hint"] == "pip install sentence-transformers" def test_deep_probe_detects_fallback_active(self, runner, monkeypatch): self._with_fake_st(monkeypatch) From 898a92062a213deaf4e9ce42b90c69e86a2dc0f9 Mon Sep 17 00:00:00 2001 From: Sameer6305 Date: Thu, 20 Aug 2026 15:59:38 +0530 Subject: [PATCH 06/20] chore: restore workflow files --- .github/workflows/benchmark.yml | 51 +++++ .github/workflows/ci.yml | 83 ++++++++ .github/workflows/codeql.yml | 97 +++++++++ .github/workflows/defender-for-devops.yml | 88 ++++++++ .github/workflows/docs.yml | 68 ++++++ .github/workflows/release.yml | 73 +++++++ .github/workflows/security-scan.yml | 239 ++++++++++++++++++++++ .github/workflows/security.yml | 42 ++++ .github/workflows/verify-action-pins.yml | 28 +++ 9 files changed, 769 insertions(+) create mode 100644 .github/workflows/benchmark.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/codeql.yml create mode 100644 .github/workflows/defender-for-devops.yml create mode 100644 .github/workflows/docs.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/security-scan.yml create mode 100644 .github/workflows/security.yml create mode 100644 .github/workflows/verify-action-pins.yml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 00000000..0157ee24 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,51 @@ +name: Semantica Performance Suite + +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + performance-test: + name: Benchmark Runner (Ubuntu/Python 3.12) + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + fetch-depth: 0 + + - name: Set up Python 3.11 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: "3.11" + cache: 'pip' + + - name: Install Dependencies + env: + + BENCHMARK_REAL_LIBS: "1" + run: | + python -m pip install --upgrade pip + pip install -e . + pip install -r benchmarks/requirements.txt + python -m spacy download en_core_web_sm + pip install rdflib neo4j faiss-cpu torch pyarrow pdfplumber python-pptx openpyxl lxml python-docx beautifulsoup4 chardet langdetect + + - name: Execute Benchmarks (Real Mode) + env: + BENCHMARK_REAL_LIBS: "1" + run: | + python benchmarks/benchmarks_runner.py + # Optional: Compare to baseline (requires previous run artifact) + # pytest-benchmark --storage file://benchmarks/results --benchmark-compare + + - name: Upload Benchmark Results + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: benchmark-report-${{ github.run_id }} + path: benchmarks/results + retention-days: 30 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..4ea31ff8 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,83 @@ +name: CI + +permissions: + contents: read + +on: + push: + branches: [main] + paths-ignore: + - 'docs/**' + - 'docs_check.py' + - '**/*.md' + pull_request: + branches: [main] + paths-ignore: + - 'docs/**' + - 'docs_check.py' + - '**/*.md' + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: '3.11' + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: '20' + cache: 'npm' + cache-dependency-path: explorer/package-lock.json + - name: Install Explorer frontend dependencies + working-directory: explorer + run: npm ci + - name: Test Explorer frontend + working-directory: explorer + run: | + npm run test:graph-store + npm run test:graph-workspace + npm run test:plugin-registry + - name: Build Explorer frontend + working-directory: explorer + run: npm run build + - name: Install pinned Python dependencies + run: | + pip install -r requirements-ci.txt + - name: Verify requirements-ci.txt is up to date + run: | + pip install uv==0.12.1 + # Re-resolve with the committed file as a constraint: upstream package + # releases must NOT fail CI (deps only change when pyproject.toml + # changes intentionally). Compare only version lines (pkg==ver), + # ignoring the -c constraint comments and the `\` line continuations + # that --generate-hashes emits. + uv pip compile pyproject.toml --python-version 3.11 --extra all \ + --constraint requirements-ci.txt -o /tmp/requirements-ci-check.txt + diff \ + <(grep -E '^[a-zA-Z0-9._-]+==' requirements-ci.txt | sed 's/ \\$//') \ + <(grep -E '^[a-zA-Z0-9._-]+==' /tmp/requirements-ci-check.txt) + - run: pip install build + # wheel is build-time only (not in requirements-ci.txt) — install the + # same pinned version [build-system] declares so --no-isolation works. + - run: pip install wheel==0.48.0 + - name: Build package (no isolation — pinned deps) + run: python -m build --no-isolation + - name: Verify Explorer frontend is packaged + run: | + python - <<'PY' + import zipfile + from pathlib import Path + + wheels = list(Path("dist").glob("*.whl")) + assert wheels, "No wheel was built" + + with zipfile.ZipFile(wheels[0]) as wheel: + names = set(wheel.namelist()) + + assert "semantica/static/index.html" in names, "Explorer index.html missing from wheel" + assert any(name.startswith("semantica/static/assets/") for name in names), "Explorer assets missing from wheel" + + print("Explorer frontend is packaged") + PY diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000..0c15e84c --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,97 @@ +name: CodeQL + +on: + push: + branches: [main] + pull_request: + branches: [main] + schedule: + - cron: '30 1 * * 1' # Every Monday 7 AM IST + +permissions: + contents: read + security-events: write + actions: read + +jobs: + analyze: + name: Analyze Python + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + # The CodeQL bundle download (github/codeql-action/init's "Setup CodeQL + # tools" step) streams a ~1GB tarball from GitHub's release CDN and + # does not retry on a transient connection reset (ECONNRESET) itself + # (github/codeql-action, unresolved as of v4 / CLI 2.26.1: the HTTP + # error is retryable but isn't retried internally). Since a `uses:` + # step can't be wrapped by a shell-level retry action, attempt init + # up to 3 times; each retry is a fresh download attempt with no + # meaningful state carried over from a failed attempt. + - name: Initialize CodeQL (attempt 1) + id: codeql-init-1 + uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 + continue-on-error: true + with: + languages: python + queries: security-and-quality + config-file: .github/codeql/codeql-config.yml + + - name: Initialize CodeQL (attempt 2) + id: codeql-init-2 + if: steps.codeql-init-1.outcome == 'failure' + uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 + continue-on-error: true + with: + languages: python + queries: security-and-quality + config-file: .github/codeql/codeql-config.yml + + - name: Initialize CodeQL (attempt 3) + id: codeql-init-3 + if: steps.codeql-init-2.outcome == 'failure' + uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 + with: + languages: python + queries: security-and-quality + config-file: .github/codeql/codeql-config.yml + + - name: Autobuild + uses: github/codeql-action/autobuild@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 + with: + category: "/language:python" + upload: false + id: codeql + + - name: Upload SARIF (Advanced Setup only) + # Uploads results only when Default Setup is not active. + # If Default Setup is still enabled, this step skips gracefully + # instead of failing the workflow with HTTP 409. + uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 + with: + sarif_file: ${{ steps.codeql.outputs.sarif-output }} + category: "/language:python" + wait-for-processing: true + continue-on-error: true + + # NOTE: Auto-dismissal by rule-id is intentionally removed. + # Dismissing every alert that matches a rule ID would silently suppress + # future real vulnerabilities of the same type. The alerts below were + # individually triaged and dismissed manually in the security-enhancement + # PR (alerts #12–#18). New alerts must be reviewed and dismissed by hand, + # or will auto-close when the underlying code no longer triggers them. + # + # If you need to dismiss a specific known-safe alert, pin its alert NUMBER + # here and remove it once CodeQL stops reporting it naturally. Example: + # + # PINNED_ALERT_NUMBERS=(12 13 14 15 16 17 18) + # for NUM in "${PINNED_ALERT_NUMBERS[@]}"; do + # gh api repos/$REPO/code-scanning/alerts/$NUM \ + # -X PATCH -f state=dismissed -f dismissed_reason="false positive" \ + # -f dismissed_comment="" + # done diff --git a/.github/workflows/defender-for-devops.yml b/.github/workflows/defender-for-devops.yml new file mode 100644 index 00000000..becb7d64 --- /dev/null +++ b/.github/workflows/defender-for-devops.yml @@ -0,0 +1,88 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. +# +# Microsoft Security DevOps (MSDO) is a command line application which integrates static analysis tools into the development cycle. +# MSDO installs, configures and runs the latest versions of static analysis tools +# (including, but not limited to, SDL/security and compliance tools). +# +# The Microsoft Security DevOps action is currently in beta and runs on the windows-latest queue, +# as well as Windows self hosted agents. ubuntu-latest support coming soon. +# +# For more information about the action , check out https://github.com/microsoft/security-devops-action +# +# Please note this workflow do not integrate your GitHub Org with Microsoft Defender For DevOps. You have to create an integration +# and provide permission before this can report data back to azure. +# Read the official documentation here : https://learn.microsoft.com/en-us/azure/defender-for-cloud/quickstart-onboard-github + +name: "Microsoft Defender For Devops" + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + schedule: + - cron: '43 17 * * 6' + +permissions: + contents: read + security-events: write + +jobs: + MSDO: + # currently only windows-latest is supported + runs-on: windows-latest + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6 + with: + dotnet-version: | + 5.0.x + 6.0.x + - name: Run Microsoft Security DevOps + uses: microsoft/security-devops-action@08976cb623803b1b36d7112d4ff9f59eae704de0 # v1.12.0 + id: msdo + with: + # checkov is intentionally excluded from this MSDO step. + # MSDO 0.215.0's guardian.cmd wrapper treats checkov's exit code 1 + # (emitted whenever any violation is found, even below the active severity + # threshold) as a fatal "tool error" and breaks the build even when + # "Active results: 0" and "Found no breaking results." The .checkov.yaml + # soft-fail setting is never read by the guardian wrapper. + # IaC security scanning continues below in this same MSDO job identity. + # That preserves the existing GitHub code-scanning configuration while + # avoiding the guardian.cmd/checkov exit-code bug in the MSDO wrapper. + tools: eslint,templateanalyzer,terrascan + - name: Upload results to Security tab + uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 + with: + sarif_file: ${{ steps.msdo.outputs.sarifFile }} + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: "3.12" + + - name: Install Checkov + run: python -m pip install checkov==3.3.1 + + - name: Run Checkov + shell: pwsh + env: + PYTHONUTF8: "1" + run: | + New-Item -ItemType Directory -Force reports | Out-Null + checkov --directory . --framework kubernetes helm dockerfile github_actions secrets bicep arm --soft-fail --output sarif --output-file-path reports/checkov.sarif + if (-not (Test-Path reports/checkov.sarif)) { + $sarif = Get-ChildItem -Path reports -Recurse -Filter *.sarif | Select-Object -First 1 + if ($null -eq $sarif) { throw "Checkov did not produce a SARIF file" } + Copy-Item $sarif.FullName reports/checkov.sarif + } + + - name: Upload Checkov results to Security tab + uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4 + if: always() + with: + sarif_file: reports/checkov.sarif diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..ca149d7e --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,68 @@ +name: Build and Deploy Documentation + +on: + push: + branches: [main] + paths: + - 'docs/**' + - 'docs_check.py' + - 'CHANGELOG.md' + - 'RELEASE.md' + pull_request: + branches: [main] + paths: + - 'docs/**' + - 'docs_check.py' + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + validate: + name: Validate Documentation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: '3.11' + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: '20' + - run: python docs_check.py + + deploy: + name: Build and Deploy to GitHub Pages + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + needs: validate + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: '20' + + - name: Export static site + run: | + cd docs + npx mintlify export --output ../export.zip + cd .. + unzip -q export.zip -d site + + - uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6 + + - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5 + with: + path: ./site + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..bbc8770c --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,73 @@ +name: Release + +on: + push: + tags: ['v*'] + +permissions: + contents: read + +jobs: + release: + runs-on: ubuntu-latest + environment: pypi + concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false + permissions: + contents: write # for the GitHub Release + id-token: write # for PyPI Trusted Publishing (OIDC) and attestation signing + attestations: write # for SLSA build provenance + # If you add another job to this workflow, give it its own explicit + # `permissions:` block rather than relying on the workflow-level default + # above (contents: read) - do not widen the workflow-level default. + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: '3.11' + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: '20' + cache: 'npm' + cache-dependency-path: explorer/package-lock.json + - name: Build Explorer frontend + working-directory: explorer + run: | + npm ci + npm run build + # Install the pinned dependency set (with hashes) so the sdist/wheel + # build runs against the same versions CI tests against. + - name: Install pinned build dependencies + run: pip install -r requirements-ci.txt + - run: pip install build + # wheel is build-time only (not in requirements-ci.txt) — install the + # same pinned version [build-system] declares so --no-isolation works. + - run: pip install wheel==0.48.0 + - name: Build package (no isolation — pinned deps) + run: python -m build --no-isolation + - name: Verify Explorer frontend is packaged + run: | + python - <<'PY' + import zipfile + from pathlib import Path + + wheels = list(Path("dist").glob("*.whl")) + assert wheels, "No wheel was built" + + with zipfile.ZipFile(wheels[0]) as wheel: + names = set(wheel.namelist()) + + assert "semantica/static/index.html" in names, "Explorer index.html missing from wheel" + assert any(name.startswith("semantica/static/assets/") for name in names), "Explorer assets missing from wheel" + + print("Explorer frontend is packaged") + PY + - name: Attest build provenance + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4 + with: + subject-path: 'dist/*' + - uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3 + with: + files: dist/* + - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1 diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml new file mode 100644 index 00000000..5b4461af --- /dev/null +++ b/.github/workflows/security-scan.yml @@ -0,0 +1,239 @@ +name: Security Scan + +on: + schedule: + - cron: '30 1 * * 1,4' # Mon/Thu 7 AM IST + push: + branches: [main] + paths-ignore: + - 'docs/**' + - 'mkdocs.yml' + - 'requirements-docs.txt' + - '**/*.md' + pull_request: + branches: [main] + paths-ignore: + - 'docs/**' + - 'mkdocs.yml' + - 'requirements-docs.txt' + - '**/*.md' + +permissions: + contents: read + +jobs: + security-scan: + runs-on: ubuntu-latest + permissions: + contents: read + security-events: write + actions: read + # Needed for the "Comment PR with Security Results" step below. Safe on + # pull_request (not pull_request_target): GitHub always forces a + # read-only token for PRs from forks regardless of this permission. + pull-requests: write + + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + # Install the pinned dependency set FIRST so Safety scans Semantica's + # exact CI/release dependency tree (requirements-ci.txt is generated + # from pyproject.toml extras, so this covers the project's real deps). + pip install -r requirements-ci.txt + # Tooling AFTER the pinned set: installing safety/bandit/semgrep/jq + # first lets the pinned requirements overwrite their transitive deps + # (e.g. rich), which breaks the safety CLI at runtime. + pip install safety bandit semgrep jq + + - name: Run Safety Check (Package Vulnerabilities) + run: | + # NOTE: Safety 3.x repurposed --output to select a console format + # (json/text/screen/...), not a file path. Writing JSON to a file + # now requires --save-json; the previous `--output safety-report.json` + # usage was silently invalid and never produced a report. + safety check --save-json safety-report.json || true + + # Guard 1: fail loudly if Safety exited before writing a report at all + # (network error, API auth failure, tool crash). Without this check a + # missing or empty file causes jq to fall back to "0", making a broken + # scanner indistinguishable from a clean scan. + if [ ! -s safety-report.json ]; then + echo "::error::Safety scan produced no report (safety-report.json is missing or empty). Treating as failure — check for network errors, API auth failures, or Safety crashes in the logs above." + exit 1 + fi + + echo "Checking for package vulnerabilities..." + + # No || echo "0" fallback: if jq fails (malformed JSON, missing key, + # vulnerabilities:null) VULNS will be empty or "null" so guard 2 below + # catches it rather than silently treating the broken report as zero. + VULNS=$(jq '.vulnerabilities | length' safety-report.json 2>/dev/null) + + # Guard 2: ensure VULNS is a non-negative integer before the -gt + # comparison. "null" (missing/null key) or "" (jq parse failure) would + # cause bash's -gt to throw an arithmetic error and fall through to the + # success branch — the same silent-pass bug as a missing file. + if ! [[ "$VULNS" =~ ^[0-9]+$ ]]; then + echo "::error::Safety report exists but 'vulnerabilities' is missing or non-numeric (got: '${VULNS}'). The report may be malformed or Safety may have written an error-only JSON. Treating as failure." + exit 1 + fi + + if [ "$VULNS" -gt 0 ]; then + echo "❌ Security vulnerabilities found: $VULNS" + echo "CI will fail to prevent merging of vulnerable dependencies" + echo "" + echo "Vulnerability details:" + jq -r '.vulnerabilities[] | "- \(.package_name)==\(.analyzed_version): \(.vulnerability_id) (\(.CVE // "no CVE assigned"))"' safety-report.json || true + exit 1 + else + echo "✅ No security vulnerabilities found" + fi + + - name: Run Bandit (Code Security Linter) + run: | + bandit -r semantica/ -f json -o bandit-report.json || true + echo "Checking for HIGH severity security issues..." + + # Count HIGH severity issues + HIGH_ISSUES=$(bandit -r semantica/ -f json -ll 2>/dev/null | jq -r '.results[]? | select(.issue_severity == "HIGH") | .test_name' 2>/dev/null | wc -l || echo "0") + + if [ "$HIGH_ISSUES" -gt 0 ]; then + echo "❌ HIGH severity security issues found: $HIGH_ISSUES" + echo "CI will fail to prevent merging of high-risk code" + echo "" + echo "High severity issues:" + bandit -r semantica/ -ll | grep "Severity: High" -A 5 -B 1 || true + exit 1 + else + echo "✅ No HIGH severity security issues found" + fi + + - name: Run Semgrep (Static Analysis) + run: | + echo "Running Semgrep static analysis..." + semgrep --config=auto --json --output=semgrep-report.json semantica/ || true + + # Run security-focused rules + echo "Checking for security patterns..." + SECURITY_ISSUES=$(semgrep --config=p/security --json semantica/ 2>/dev/null | jq '.results | length' 2>/dev/null || echo "0") + + if [ "$SECURITY_ISSUES" -gt 0 ]; then + echo "⚠️ Security patterns found: $SECURITY_ISSUES" + echo "Review these findings for potential improvements" + semgrep --config=p/security semantica/ || true + else + echo "✅ No security patterns found" + fi + + - name: Upload Security Reports + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: security-reports + retention-days: 14 + path: | + safety-report.json + bandit-report.json + semgrep-report.json + + - name: Comment PR with Security Results + if: github.event_name == 'pull_request' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 + with: + script: | + const fs = require('fs'); + + // Renders one tool's findings as a section. `items` is already + // the list of pre-formatted "- `thing` in `where`" strings; this + // just handles the found/not-found/report-missing framing and + // collapses long lists into a
block so the comment + // doesn't turn into a wall of text. + function renderSection(title, reportPath, parse) { + let data; + try { + data = JSON.parse(fs.readFileSync(reportPath, 'utf8')); + } catch (e) { + return [ + `### ${title}`, + `⚠️ No report found at \`${reportPath}\` — the scan may have failed before producing output. Check the job logs.`, + ].join('\n'); + } + + const items = parse(data); + if (items.length === 0) { + return [`### ${title}`, `✅ No findings.`].join('\n'); + } + + const lines = [`### ${title}`, `Found **${items.length}**.`, '']; + const shown = items.slice(0, 15); + if (items.length > 15) { + lines.push('
', 'Show all findings', ''); + lines.push(...items); + lines.push('', '
'); + } else { + lines.push(...shown); + } + return lines.join('\n'); + } + + const safetySection = renderSection( + 'Safety — dependency vulnerabilities', + 'safety-report.json', + (data) => (data.vulnerabilities || []).map( + (v) => `- \`${v.package_name}==${v.analyzed_version}\`: ${v.vulnerability_id}` + + (v.CVE ? ` (${v.CVE})` : '') + ` — ${v.advisory || 'no advisory text'}` + ) + ); + + const banditSection = renderSection( + 'Bandit — HIGH-severity code issues', + 'bandit-report.json', + (data) => (data.results || []) + .filter((issue) => issue.issue_severity === 'HIGH') + .map((issue) => `- \`${issue.test_name}\` in \`${issue.filename}:${issue.line_number}\``) + ); + + const semgrepSection = renderSection( + 'Semgrep — static analysis patterns', + 'semgrep-report.json', + (data) => (data.results || []).map( + (issue) => `- \`${issue.check_id}\` in \`${issue.path}:${issue.start?.line ?? '?'}\`` + ) + ); + + const comment = [ + '# 🔒 Security Scan Results', + '', + safetySection, + '', + banditSection, + '', + semgrepSection, + '', + '---', + '', + '*This security scan runs automatically on source-code PRs and bi-weekly (skipped for doc/markdown-only changes).*', + '', + '📊 **Security Policy**: CI fails on Safety vulnerabilities and Bandit HIGH-severity findings. Semgrep findings above are informational and do not block merge.', + ].join('\n'); + + try { + await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: comment, + }); + console.log('✅ Security comment posted successfully'); + } catch (error) { + console.log('⚠️ Could not post security comment:', error.message); + console.log('📋 Security scan results saved to artifacts'); + } diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 00000000..412e7eaa --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,42 @@ +name: Security + +on: + schedule: + - cron: '0 0 * * 1' + workflow_dispatch: + pull_request: + branches: [main] + paths: + - 'pyproject.toml' + - 'requirements-ci.txt' + - '.github/workflows/security.yml' + +permissions: + contents: read + +jobs: + audit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: '3.11' + # Upgrade first: actions/setup-python's baked-in setuptools has been + # behind known-vulnerable floors before (e.g. PYSEC-2026-3447 / + # setuptools 75.1.0), so don't trust the preinstalled one. + - run: python -m pip install --upgrade pip setuptools + # Audit the pinned dependency set (requirements-ci.txt is compiled from + # pyproject.toml with --extra all — the same coverage as the [all] + # extra, minus the Linux-only gpu set — so this keeps scan parity with + # CI/release builds without a time-dependent resolution). This is the + # fix for PYSEC-2024-38 (#869): the bare-env job never had fastapi or + # python-multipart installed to look at. + - run: pip install -r requirements-ci.txt + # PR runs gate on findings, since they're scoped to actual + # pyproject.toml changes under review. The schedule/workflow_dispatch + # runs stay non-blocking until a full pass over pre-existing findings + # across the whole [all] tree has been done. + - run: pip install pip-audit + - run: pip-audit -r requirements-ci.txt + continue-on-error: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/verify-action-pins.yml b/.github/workflows/verify-action-pins.yml new file mode 100644 index 00000000..6e7dada9 --- /dev/null +++ b/.github/workflows/verify-action-pins.yml @@ -0,0 +1,28 @@ +name: Verify Action Pins + +on: + pull_request: + paths: + - '.github/workflows/**' + - '.github/scripts/verify-action-pins.sh' + push: + branches: [main] + paths: + - '.github/workflows/**' + - '.github/scripts/verify-action-pins.sh' + schedule: + - cron: '0 3 * * 1' # weekly, in case an upstream tag is deliberately moved + workflow_dispatch: + +permissions: + contents: read + +jobs: + verify: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - name: Verify pinned action SHAs match their tag comments + env: + GH_TOKEN: ${{ github.token }} + run: bash .github/scripts/verify-action-pins.sh From 283b7ada0c4bbae007f7557c194a64f8602d3ce1 Mon Sep 17 00:00:00 2001 From: Aldrin Joseph Date: Sat, 22 Aug 2026 16:22:38 +0530 Subject: [PATCH 07/20] fix(context): accept analyzer vocabulary in causal edges (#1184) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get_causal_chain() matched only the canonical uppercase spellings (CAUSED, INFLUENCED, PRECEDENT_FOR), while CausalChainAnalyzer's vocabulary includes the present-tense forms (causes, influences, leads_to, supports) — and the two differ in word form, not just case, so case-insensitive matching alone would still miss them. An edge recorded as "causes" produced an empty audit chain. Storage normalizes both vocabularies onto the canonical types via _CAUSAL_EDGE_ALIASES; traversal accepts the union (_CAUSAL_TRAVERSAL_TYPES). add_causal_relationship() now accepts either spelling and stores the canonical form. --- semantica/context/context_graph.py | 30 ++++++++++-- .../test_decision_causal_edge_regression.py | 48 +++++++++++++++++++ 2 files changed, 73 insertions(+), 5 deletions(-) diff --git a/semantica/context/context_graph.py b/semantica/context/context_graph.py index ac3f134f..6a095c69 100644 --- a/semantica/context/context_graph.py +++ b/semantica/context/context_graph.py @@ -438,6 +438,23 @@ _ATTRS_MISSING = object() #: entities and timestamps. _CAUSAL_EDGE_TYPES = ("CAUSED", "INFLUENCED", "PRECEDENT_FOR") +# Causal edges circulate under two vocabularies: this module's canonical +# spellings above, and the present-tense spellings CausalChainAnalyzer also +# accepts ("causes", "influences", "leads_to", "supports"). The present-tense +# forms normalize onto the canonical types for storage; traversal accepts +# both vocabularies so an edge recorded either way is never invisible. +_CAUSAL_EDGE_ALIASES = { + "CAUSES": "CAUSED", + "CAUSED": "CAUSED", + "INFLUENCES": "INFLUENCED", + "INFLUENCED": "INFLUENCED", + "PRECEDES": "PRECEDENT_FOR", + "PRECEDENT_FOR": "PRECEDENT_FOR", +} +_CAUSAL_TRAVERSAL_TYPES = frozenset(_CAUSAL_EDGE_ALIASES) | { + "LEADS_TO", "LEAD_TO", "SUPPORTS", "SUPPORT", +} + class ContextGraph: """ @@ -2745,9 +2762,12 @@ class ContextGraph: target_decision_id: Target decision ID relationship_type: Type of relationship (CAUSED, INFLUENCED, PRECEDENT_FOR) """ - valid_types = ["CAUSED", "INFLUENCED", "PRECEDENT_FOR"] - if relationship_type not in valid_types: - raise ValueError(f"Relationship type must be one of: {valid_types}") + # Normalize so callers may use either vocabulary's spelling + # ("causes" from CausalChainAnalyzer, or "CAUSED" from this module's + # canonical constant); the stored form is always canonical. + relationship_type = _CAUSAL_EDGE_ALIASES.get(relationship_type.upper()) + if relationship_type is None: + raise ValueError(f"Relationship type must be one of: {_CAUSAL_EDGE_TYPES}") # Check if decisions exist - if not, skip adding relationship if source_decision_id not in self.nodes or target_decision_id not in self.nodes: @@ -2839,11 +2859,11 @@ class ContextGraph: # Find connected decisions for edge in self.edges: if direction == "upstream": - if edge.target_id == current_id and edge.edge_type in ["CAUSED", "INFLUENCED", "PRECEDENT_FOR"]: + if edge.target_id == current_id and edge.edge_type.upper() in _CAUSAL_TRAVERSAL_TYPES: if edge.source_id not in visited and depth < max_depth: queue.append((edge.source_id, depth + 1)) else: # downstream - if edge.source_id == current_id and edge.edge_type in ["CAUSED", "INFLUENCED", "PRECEDENT_FOR"]: + if edge.source_id == current_id and edge.edge_type.upper() in _CAUSAL_TRAVERSAL_TYPES: if edge.target_id not in visited and depth < max_depth: queue.append((edge.target_id, depth + 1)) diff --git a/tests/context/test_decision_causal_edge_regression.py b/tests/context/test_decision_causal_edge_regression.py index 91bade6d..ed411251 100644 --- a/tests/context/test_decision_causal_edge_regression.py +++ b/tests/context/test_decision_causal_edge_regression.py @@ -323,3 +323,51 @@ def test_entity_based_inference_still_applies_without_explicit_edges(): hop["from"] == earlier and hop["to"] == later and hop["type"] == "influences" for hop in hops ) + + +def test_get_causal_chain_accepts_lowercase_causal_edge_types(): + """Issue #1184: edges recorded with the analyzer's lowercase vocabulary + must be traversed by get_causal_chain(). + + CausalChainAnalyzer documents causal types as lowercase ("causes", + "influences", ...) while get_causal_chain() matched only the uppercase + spellings, so an edge recorded as "causes" produced an empty audit + chain — silent and in the dangerous direction. + """ + graph = ContextGraph(advanced_analytics=True) + cause = graph.record_decision( + category="a", scenario="upstream", reasoning="r", + outcome="x", confidence=0.9, + ) + effect = graph.record_decision( + category="b", scenario="downstream", reasoning="r", + outcome="y", confidence=0.9, + ) + graph.add_edge(cause, effect, "causes") + + chain = graph.get_causal_chain(effect, direction="upstream") + + assert [decision.decision_id for decision in chain] == [cause] + + +def test_add_causal_relationship_accepts_any_case_and_stores_canonical(): + """Issue #1184: add_causal_relationship() should accept either spelling + and store the canonical uppercase vocabulary.""" + graph = ContextGraph(advanced_analytics=True) + cause = graph.record_decision( + category="a", scenario="upstream", reasoning="r", + outcome="x", confidence=0.9, + ) + effect = graph.record_decision( + category="b", scenario="downstream", reasoning="r", + outcome="y", confidence=0.9, + ) + + graph.add_causal_relationship(cause, effect, relationship_type="causes") + + edges = [ + edge for edge in graph.edges + if edge.source_id == cause and edge.target_id == effect + ] + assert edges, "add_causal_relationship must store the edge" + assert edges[0].edge_type == "CAUSED" From 2d976963ab5630e8784b77cb935d62a5bdd42a36 Mon Sep 17 00:00:00 2001 From: Aldrin Joseph Date: Sat, 22 Aug 2026 16:29:44 +0530 Subject: [PATCH 08/20] fix(context): keep ValueError for non-string causal relationship types (#1184) Review feedback: normalization must not turn invalid inputs into AttributeError. Non-string relationship types now raise ValueError before normalization, matching the pre-change behavior; strings are stripped before alias lookup. --- semantica/context/context_graph.py | 7 +++++-- .../test_decision_causal_edge_regression.py | 20 +++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/semantica/context/context_graph.py b/semantica/context/context_graph.py index 6a095c69..d18c2b54 100644 --- a/semantica/context/context_graph.py +++ b/semantica/context/context_graph.py @@ -2764,8 +2764,11 @@ class ContextGraph: """ # Normalize so callers may use either vocabulary's spelling # ("causes" from CausalChainAnalyzer, or "CAUSED" from this module's - # canonical constant); the stored form is always canonical. - relationship_type = _CAUSAL_EDGE_ALIASES.get(relationship_type.upper()) + # canonical constant); the stored form is always canonical. Invalid + # inputs keep raising ValueError rather than AttributeError. + if not isinstance(relationship_type, str): + raise ValueError(f"Relationship type must be one of: {_CAUSAL_EDGE_TYPES}") + relationship_type = _CAUSAL_EDGE_ALIASES.get(relationship_type.strip().upper()) if relationship_type is None: raise ValueError(f"Relationship type must be one of: {_CAUSAL_EDGE_TYPES}") diff --git a/tests/context/test_decision_causal_edge_regression.py b/tests/context/test_decision_causal_edge_regression.py index ed411251..211864d0 100644 --- a/tests/context/test_decision_causal_edge_regression.py +++ b/tests/context/test_decision_causal_edge_regression.py @@ -7,6 +7,8 @@ extraction found nothing, the chain came back empty even though an explicit ``CAUSED`` edge was stored in the graph. """ +import pytest + from semantica.context import ContextGraph from semantica.context.context_graph import ContextEdge @@ -371,3 +373,21 @@ def test_add_causal_relationship_accepts_any_case_and_stores_canonical(): ] assert edges, "add_causal_relationship must store the edge" assert edges[0].edge_type == "CAUSED" + + +def test_add_causal_relationship_rejects_non_string_with_value_error(): + """Invalid relationship types must keep raising ValueError (issue #1184 + follow-up): normalization must not turn them into AttributeError.""" + graph = ContextGraph(advanced_analytics=True) + cause = graph.record_decision( + category="a", scenario="upstream", reasoning="r", + outcome="x", confidence=0.9, + ) + effect = graph.record_decision( + category="b", scenario="downstream", reasoning="r", + outcome="y", confidence=0.9, + ) + + for bad_type in (None, 42, ["CAUSED"]): + with pytest.raises(ValueError): + graph.add_causal_relationship(cause, effect, relationship_type=bad_type) From 3c99f447e6e55ba074771a30d0ecfff6e5b9dc9e Mon Sep 17 00:00:00 2001 From: Aldrin Joseph Date: Sat, 22 Aug 2026 19:42:56 +0530 Subject: [PATCH 09/20] docs(shacl): document that rdfs:range + RDFS entailment makes sh:class unfalsifiable (#1182) * docs(shacl): warn that rdfs:range makes sh:class unfalsifiable under entailment (#1130) * docs(shacl): self-contained pitfall example, sh:node coverage, and wrapper clarifications (#1130) --- docs/guides/shacl-validation.md | 39 +++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/docs/guides/shacl-validation.md b/docs/guides/shacl-validation.md index a2b81ac2..ad797821 100644 --- a/docs/guides/shacl-validation.md +++ b/docs/guides/shacl-validation.md @@ -381,6 +381,45 @@ print(f"Violations after remediation: {report2.violation_count}") - **Forgetting RDF serialization**: You must serialize your graph (often via a temporary file using `export_rdf`) before validating it. - **Treating validation as a one-time step**: Validation should be integrated as an automated step in your CI/CD pipeline or data ingestion flow, acting as a recurring gatekeeper rather than a one-off script. - **Ignoring validation reports**: A graph that does not conform must be remediated. Failing to review the `violation_count` and address the issues negates the purpose of SHACL validation. +- **Validating `sh:class`/`sh:node` range checks on a property that declares `rdfs:range` with RDFS entailment on**: RDFS is an entailment rule, not a constraint. When pyshacl runs with `inference="rdfs"`, it infers the range class onto every object of the property, so class-based constraints on that property can never fail — the report says `conforms: True` on data that does not conform: + + ```python + from pyshacl import validate + from rdflib import Graph + + data = Graph() + data.parse( + data=""" + @prefix ex: . + @prefix rdfs: . + ex:contains rdfs:domain ex:Container ; rdfs:range ex:Item . + ex:box a ex:Container ; ex:contains ex:notAnItem . + ex:notAnItem a ex:Fish . + """, + format="turtle", + ) + + shapes = Graph() + shapes.parse( + data=""" + @prefix ex: . + @prefix sh: . + ex:ContainerShape a sh:NodeShape ; + sh:targetClass ex:Container ; + sh:property [ sh:path ex:contains ; sh:class ex:Item ] . + """, + format="turtle", + ) + + for inference in ("none", "rdfs"): + conforms, _, _ = validate(data, shacl_graph=shapes, inference=inference) + print(inference, conforms) + # none False <- correct: notAnItem is a Fish, not an Item + # rdfs True <- the entailment manufactured the type + ``` + + Mitigations: prefer not to declare `rdfs:range` on properties you intend to constrain with `sh:class`; when class membership is the thing under test, run validation without RDFS entailment (`inference="none"`); or express the check as a constraint the entailment cannot satisfy (for example a literal property constraint). Note the trade-off: with entailment off, `sh:targetClass` no longer reaches subclasses, so subclass hierarchies need explicit typing or inference-aware target selection. Semantica's own `_run_pyshacl` wrapper already calls pyshacl with `inference="none"`, so this pitfall only bites when calling `pyshacl.validate` directly with entailment enabled. +- **Trusting `conforms: True` without checking the inference mode**: an inference-enabled run can hide the exact violations the shapes were written to catch (see above). Record which inference mode validation ran under alongside the result, and re-run shape sets that contain `sh:class`/`sh:node` with entailment off before treating a pass as authoritative. --- From d3f37f798e0ecd03a9c360d87c744a8481ad9612 Mon Sep 17 00:00:00 2001 From: Shahzaib Ahmad Date: Sat, 22 Aug 2026 20:56:15 +0500 Subject: [PATCH 10/20] Fix HuggingFace NER kwargs handling (#1188) Co-authored-by: Shahzaib Ahmad Co-authored-by: Zohaib Hassnain <109234410+ZohaibHassan16@users.noreply.github.com> --- semantica/semantic_extract/methods.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/semantica/semantic_extract/methods.py b/semantica/semantic_extract/methods.py index f0559dec..e7ac62b2 100644 --- a/semantica/semantic_extract/methods.py +++ b/semantica/semantic_extract/methods.py @@ -779,9 +779,11 @@ def extract_entities_huggingface( """ loader = HuggingFaceModelLoader(device=device) # Pass kwargs (like aggregation_strategy) to load_ner_model - model_obj = loader.load_ner_model(model, **kwargs) + loader_kwargs = { + key: value for key, value in kwargs.items() if key != "huggingface_model" + } + model_obj = loader.load_ner_model(model, **loader_kwargs) results = loader.extract_entities(model_obj, text) - entities = [] # Check if manual aggregation is needed (raw IOB tags detected) From 50f2f82b95f48a0c66d8be6b365493cd4a288335 Mon Sep 17 00:00:00 2001 From: mikemikimike <13286568797@163.com> Date: Sat, 22 Aug 2026 23:58:00 +0800 Subject: [PATCH 11/20] feat(ontology): expose public SHACL validation API --- docs/guides/shacl-validation.md | 43 ++++++++++++------------ semantica/ontology/__init__.py | 2 ++ semantica/ontology/ontology_validator.py | 19 +++++++++-- tests/ontology/test_ontology_advanced.py | 24 +++++++++++++ 4 files changed, 64 insertions(+), 24 deletions(-) diff --git a/docs/guides/shacl-validation.md b/docs/guides/shacl-validation.md index ad797821..eafea841 100644 --- a/docs/guides/shacl-validation.md +++ b/docs/guides/shacl-validation.md @@ -8,7 +8,7 @@ icon: "shield-check" SHACL (Shapes Constraint Language) is a standard for validating graph-based data. While an ontology defines the conceptual *schema* (the "what" exists in your domain), SHACL defines the structural *rules and constraints* (the "how" it should be structured). -In Semantica, `SHACLGenerator` produces constraint rules (shapes) based on your ontology, and `_run_pyshacl` evaluates your actual data against these rules. If a node violates a rule (e.g., missing a required property or using the wrong datatype), a detailed violation report is generated. +In Semantica, `SHACLGenerator` produces constraint rules (shapes) based on your ontology, and the public `run_shacl_validation` function evaluates your actual data against these rules. If a node violates a rule (e.g., missing a required property or using the wrong datatype), a detailed violation report is generated. The historical `run_shacl_validation` name remains available as a compatibility alias. ## Why Use SHACL Validation? @@ -55,7 +55,7 @@ Let's look at a simple, universally understood example: ensuring every `Employee ```python from semantica.context import ContextGraph from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape -from semantica.ontology.ontology_validator import _run_pyshacl +from semantica.ontology import run_shacl_validation # 1. Prepare your data graph graph = ContextGraph() @@ -95,7 +95,7 @@ data_ttl = """ """ # 5. Run Validation -report = _run_pyshacl(data_ttl, shacl_ttl) +report = run_shacl_validation(data_ttl, shacl_ttl) # 6. Analyze the Report print(f"Graph conforms: {report.conforms}") @@ -265,10 +265,10 @@ cve_id_shape = NodeShape( ## Step 4 — Run validation and read the report -Serialize the graph to RDF, then run `_run_pyshacl` against the shapes. +Serialize the graph to RDF, then run `run_shacl_validation` against the shapes. ```python -from semantica.ontology.ontology_validator import _run_pyshacl +from semantica.ontology import run_shacl_validation # Prepare your RDF data string (since export_rdf primarily exports structural metadata, # you typically serialize your custom data graph to Turtle using rdflib or similar). @@ -281,7 +281,7 @@ data_ttl = """ """ # Run SHACL validation -report = _run_pyshacl( +report = run_shacl_validation( data_ttl, shacl_ttl, data_graph_format="turtle", @@ -366,8 +366,8 @@ print(f"Malware nodes missing 'family': {len(missing_family)}") # e.g. graph.update_node(node_id, {"family": "UNKNOWN — requires triage"}) # After remediation, re-run validation to confirm the fix -# (re-export the patched graph to Turtle first, then call _run_pyshacl again) -report2 = _run_pyshacl(patched_data_ttl, shacl_ttl) +# (re-export the patched graph to Turtle first, then call run_shacl_validation again) +report2 = run_shacl_validation(patched_data_ttl, shacl_ttl) print(f"Violations after remediation: {report2.violation_count}") # Violations after remediation: 0 ``` @@ -377,7 +377,7 @@ print(f"Violations after remediation: {report2.violation_count}") ## Common Pitfalls - **Assuming the ontology automatically enforces data quality**: `SHACLGenerator` generates shapes based on what it observes in the data. If your data is missing a field, the generator won't know it was mandatory unless you explicitly inject the constraint (as shown in Step 3). -- **Passing `ContextGraph` directly to SHACL validators**: The `_run_pyshacl` function expects an RDF string (like Turtle format), not a raw Python dictionary or `ContextGraph` object. +- **Passing `ContextGraph` directly to SHACL validators**: The `run_shacl_validation` function expects an RDF string (like Turtle format), not a raw Python dictionary or `ContextGraph` object. - **Forgetting RDF serialization**: You must serialize your graph (often via a temporary file using `export_rdf`) before validating it. - **Treating validation as a one-time step**: Validation should be integrated as an automated step in your CI/CD pipeline or data ingestion flow, acting as a recurring gatekeeper rather than a one-off script. - **Ignoring validation reports**: A graph that does not conform must be remediated. Failing to review the `violation_count` and address the issues negates the purpose of SHACL validation. @@ -418,7 +418,7 @@ print(f"Violations after remediation: {report2.violation_count}") # rdfs True <- the entailment manufactured the type ``` - Mitigations: prefer not to declare `rdfs:range` on properties you intend to constrain with `sh:class`; when class membership is the thing under test, run validation without RDFS entailment (`inference="none"`); or express the check as a constraint the entailment cannot satisfy (for example a literal property constraint). Note the trade-off: with entailment off, `sh:targetClass` no longer reaches subclasses, so subclass hierarchies need explicit typing or inference-aware target selection. Semantica's own `_run_pyshacl` wrapper already calls pyshacl with `inference="none"`, so this pitfall only bites when calling `pyshacl.validate` directly with entailment enabled. + Mitigations: prefer not to declare `rdfs:range` on properties you intend to constrain with `sh:class`; when class membership is the thing under test, run validation without RDFS entailment (`inference="none"`); or express the check as a constraint the entailment cannot satisfy (for example a literal property constraint). Note the trade-off: with entailment off, `sh:targetClass` no longer reaches subclasses, so subclass hierarchies need explicit typing or inference-aware target selection. Semantica's own `run_shacl_validation` wrapper already calls pyshacl with `inference="none"`, so this pitfall only bites when calling `pyshacl.validate` directly with entailment enabled. - **Trusting `conforms: True` without checking the inference mode**: an inference-enabled run can hide the exact violations the shapes were written to catch (see above). Record which inference mode validation ran under alongside the result, and re-run shape sets that contain `sh:class`/`sh:node` with entailment off before treating a pass as authoritative. --- @@ -435,7 +435,7 @@ A DoD CTI team enforces STIX-compatible constraints on a threat graph before sha from semantica.context import AgentContext, ContextGraph from semantica.vector_store import VectorStore from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape -from semantica.ontology.ontology_validator import _run_pyshacl +from semantica.ontology import run_shacl_validation graph = ContextGraph() ctx = AgentContext( @@ -487,7 +487,7 @@ data_ttl = """ a ex:Malware . """ -report = _run_pyshacl(data_ttl, shacl_ttl) +report = run_shacl_validation(data_ttl, shacl_ttl) print(f"CTI graph conforms : {report.conforms}") print(f"Violations : {report.violation_count}") print(f"Warnings : {report.warning_count}") @@ -508,7 +508,7 @@ A SOC team validates zero-trust policy nodes before publishing them to the polic ```python from semantica.context import ContextGraph from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape -from semantica.ontology.ontology_validator import _run_pyshacl +from semantica.ontology import run_shacl_validation graph = ContextGraph() graph.add_node("policy-001", "Policy", "MFA Required for Tier-1 Resources", @@ -555,7 +555,7 @@ data_ttl = """ a ex:Policy . """ -report = _run_pyshacl(data_ttl, shacl_ttl) +report = run_shacl_validation(data_ttl, shacl_ttl) print(f"Policy graph conforms: {report.conforms}") # Policy graph conforms: False @@ -573,7 +573,7 @@ A clinical informatics team validates trial ontology nodes before loading them i ```python from semantica.ontology import LLMOntologyGenerator, SHACLGenerator, PropertyShape -from semantica.ontology.ontology_validator import _run_pyshacl +from semantica.ontology import run_shacl_validation from semantica.export import export_rdf import tempfile, os @@ -625,7 +625,7 @@ with open(tmp.name) as f: data_ttl = f.read() os.unlink(tmp.name) -report = _run_pyshacl(data_ttl, shacl_ttl) +report = run_shacl_validation(data_ttl, shacl_ttl) print(f"Trial data conforms: {report.conforms}") print(f"Warnings : {report.warning_count}") ``` @@ -639,7 +639,7 @@ A credit risk team validates every `LoanApplication` node against Basel III CRE2 ```python from semantica.context import ContextGraph from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape -from semantica.ontology.ontology_validator import _run_pyshacl +from semantica.ontology import run_shacl_validation graph = ContextGraph() graph.add_node("loan-001", "LoanApplication", "Prime mortgage APP-2025-88421", @@ -684,7 +684,7 @@ data_ttl = """ ex:ltv "0.65" . """ -report = _run_pyshacl(data_ttl, shacl_ttl) +report = run_shacl_validation(data_ttl, shacl_ttl) print(f"Loan portfolio conforms: {report.conforms}") # Loan portfolio conforms: False @@ -714,14 +714,14 @@ Call this function as a pre-publish gate; exit code 1 blocks the pipeline. ```python import sys from semantica.ontology import OntologyGenerator, SHACLGenerator -from semantica.ontology.ontology_validator import _run_pyshacl +from semantica.ontology import run_shacl_validation def validate_before_publish(data_graph_str: str, ontology: dict) -> None: shacl_gen = SHACLGenerator(base_uri="https://example.org/shapes/") shacl_graph = shacl_gen.generate(ontology) shacl_ttl = shacl_gen.serialize(shacl_graph, format="turtle") - report = _run_pyshacl(data_graph_str, shacl_ttl) + report = run_shacl_validation(data_graph_str, shacl_ttl) if not report.conforms: print(f"Graph validation FAILED — {report.violation_count} violation(s)") @@ -739,7 +739,6 @@ def validate_before_publish(data_graph_str: str, ontology: dict) -> None: - [Ontology Management](ontology) — generate the OWL ontology that SHACL shapes are derived from - [Reasoning & Rules](reasoning) — complement SHACL structural constraints with logical inference rules -- [Export & Serialization](export) — serialize graph data to Turtle/RDF/XML for `_run_pyshacl` input +- [Export & Serialization](export) — serialize graph data to Turtle/RDF/XML for `run_shacl_validation` input - [Conflict Resolution](conflict-resolution) — detect and resolve data conflicts before SHACL validation - [Change Management](change-management) — version-gate SHACL shapes alongside ontology versions - diff --git a/semantica/ontology/__init__.py b/semantica/ontology/__init__.py index 98edff7a..27b1318a 100644 --- a/semantica/ontology/__init__.py +++ b/semantica/ontology/__init__.py @@ -159,6 +159,7 @@ from .ontology_validator import ( SHACLValidationReport, SHACLViolation, ValidationResult, + run_shacl_validation, validate_ontology, ) from .owl_generator import OWLGenerator @@ -192,6 +193,7 @@ __all__ = [ "PropertyShape", "SHACLValidationReport", "SHACLViolation", + "run_shacl_validation", # OWL/RDF generation "OWLGenerator", # Requirements and competency questions diff --git a/semantica/ontology/ontology_validator.py b/semantica/ontology/ontology_validator.py index 5d9d10df..85adb0a8 100644 --- a/semantica/ontology/ontology_validator.py +++ b/semantica/ontology/ontology_validator.py @@ -145,14 +145,14 @@ class SHACLValidationReport: } -def _run_pyshacl( +def run_shacl_validation( data_graph_str: str, shacl_str: str, data_graph_format: str = "turtle", shacl_format: str = "turtle", ) -> SHACLValidationReport: """ - Run pyshacl validation and return a structured SHACLValidationReport. + Run pySHACL validation and return a structured SHACLValidationReport. Args: data_graph_str: Serialized data graph string. @@ -272,6 +272,21 @@ def _run_pyshacl( raw_report=results_text, ) + +def _run_pyshacl( + data_graph_str: str, + shacl_str: str, + data_graph_format: str = "turtle", + shacl_format: str = "turtle", +) -> SHACLValidationReport: + """Backward-compatible alias for :func:`run_shacl_validation`.""" + return run_shacl_validation( + data_graph_str, + shacl_str, + data_graph_format=data_graph_format, + shacl_format=shacl_format, + ) + @dataclass class ValidationResult: """Result of an ontology validation operation.""" diff --git a/tests/ontology/test_ontology_advanced.py b/tests/ontology/test_ontology_advanced.py index 98d5dc6e..149a8789 100644 --- a/tests/ontology/test_ontology_advanced.py +++ b/tests/ontology/test_ontology_advanced.py @@ -525,6 +525,30 @@ class TestSHACLHierarchicalAndValidation(unittest.TestCase): self.assertEqual(mc[0].max_count, 2) # 33 + def test_public_run_shacl_validation_api(self): + """The public API validates data and retains the legacy alias.""" + try: + import pyshacl # noqa: F401 + import rdflib # noqa: F401 + except ImportError: + self.skipTest("pyshacl/rdflib not installed") + from semantica.ontology import run_shacl_validation + from semantica.ontology.ontology_validator import _run_pyshacl + + data = "@prefix ex: . ex:alice a ex:Person ." + shacl = """ + @prefix ex: . + @prefix sh: . + ex:PersonShape a sh:NodeShape ; sh:targetClass ex:Person ; + sh:property [ sh:path ex:name ; sh:minCount 1 ] . + """ + public_report = run_shacl_validation(data, shacl) + legacy_report = _run_pyshacl(data, shacl) + self.assertFalse(public_report.conforms) + self.assertEqual(public_report.violation_count, 1) + self.assertEqual(legacy_report.to_dict(), public_report.to_dict()) + + # 34 def test_shacl_violation_to_dict(self): from semantica.ontology.ontology_validator import SHACLViolation v = SHACLViolation( From fe3baad67c25106d0a84e122d109fe2e3fab6be7 Mon Sep 17 00:00:00 2001 From: mikemikimike <13286568797@163.com> Date: Sun, 23 Aug 2026 00:02:57 +0800 Subject: [PATCH 12/20] docs(shacl): correct legacy alias name --- docs/guides/shacl-validation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guides/shacl-validation.md b/docs/guides/shacl-validation.md index eafea841..76c07b89 100644 --- a/docs/guides/shacl-validation.md +++ b/docs/guides/shacl-validation.md @@ -8,7 +8,7 @@ icon: "shield-check" SHACL (Shapes Constraint Language) is a standard for validating graph-based data. While an ontology defines the conceptual *schema* (the "what" exists in your domain), SHACL defines the structural *rules and constraints* (the "how" it should be structured). -In Semantica, `SHACLGenerator` produces constraint rules (shapes) based on your ontology, and the public `run_shacl_validation` function evaluates your actual data against these rules. If a node violates a rule (e.g., missing a required property or using the wrong datatype), a detailed violation report is generated. The historical `run_shacl_validation` name remains available as a compatibility alias. +In Semantica, `SHACLGenerator` produces constraint rules (shapes) based on your ontology, and the public `run_shacl_validation` function evaluates your actual data against these rules. If a node violates a rule (e.g., missing a required property or using the wrong datatype), a detailed violation report is generated. The historical `_run_pyshacl` name remains available as a compatibility alias. ## Why Use SHACL Validation? From 6cbe0ae43846021d056d8d0e4151d817b37d80b7 Mon Sep 17 00:00:00 2001 From: mikemikimike <13286568797@163.com> Date: Sun, 23 Aug 2026 00:13:12 +0800 Subject: [PATCH 13/20] test(shacl): cover conforming validation result --- tests/ontology/test_ontology_advanced.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/ontology/test_ontology_advanced.py b/tests/ontology/test_ontology_advanced.py index 149a8789..7b47f3a3 100644 --- a/tests/ontology/test_ontology_advanced.py +++ b/tests/ontology/test_ontology_advanced.py @@ -549,6 +549,30 @@ class TestSHACLHierarchicalAndValidation(unittest.TestCase): self.assertEqual(legacy_report.to_dict(), public_report.to_dict()) # 34 + def test_public_run_shacl_validation_conforming_graph(self): + """The public API reports a valid graph without violations.""" + try: + import pyshacl # noqa: F401 + import rdflib # noqa: F401 + except ImportError: + self.skipTest("pyshacl/rdflib not installed") + from semantica.ontology import run_shacl_validation + + data = """ + @prefix ex: . + ex:alice a ex:Person ; ex:name "Alice" . + """ + shacl = """ + @prefix ex: . + @prefix sh: . + ex:PersonShape a sh:NodeShape ; sh:targetClass ex:Person ; + sh:property [ sh:path ex:name ; sh:minCount 1 ] . + """ + + report = run_shacl_validation(data, shacl) + + self.assertTrue(report.conforms) + self.assertEqual(report.violation_count, 0) def test_shacl_violation_to_dict(self): from semantica.ontology.ontology_validator import SHACLViolation v = SHACLViolation( From 9123dcc0bdeef8890d83f91d6e7456d22afc7aa1 Mon Sep 17 00:00:00 2001 From: Nitish Reddy M Date: Sat, 22 Aug 2026 12:14:17 -0400 Subject: [PATCH 14/20] fix(export): mint JSON-LD document @id from content, not the clock (#1181) Closes #1147 --- semantica/export/json_exporter.py | 60 ++++++++-- tests/export/test_jsonld_document_iri.py | 134 +++++++++++++++++++++++ tests/export/test_timestamp_timezones.py | 27 +++-- 3 files changed, 205 insertions(+), 16 deletions(-) create mode 100644 tests/export/test_jsonld_document_iri.py diff --git a/semantica/export/json_exporter.py b/semantica/export/json_exporter.py index 0c3b492c..e31c39d8 100644 --- a/semantica/export/json_exporter.py +++ b/semantica/export/json_exporter.py @@ -28,12 +28,35 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Union from ..utils.exceptions import ProcessingError, ValidationError -from ..utils.helpers import ensure_directory, utc_now_iso, write_json_file +from ..utils.helpers import ensure_directory, hash_data, utc_now_iso, write_json_file from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker from .rdf_exporter import SEMANTICA_NS, mint_entity_iri, mint_relationship_iri +def _content_iri(prefix: str, payload: Any) -> str: + """Mint a document IRI from what was exported, not when. + + Minting from ``utc_now_iso()`` gave every export of the same graph a new + identity a few microseconds apart, so re-exporting an unchanged graph was + never idempotent and merging exports duplicated every node (#1147). This + mirrors ``mint_entity_iri`` (#1109): identical content hashes to the same + IRI, and any change to the content changes it too. ``default=str`` keeps + the hash defined for values ``json.dumps`` would otherwise reject, such as + ``datetime`` objects a caller may have left in the graph. + + Args: + prefix: IRI prefix the digest is appended to + payload: JSON-serializable value whose content determines the digest + + Returns: + A stable IRI of the form ``{prefix}{16-hex-char digest}`` + """ + canonical = json.dumps(payload, sort_keys=True, default=str) + digest = hash_data(canonical)[:16] + return f"{prefix}{digest}" + + def _is_jsonld_document(data: Dict[str, Any]) -> bool: """ Report whether a dictionary is already a JSON-LD document. @@ -230,7 +253,10 @@ class JSONExporter: - statistics: Statistics dictionary (optional) file_path: Output JSON file path format: Export format - 'json' or 'json-ld' (default: self.format) - **options: Additional options passed to conversion methods + **options: Additional options passed to conversion methods: + - graph_uri: Caller-supplied IRI for the graph node when + format='json-ld', overriding the default content-derived + IRI (see #1147) Example: >>> kg = { @@ -401,7 +427,9 @@ class JSONExporter: data: Data to convert (dict, list, or any value) include_metadata: Whether to include metadata (default: True) include_provenance: Whether to include provenance (default: True) - **options: Additional options passed to knowledge graph conversion + **options: Additional options passed to knowledge graph conversion: + - document_uri: Caller-supplied IRI for the document node, + overriding the default content-derived IRI (see #1147) Returns: Dictionary in JSON-LD format with @context, @graph/@value, and metadata @@ -451,13 +479,17 @@ class JSONExporter: # Add metadata and provenance if requested if include_metadata: - self._attach_document_metadata(jsonld, include_provenance) + self._attach_document_metadata( + jsonld, include_provenance, options.get("document_uri") + ) return jsonld @staticmethod def _attach_document_metadata( - jsonld: Dict[str, Any], include_provenance: bool + jsonld: Dict[str, Any], + include_provenance: bool, + document_uri: Optional[str] = None, ) -> None: """ Attach the export's own metadata without naming the graph. @@ -473,6 +505,9 @@ class JSONExporter: Args: jsonld: Document being built, modified in place include_provenance: Whether to record how and when it was exported + document_uri: Caller-supplied IRI for the document node. Falls back + to a content-derived IRI (#1147) so re-exporting unchanged data + is idempotent instead of minting a new identity every time. """ # A caller may hand us a document that is deliberately a named graph. # That name is theirs to keep, but our own statements must not end up @@ -483,7 +518,10 @@ class JSONExporter: # Do not overwrite an identifier the payload already carries: the # knowledge-graph conversion names its own document node. if "@id" not in jsonld or payload_is_named_graph: - document["@id"] = f"https://semantica.dev/data/{utc_now_iso()}" + content = {key: value for key, value in jsonld.items() if key != "@context"} + document["@id"] = document_uri or _content_iri( + "https://semantica.dev/data/", content + ) if include_provenance: document["semantica:exportedAt"] = utc_now_iso() document["semantica:format"] = "json-ld" @@ -553,7 +591,9 @@ class JSONExporter: - entities: List of entity dictionaries - relationships: List of relationship dictionaries - metadata: Metadata dictionary (optional) - **options: Additional options (unused) + **options: Additional options: + - graph_uri: Caller-supplied IRI for the graph node, + overriding the default content-derived IRI (see #1147) Returns: Dictionary in JSON-LD format with @context, @id, @type, and graph data @@ -566,7 +606,11 @@ class JSONExporter: "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdfs": "http://www.w3.org/2000/01/rdf-schema#", }, - "@id": f"https://semantica.dev/graph/{utc_now_iso()}", + # Minted from the graph's own content rather than the wall clock + # (#1147): re-exporting an unchanged graph must produce the same + # subject, or merging repeated exports duplicates every node. + "@id": options.get("graph_uri") + or _content_iri("https://semantica.dev/graph/", kg), "@type": "semantica:KnowledgeGraph", } diff --git a/tests/export/test_jsonld_document_iri.py b/tests/export/test_jsonld_document_iri.py new file mode 100644 index 00000000..34b2366e --- /dev/null +++ b/tests/export/test_jsonld_document_iri.py @@ -0,0 +1,134 @@ +"""The document IRI of a JSON-LD export must depend on content, not the clock +(issue #1147). + +``_convert_kg_to_jsonld`` minted the graph's ``@id`` from ``utc_now_iso()``, +and the generic ``_attach_document_metadata`` path did the same for a plain +document ``@id``. Exporting an unchanged graph therefore produced a new +subject every time: three exports of one one-entity graph merged into 3 +``semantica:KnowledgeGraph`` nodes and 15 triples for what should have been a +single graph. Neither identifier resolves and the timestamp is already +recorded correctly in ``semantica:exportedAt``, so the fix mints the IRI from +the exported content instead (mirroring ``mint_entity_iri``, #1109), with an +optional caller-supplied override for callers who already name their graphs. +""" + +import json + +from rdflib import RDF, Graph, URIRef + +from semantica.export.json_exporter import JSONExporter + +KG = { + "entities": [{"id": "https://example.org/e1", "text": "Acme Corp", "type": "ORG"}], + "relationships": [], +} + +OTHER_KG = { + "entities": [ + {"id": "https://example.org/e1", "text": "Acme Corp Renamed", "type": "ORG"} + ], + "relationships": [], +} + + +def _export(kg, tmp_path, name="out.jsonld", **options): + path = tmp_path / name + JSONExporter().export_knowledge_graph(kg, path, format="json-ld", **options) + return path + + +def test_reexporting_an_unchanged_graph_is_idempotent(tmp_path): + """The whole point of an identifier: same content, same @id.""" + first = json.loads(_export(KG, tmp_path, "a.jsonld").read_text()) + second = json.loads(_export(KG, tmp_path, "b.jsonld").read_text()) + + assert first["@id"] == second["@id"] + + +def test_a_changed_graph_gets_a_different_id(tmp_path): + unchanged = json.loads(_export(KG, tmp_path, "a.jsonld").read_text()) + changed = json.loads(_export(OTHER_KG, tmp_path, "b.jsonld").read_text()) + + assert unchanged["@id"] != changed["@id"] + + +def test_merging_repeated_exports_yields_one_graph_node(tmp_path): + """Regression for the exact repro in #1147: churn no longer multiplies nodes.""" + merged = Graph() + for i in range(3): + path = _export(KG, tmp_path, f"churn{i}.jsonld") + merged.parse(str(path), format="json-ld") + + # Exactly one subject typed as a KnowledgeGraph, regardless of how many + # times the unchanged graph was exported and merged. + kg_nodes = set( + merged.subjects(RDF.type, URIRef("https://semantica.dev/ns#KnowledgeGraph")) + ) + assert len(kg_nodes) == 1 + + entity_nodes = set( + merged.subjects(RDF.type, URIRef("https://semantica.dev/vocab/ORG")) + ) + assert len(entity_nodes) == 1 + + +def test_exported_at_still_varies_between_exports(tmp_path): + """Identity is now content-derived, but provenance still records each run.""" + first = json.loads(_export(KG, tmp_path, "a.jsonld").read_text()) + second = json.loads(_export(KG, tmp_path, "b.jsonld").read_text()) + + assert first["@id"] == second["@id"] + assert first["semantica:exportedAt"] != second["semantica:exportedAt"] + + +def test_caller_supplied_graph_uri_is_honored(tmp_path): + path = _export(KG, tmp_path, graph_uri="https://example.org/my-graph") + document = json.loads(path.read_text()) + + assert document["@id"] == "https://example.org/my-graph" + + +def _document_node_id(document): + """The generic (non-knowledge-graph) path hangs its own @id off a member + of @graph rather than the top level, to avoid re-creating the named-graph + bug fixed by #1145. Find that member and return its @id.""" + for node in document["@graph"]: + if "semantica:exportedAt" in node: + return node["@id"] + raise AssertionError(f"no document metadata node in @graph: {document}") + + +def test_caller_supplied_document_uri_is_honored_for_a_generic_export(tmp_path): + payload = {"note": "no entities or relationships here"} + path = tmp_path / "generic.jsonld" + JSONExporter().export( + payload, path, format="json-ld", document_uri="https://example.org/my-doc" + ) + document = json.loads(path.read_text()) + + assert _document_node_id(document) == "https://example.org/my-doc" + + +def test_generic_document_id_is_also_content_derived(tmp_path): + """The non-knowledge-graph path (_attach_document_metadata) gets the same fix.""" + payload = {"note": "plain data, no @id of its own"} + + first = tmp_path / "a.jsonld" + second = tmp_path / "b.jsonld" + JSONExporter().export(payload, first, format="json-ld") + JSONExporter().export(dict(payload), second, format="json-ld") + + first_id = _document_node_id(json.loads(first.read_text())) + second_id = _document_node_id(json.loads(second.read_text())) + assert first_id == second_id + + +def test_document_id_still_differs_for_different_generic_payloads(tmp_path): + a = tmp_path / "a.jsonld" + b = tmp_path / "b.jsonld" + JSONExporter().export({"note": "one"}, a, format="json-ld") + JSONExporter().export({"note": "two"}, b, format="json-ld") + + a_id = _document_node_id(json.loads(a.read_text())) + b_id = _document_node_id(json.loads(b.read_text())) + assert a_id != b_id diff --git a/tests/export/test_timestamp_timezones.py b/tests/export/test_timestamp_timezones.py index df34dd29..b97b658e 100644 --- a/tests/export/test_timestamp_timezones.py +++ b/tests/export/test_timestamp_timezones.py @@ -107,20 +107,31 @@ def test_exported_timestamp_survives_a_timezone_qualified_sparql_filter(): assert len(rows) == 1, "the export was dropped by a timezone-qualified filter" -def test_document_iri_carrying_an_offset_is_a_valid_iri(): - """The offset puts '+' and ':' in the @id; both are legal in a path.""" +def test_document_iri_is_a_valid_iri(): + """The graph @id must be a valid IRI regardless of how it is minted. + + Before #1147, this @id was minted from the offset-carrying timestamp + itself (``+00:00`` interpolated straight into the path), so this test + asserted the offset survived without breaking IRI validity. #1147 mints + the @id from the graph's content instead, so the timestamp no longer + appears here at all — it stays in ``semantica:exportedAt`` (still + offset-aware, per ``test_jsonld_export_timestamp_is_offset_aware`` above). + What's left worth guarding is the general case: whatever the @id is + minted from, it has to be a valid IRI that round-trips through RDF. + """ rdflib = pytest.importorskip("rdflib") document_iri = JSONExporter()._convert_kg_to_jsonld(KG)["@id"] - assert "+00:00" in document_iri assert rdflib.term._is_valid_uri(document_iri) graph = rdflib.Graph() - graph.add(( - rdflib.URIRef(document_iri), - rdflib.RDF.type, - rdflib.URIRef("https://semantica.dev/ns#KnowledgeGraph"), - )) + graph.add( + ( + rdflib.URIRef(document_iri), + rdflib.RDF.type, + rdflib.URIRef("https://semantica.dev/ns#KnowledgeGraph"), + ) + ) reparsed = rdflib.Graph().parse(data=graph.serialize(format="nt"), format="nt") assert document_iri in {str(s) for s in reparsed.subjects()} From b891902d6d501df683449b41f4cff99fe59ba7eb Mon Sep 17 00:00:00 2001 From: mikemikimike <13286568797@163.com> Date: Sun, 23 Aug 2026 00:15:41 +0800 Subject: [PATCH 15/20] test(shacl): compare stable report fields --- tests/ontology/test_ontology_advanced.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/ontology/test_ontology_advanced.py b/tests/ontology/test_ontology_advanced.py index 7b47f3a3..dfdde003 100644 --- a/tests/ontology/test_ontology_advanced.py +++ b/tests/ontology/test_ontology_advanced.py @@ -546,7 +546,18 @@ class TestSHACLHierarchicalAndValidation(unittest.TestCase): legacy_report = _run_pyshacl(data, shacl) self.assertFalse(public_report.conforms) self.assertEqual(public_report.violation_count, 1) - self.assertEqual(legacy_report.to_dict(), public_report.to_dict()) + self.assertEqual(legacy_report.conforms, public_report.conforms) + self.assertEqual(legacy_report.violation_count, public_report.violation_count) + self.assertEqual( + [ + (v.focus_node, v.result_path, v.constraint, v.severity, v.message) + for v in legacy_report.violations + ], + [ + (v.focus_node, v.result_path, v.constraint, v.severity, v.message) + for v in public_report.violations + ], + ) # 34 def test_public_run_shacl_validation_conforming_graph(self): From 248ae57694e022fd647a30f8a81fdc59b47721a8 Mon Sep 17 00:00:00 2001 From: Aldrin Joseph Date: Sat, 22 Aug 2026 22:05:37 +0530 Subject: [PATCH 16/20] fix(context): extend causal vocabulary normalization to sibling methods (#1184) Review feedback: analyze_decision_influence(), trace_decision_causality(), and find_precedents() had the same vocabulary split as get_causal_chain(). The first two read edge_type_index, which is keyed by the RAW edge_type string, so they now filter index keys by normalized type; find_precedents() accepts the analyzer's 'precedes' spelling alongside PRECEDENT_FOR. Adds regression tests for all three call sites. --- semantica/context/context_graph.py | 24 ++++++-- .../test_decision_causal_edge_regression.py | 61 +++++++++++++++++++ 2 files changed, 79 insertions(+), 6 deletions(-) diff --git a/semantica/context/context_graph.py b/semantica/context/context_graph.py index d18c2b54..7f781a4b 100644 --- a/semantica/context/context_graph.py +++ b/semantica/context/context_graph.py @@ -2889,10 +2889,13 @@ class ContextGraph: Returns: List of precedent decisions """ - # Find decisions connected via PRECEDENT_FOR relationships + # Find decisions connected via PRECEDENT_FOR relationships, accepting + # the analyzer vocabulary's "precedes" spelling as well (issue #1184). precedent_ids = [] for edge in self.edges: - if edge.target_id == decision_id and edge.edge_type == "PRECEDENT_FOR": + if edge.target_id == decision_id and edge.edge_type.upper() in { + "PRECEDENT_FOR", "PRECEDES", + }: precedent_ids.append(edge.source_id) # Convert to Decision objects @@ -3453,8 +3456,13 @@ class ContextGraph: # Explicit causal relationships recorded via add_causal_relationship() are # ground truth and always count as direct influence, in either direction. - for edge_type in _CAUSAL_EDGE_TYPES: - for edge in self.edge_type_index.get(edge_type, []): + # The index is keyed by the raw edge_type string ("causes" and "CAUSED" + # are separate keys), so filter by normalized type instead of iterating + # a fixed spelling list. + for edge_type, edges in self.edge_type_index.items(): + if edge_type.upper() not in _CAUSAL_TRAVERSAL_TYPES: + continue + for edge in edges: if edge.source_id == decision_id and edge.target_id in self._decisions: direct_influence.add(edge.target_id) elif edge.target_id == decision_id and edge.source_id in self._decisions: @@ -3600,8 +3608,12 @@ class ContextGraph: # record_decision() (e.g. a graph restored via from_dict), so only # causes with a known decision record are kept. incoming_causal_edges = defaultdict(list) - for edge_type in _CAUSAL_EDGE_TYPES: - for edge in self.edge_type_index.get(edge_type, []): + # The index is keyed by the raw edge_type string ("causes" and + # "CAUSED" are separate keys), so filter by normalized type. + for edge_type, edges in self.edge_type_index.items(): + if edge_type.upper() not in _CAUSAL_TRAVERSAL_TYPES: + continue + for edge in edges: if edge.source_id in self._decisions: incoming_causal_edges[edge.target_id].append(edge) diff --git a/tests/context/test_decision_causal_edge_regression.py b/tests/context/test_decision_causal_edge_regression.py index 211864d0..eae7eae3 100644 --- a/tests/context/test_decision_causal_edge_regression.py +++ b/tests/context/test_decision_causal_edge_regression.py @@ -391,3 +391,64 @@ def test_add_causal_relationship_rejects_non_string_with_value_error(): for bad_type in (None, 42, ["CAUSED"]): with pytest.raises(ValueError): graph.add_causal_relationship(cause, effect, relationship_type=bad_type) + + +def test_analyze_decision_influence_sees_lowercase_causal_edge(): + """Issue #1184 follow-up: influence analysis reads the same edge index as + the causal traversal, so lowercase edges must count as direct influence.""" + graph = ContextGraph(advanced_analytics=True) + cause = graph.record_decision( + category="a", scenario="upstream", reasoning="r", + outcome="x", confidence=0.9, + ) + effect = graph.record_decision( + category="b", scenario="downstream", reasoning="r", + outcome="y", confidence=0.9, + ) + graph.add_edge(cause, effect, "causes") + + impact = graph.analyze_decision_influence(cause) + + direct_ids = {entry["decision_id"] for entry in impact["direct_influence"]} + assert effect in direct_ids + + +def test_trace_decision_causality_sees_lowercase_causal_edge(): + """Issue #1184 follow-up: the trace must not return an empty audit chain + for a decision with an explicit lowercase upstream causal edge.""" + graph = ContextGraph(advanced_analytics=True) + cause = graph.record_decision( + category="a", scenario="upstream", reasoning="r", + outcome="x", confidence=0.9, + ) + effect = graph.record_decision( + category="b", scenario="downstream", reasoning="r", + outcome="y", confidence=0.9, + ) + graph.add_edge(cause, effect, "causes") + + trace = graph.trace_decision_causality(effect) + + assert any( + hop["from"] == cause and hop["to"] == effect + for chain in trace for hop in chain["hops"] + ), "lowercase causal edge must appear in the traced chain" + + +def test_find_precedents_sees_lowercase_precedent_edge(): + """Issue #1184 follow-up: precedent lookup must accept the analyzer's + spelling alongside the canonical PRECEDENT_FOR.""" + graph = ContextGraph(advanced_analytics=True) + precedent = graph.record_decision( + category="a", scenario="earlier", reasoning="r", + outcome="x", confidence=0.9, + ) + later = graph.record_decision( + category="b", scenario="later", reasoning="r", + outcome="y", confidence=0.9, + ) + graph.add_edge(precedent, later, "precedes") + + precedents = graph.find_precedents(later) + + assert [d.decision_id for d in precedents] == [precedent] From 727b0383cc1ad0c1f894e6887d62ec5499c54d1a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 22 Aug 2026 22:34:54 +0530 Subject: [PATCH 17/20] security(deps): bump botocore from 1.43.69 to 1.43.73 (#1047) Bumps [botocore](https://github.com/boto/botocore) from 1.43.69 to 1.43.73. - [Commits](https://github.com/boto/botocore/compare/1.43.69...1.43.73) --- updated-dependencies: - dependency-name: botocore dependency-version: 1.43.71 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements-ci.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-ci.txt b/requirements-ci.txt index 1126111c..b8ca419a 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -403,9 +403,9 @@ boto3==1.43.69 \ --hash=sha256:4eb494d05b2bd08a7eee61b8ac4c34745c99e9bbce435c91f8d15d372dd8c2db \ --hash=sha256:76297a0b415849c63575ae08a4f1661b2dc8ee0100f104b86f98aa69b47fa2c7 # via semantica (pyproject.toml) -botocore==1.43.69 \ - --hash=sha256:5caa46b740d9a886137146ffbb69edb691f702bfe74c64e85621947ae00181fd \ - --hash=sha256:b1f0e01c53d6b84ee9c184ebf3636c3b3aef85e0ae8498c74afb8734ff224f87 +botocore==1.43.73 \ + --hash=sha256:068433028e011ccbeab1dd7c46b1090c24e378397693c66e67ca571176498daa \ + --hash=sha256:0fa1e63c24b3531be3e1bc1687a88b3be9e63a430153f24edd93efc162bb1c51 # via # boto3 # s3transfer From 331c857672435b01f4c6b2b1618685635ad84e69 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 22 Aug 2026 22:36:28 +0530 Subject: [PATCH 18/20] security(deps): bump agno from 2.8.7 to 2.9.0 (#1050) Bumps [agno](https://github.com/agno-agi/agno) from 2.8.7 to 2.9.0. - [Release notes](https://github.com/agno-agi/agno/releases) - [Commits](https://github.com/agno-agi/agno/compare/v2.8.7...v2.9.0) --- updated-dependencies: - dependency-name: agno dependency-version: 2.9.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements-ci.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-ci.txt b/requirements-ci.txt index b8ca419a..756fd829 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -6,9 +6,9 @@ accelerate==1.14.0 \ # via # docling-ibm-models # docling-slim -agno==2.8.7 \ - --hash=sha256:6a2763eb469163f7b79ab1da6ca2f22d8619f6b9d614574f975d9c12bb4323ea \ - --hash=sha256:d49396a2062ee6994ca82695b9bd1e1b95667fec432c544afa38133e564bf090 +agno==2.9.0 \ + --hash=sha256:7777674b3931b341fad4fcf02a61b185a08588c509101348facf87feb2144c0c \ + --hash=sha256:7d9c134703e3c2798023cd57dcb9caa8e1174f6914813f9b130becfc3521a46f # via semantica (pyproject.toml) agnoctl==0.1.3 \ --hash=sha256:6fce1d2482b1f2e0a3d14b0a7c12fbd49d8df4f0bf0a4fd9fd91753cbff5efdc \ From 48f219cd5cddf5af26160053ecae9e1543583277 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 22 Aug 2026 22:43:30 +0530 Subject: [PATCH 19/20] deps(deps): bump google-genai from 2.17.0 to 2.18.1 (#1163) Bumps [google-genai](https://github.com/googleapis/python-genai) from 2.17.0 to 2.18.1. - [Release notes](https://github.com/googleapis/python-genai/releases) - [Changelog](https://github.com/googleapis/python-genai/blob/main/CHANGELOG.md) - [Commits](https://github.com/googleapis/python-genai/compare/v2.17.0...v2.18.1) --- updated-dependencies: - dependency-name: google-genai dependency-version: 2.18.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements-ci.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-ci.txt b/requirements-ci.txt index 756fd829..5f34aa6b 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -1731,9 +1731,9 @@ google-crc32c==1.8.0 \ # via # google-cloud-storage # google-resumable-media -google-genai==2.17.0 \ - --hash=sha256:6b640a2390c82b4a240873eddb9f518c6d2c33244b2de16ed3d14526a6da7f57 \ - --hash=sha256:a4835563c60aee646c9c4b261c507aa4a624710d25017012d20dc65abf3d9a54 +google-genai==2.18.1 \ + --hash=sha256:36a5949233e64a60f6cc4521bff7a76b7c569d0aa227bbe9fa642213b8a3a3b2 \ + --hash=sha256:a1e2be75c16234adc6641afd1ad4dd44218c9eec005d938bdc428585a048918a # via semantica (pyproject.toml) google-resumable-media==2.10.1 \ --hash=sha256:224975032ddb73f7ed9e2f0f4cc08ed1b06874c52d48cc8533e3eb72980b21a0 \ From f124df4229210c4bc35b544adcdfe71e652eef59 Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Sun, 23 Aug 2026 15:51:25 +0530 Subject: [PATCH 20/20] fix: prevent duplicate dimension kwarg crash in create_index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit vector_store_config.get_all() always includes a "dimension" key, so forwarding it via **config into VectorIndexer(dimension=dimension, **config) raised "got multiple values for keyword argument 'dimension'" any time the default index-creation path ran with the default config — including `semantica embed index`, which is exactly the second half of the #994 quick-start pipeline this PR fixes. --- semantica/vector_store/methods.py | 6 +++++- tests/vector_store/test_vector_store.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/semantica/vector_store/methods.py b/semantica/vector_store/methods.py index d74d763b..5a558b27 100644 --- a/semantica/vector_store/methods.py +++ b/semantica/vector_store/methods.py @@ -297,7 +297,11 @@ def create_index( config = vector_store_config.get_all() backend = config.get("default_backend", "faiss") dimension = config.get("dimension", 768) - indexer = VectorIndexer(backend=backend, dimension=dimension, **config) + # backend/dimension are already passed explicitly; drop them from the + # forwarded config so VectorIndexer(..., **remaining_config) doesn't + # receive duplicate keyword arguments. + remaining_config = {k: v for k, v in config.items() if k not in ("default_backend", "dimension")} + indexer = VectorIndexer(backend=backend, dimension=dimension, **remaining_config) return indexer.create_index(vectors, ids, **options) diff --git a/tests/vector_store/test_vector_store.py b/tests/vector_store/test_vector_store.py index 219f88c2..80ff3e59 100644 --- a/tests/vector_store/test_vector_store.py +++ b/tests/vector_store/test_vector_store.py @@ -243,5 +243,19 @@ class TestVectorStore(unittest.TestCase): shutil.rmtree(tmpdir, ignore_errors=True) +class TestCreateIndexFunction(unittest.TestCase): + """create_index() forwards vector_store_config's defaults into VectorIndexer, + which already receives backend/dimension as explicit args. Regression for the + 'got multiple values for keyword argument dimension' crash on the default + (unmocked) config, hit by e.g. `semantica embed index`.""" + + def test_create_index_with_default_config(self): + from semantica.vector_store.methods import create_index + + vectors = [np.array([0.1, 0.2, 0.3]), np.array([0.4, 0.5, 0.6])] + index = create_index(vectors, ids=["a", "b"]) + self.assertIsNotNone(index) + + if __name__ == '__main__': unittest.main()