mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-09-01 04:00:28 +00:00
Compare commits
42
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bf292ccbbc | ||
|
|
64d942503b | ||
|
|
471a420b10 | ||
|
|
a4aa71ad87 | ||
|
|
fa87a1a9be | ||
|
|
08c78bfb40 | ||
|
|
56b174781f | ||
|
|
dfda4c561a | ||
|
|
ea0dd17bff | ||
|
|
f6cd62411b | ||
|
|
cac6dfbe45 | ||
|
|
80e9737542 | ||
|
|
14fb975fa1 | ||
|
|
8b0ac61afd | ||
|
|
9d98eedaa1 | ||
|
|
5ffb212a8f | ||
|
|
ca7f743dab | ||
|
|
5cf59fdd88 | ||
|
|
0384a8de30 | ||
|
|
f3d5932c24 | ||
|
|
a5aac7e22a | ||
|
|
3448ac0689 | ||
|
|
70dfbf151c | ||
|
|
47446ebdde | ||
|
|
c8a591e89e | ||
|
|
ab86127e4e | ||
|
|
30592b1285 | ||
|
|
aceb69a5bc | ||
|
|
e13c953bd8 | ||
|
|
85d6ccd0a5 | ||
|
|
19ff5bf200 | ||
|
|
4bf525d409 | ||
|
|
8858beb6d9 | ||
|
|
d3183d0ab3 | ||
|
|
da642f12fa | ||
|
|
5376f046ca | ||
|
|
56d9e9a857 | ||
|
|
dfd668c206 | ||
|
|
e74d0a274d | ||
|
|
b570794515 | ||
|
|
a94cec3b36 | ||
|
|
f9b1295d14 |
@@ -6,6 +6,7 @@
|
||||
!README.md
|
||||
!LICENSE
|
||||
!MANIFEST.in
|
||||
!requirements-ci.txt
|
||||
!semantica/
|
||||
!semantica/**
|
||||
!integrations/
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
name: 'Setup Semantica'
|
||||
description: 'Install Python, cache pip, and install the semantica package into a workflow'
|
||||
author: 'Semantica'
|
||||
|
||||
inputs:
|
||||
python-version:
|
||||
description: 'Python version to set up'
|
||||
required: false
|
||||
default: '3.11'
|
||||
version:
|
||||
description: 'Version constraint to append to the pip spec, e.g. "==0.6.7" or ">=0.6,<0.7". Leave empty for the latest release.'
|
||||
required: false
|
||||
default: ''
|
||||
extras:
|
||||
description: 'Comma-separated extras to install, e.g. "explorer,all"'
|
||||
required: false
|
||||
default: ''
|
||||
cache:
|
||||
description: 'Pip cache mode passed straight to actions/setup-python ("pip" to enable). Left empty (disabled) by default because this action is meant to run standalone in any caller repo, and actions/setup-python errors out if it cannot find a requirements.txt/pyproject.toml/setup.py/poetry.lock to key the cache on. Opt in only when the caller repo has one of those files.'
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
outputs:
|
||||
version:
|
||||
description: 'The installed semantica version'
|
||||
value: ${{ steps.verify.outputs.version }}
|
||||
|
||||
runs:
|
||||
using: 'composite'
|
||||
steps:
|
||||
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
||||
with:
|
||||
python-version: ${{ inputs.python-version }}
|
||||
cache: ${{ inputs.cache }}
|
||||
|
||||
- name: Install semantica
|
||||
shell: bash
|
||||
env:
|
||||
SEMANTICA_EXTRAS: ${{ inputs.extras }}
|
||||
SEMANTICA_VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
if [ -n "$SEMANTICA_EXTRAS" ]; then
|
||||
spec="semantica[$SEMANTICA_EXTRAS]$SEMANTICA_VERSION"
|
||||
else
|
||||
spec="semantica$SEMANTICA_VERSION"
|
||||
fi
|
||||
python -m pip install -- "$spec"
|
||||
|
||||
- name: Verify install
|
||||
id: verify
|
||||
shell: bash
|
||||
run: |
|
||||
VERSION=$(python -c "import semantica; print(semantica.__version__)")
|
||||
echo "Installed semantica $VERSION"
|
||||
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
||||
@@ -101,6 +101,29 @@ updates:
|
||||
allow:
|
||||
- dependency-type: "production"
|
||||
|
||||
# Explorer frontend (npm)
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/explorer"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
day: "monday"
|
||||
time: "03:30" # 3:30 AM UTC (9:00 AM IST)
|
||||
open-pull-requests-limit: 10
|
||||
reviewers:
|
||||
- "KaifAhmad1"
|
||||
assignees:
|
||||
- "KaifAhmad1"
|
||||
commit-message:
|
||||
prefix: "security"
|
||||
include: "scope"
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "javascript"
|
||||
- "security"
|
||||
allow:
|
||||
- dependency-type: "production"
|
||||
- dependency-type: "development"
|
||||
|
||||
# Docker dependencies (if you use Docker)
|
||||
- package-ecosystem: "docker"
|
||||
directory: "/"
|
||||
|
||||
@@ -33,15 +33,29 @@ jobs:
|
||||
- name: Install Explorer frontend dependencies
|
||||
working-directory: explorer
|
||||
run: npm ci
|
||||
- name: Install Playwright Chromium
|
||||
working-directory: explorer
|
||||
run: npx playwright install --with-deps chromium
|
||||
- name: Test Explorer frontend
|
||||
working-directory: explorer
|
||||
run: |
|
||||
npm run test:graph-store
|
||||
npm run test:graph-workspace
|
||||
npm run test:plugin-registry
|
||||
npm run test:deterministic-e2e
|
||||
- name: Build Explorer frontend
|
||||
working-directory: explorer
|
||||
run: npm run build
|
||||
- name: Install Explorer backend test dependencies
|
||||
run: |
|
||||
# Run the deterministic backend path before the all-extras CI
|
||||
# environment is installed. The Explorer extra supplies the
|
||||
# production API dependencies without importing optional vector
|
||||
# providers such as Pinecone during test collection.
|
||||
pip install -e ".[explorer]" pytest==9.1.1
|
||||
- name: Test deterministic Explorer backend path
|
||||
run: |
|
||||
pytest -q tests/explorer/test_explorer_deterministic_rendering_e2e.py
|
||||
- name: Install pinned Python dependencies
|
||||
run: |
|
||||
pip install -r requirements-ci.txt
|
||||
@@ -58,7 +72,7 @@ jobs:
|
||||
diff \
|
||||
<(grep -E '^[a-zA-Z0-9._-]+==' requirements-ci.txt | sed 's/ \\$//') \
|
||||
<(grep -E '^[a-zA-Z0-9._-]+==' /tmp/requirements-ci-check.txt)
|
||||
- run: pip install build
|
||||
- run: pip install build==1.6.0
|
||||
# wheel is build-time only (not in requirements-ci.txt) — install the
|
||||
# same pinned version [build-system] declares so --no-isolation works.
|
||||
- run: pip install wheel==0.48.0
|
||||
|
||||
@@ -10,13 +10,15 @@ on:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
actions: read
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze Python
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write # for github/codeql-action/upload-sarif below
|
||||
actions: read # for github/codeql-action/init's CodeQL bundle cache lookup
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
@@ -32,7 +34,7 @@ jobs:
|
||||
# meaningful state carried over from a failed attempt.
|
||||
- name: Initialize CodeQL (attempt 1)
|
||||
id: codeql-init-1
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/init@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
|
||||
continue-on-error: true
|
||||
with:
|
||||
languages: python
|
||||
@@ -42,7 +44,7 @@ jobs:
|
||||
- name: Initialize CodeQL (attempt 2)
|
||||
id: codeql-init-2
|
||||
if: steps.codeql-init-1.outcome == 'failure'
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/init@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
|
||||
continue-on-error: true
|
||||
with:
|
||||
languages: python
|
||||
@@ -52,17 +54,17 @@ jobs:
|
||||
- name: Initialize CodeQL (attempt 3)
|
||||
id: codeql-init-3
|
||||
if: steps.codeql-init-2.outcome == 'failure'
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/init@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
|
||||
with:
|
||||
languages: python
|
||||
queries: security-and-quality
|
||||
config-file: .github/codeql/codeql-config.yml
|
||||
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/autobuild@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/analyze@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
|
||||
with:
|
||||
category: "/language:python"
|
||||
upload: false
|
||||
@@ -72,7 +74,7 @@ jobs:
|
||||
# Uploads results only when Default Setup is not active.
|
||||
# If Default Setup is still enabled, this step skips gracefully
|
||||
# instead of failing the workflow with HTTP 409.
|
||||
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
|
||||
with:
|
||||
sarif_file: ${{ steps.codeql.outputs.sarif-output }}
|
||||
category: "/language:python"
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
name: Container Security Scan
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# Mirrors .dockerignore's opt-in list exactly - anything not listed there
|
||||
# can't reach the build context, so it can't change the built image.
|
||||
paths:
|
||||
- 'Dockerfile'
|
||||
- '.dockerignore'
|
||||
- 'pyproject.toml'
|
||||
- 'README.md'
|
||||
- 'LICENSE'
|
||||
- 'MANIFEST.in'
|
||||
- 'requirements-ci.txt'
|
||||
- 'semantica/**'
|
||||
- 'integrations/**'
|
||||
- 'explorer/**'
|
||||
- '.github/workflows/container-scan.yml'
|
||||
schedule:
|
||||
- cron: '30 2 * * 1' # weekly, catches new CVEs published against the base image between pushes
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
scan:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write # for github/codeql-action/upload-sarif below
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
|
||||
- name: Build image
|
||||
run: docker build -t semantica:scan .
|
||||
|
||||
# Run Trivy as a digest-pinned image rather than the aquasecurity/trivy-action
|
||||
# marketplace wrapper: the aquasecurity GitHub org has an IP allow list on its
|
||||
# API that 403s verify-action-pins.sh's live tag->SHA check from Actions-runner
|
||||
# IPs, and this repo already treats Trivy's action pin as a known past target
|
||||
# for tag-repointing (see the LiteLLM/Trivy 2026 incident note above). Pulling
|
||||
# by sha256 digest from Docker Hub is immutable and verifiable independently of
|
||||
# GitHub's API, so it sidesteps both problems at once instead of carving a skip
|
||||
# exception into the pin verifier for an org already flagged as higher-risk.
|
||||
#
|
||||
# Report-only for now: this is Trivy's first run against this image, so we
|
||||
# don't yet know the CRITICAL/HIGH baseline. Findings still land in the
|
||||
# Security tab either way. Once triaged, add `--exit-code 1` (like
|
||||
# Safety/Bandit-HIGH in security-scan.yml) to make it a hard gate.
|
||||
- name: Scan image for vulnerabilities (Trivy)
|
||||
run: |
|
||||
docker run --rm \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||
-v "$PWD:/output" \
|
||||
aquasec/trivy@sha256:62b1e65e8869bc4b4c6aa4fa2b21595256c7c2f6018a9d9ad61caf87187c1969 \
|
||||
image --format sarif --output /output/trivy-results.sarif \
|
||||
--severity CRITICAL,HIGH --ignore-unfixed semantica:scan
|
||||
|
||||
- name: Upload Trivy SARIF
|
||||
if: always()
|
||||
uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
category: trivy-container
|
||||
|
||||
- name: Generate SBOM (Syft)
|
||||
if: always()
|
||||
uses: anchore/sbom-action@3ad7283483fc7af8ff2b4ea19663c2d5ca935e26 # v0.24.2
|
||||
with:
|
||||
image: semantica:scan
|
||||
format: spdx-json
|
||||
output-file: semantica-sbom.spdx.json
|
||||
@@ -28,12 +28,14 @@ on:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
|
||||
jobs:
|
||||
MSDO:
|
||||
# currently only windows-latest is supported
|
||||
runs-on: windows-latest
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write # for github/codeql-action/upload-sarif below
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
@@ -57,7 +59,7 @@ jobs:
|
||||
# avoiding the guardian.cmd/checkov exit-code bug in the MSDO wrapper.
|
||||
tools: eslint,templateanalyzer,terrascan
|
||||
- name: Upload results to Security tab
|
||||
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
|
||||
with:
|
||||
sarif_file: ${{ steps.msdo.outputs.sarifFile }}
|
||||
|
||||
@@ -82,7 +84,7 @@ jobs:
|
||||
}
|
||||
|
||||
- name: Upload Checkov results to Security tab
|
||||
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
|
||||
if: always()
|
||||
with:
|
||||
sarif_file: reports/checkov.sarif
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
name: Install Matrix
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 6 * * 1' # weekly, catches upstream dependency breakage between releases
|
||||
workflow_run:
|
||||
# The Release workflow publishes the GitHub release *before* it uploads to
|
||||
# PyPI (see release.yml), so triggering on `release: published` would race
|
||||
# the PyPI upload and could pass by silently installing the prior version.
|
||||
# workflow_run fires only after the whole Release workflow - including the
|
||||
# PyPI publish step - has finished.
|
||||
workflows: ['Release']
|
||||
types: [completed]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
verify-install:
|
||||
if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success'
|
||||
name: pip install semantica (${{ matrix.os }}, py${{ matrix.python-version }})
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest, windows-latest]
|
||||
python-version: ['3.9', '3.10', '3.11', '3.12']
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
|
||||
- name: Pin expected version for release-triggered runs
|
||||
id: expected-version
|
||||
if: github.event_name == 'workflow_run'
|
||||
shell: bash
|
||||
env:
|
||||
EXPECTED_TAG: ${{ github.event.workflow_run.head_branch }}
|
||||
run: |
|
||||
expected="${EXPECTED_TAG#v}"
|
||||
if [ -z "$expected" ]; then
|
||||
echo "::error::Could not determine a release tag from the triggering workflow run (head_branch was empty)."
|
||||
exit 1
|
||||
fi
|
||||
echo "constraint===$expected" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- id: setup-semantica
|
||||
uses: ./.github/actions/setup-semantica
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: 'pip'
|
||||
version: ${{ steps.expected-version.outputs.constraint }}
|
||||
|
||||
- name: Smoke test import
|
||||
shell: bash
|
||||
run: |
|
||||
python -c "
|
||||
import semantica
|
||||
print('semantica', semantica.__version__, 'installed and importable')
|
||||
"
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
cancel-in-progress: false
|
||||
permissions:
|
||||
contents: write # for the GitHub Release
|
||||
id-token: write # for PyPI Trusted Publishing (OIDC) and attestation signing
|
||||
id-token: write # for PyPI Trusted Publishing (OIDC), attestation signing, and Sigstore
|
||||
attestations: write # for SLSA build provenance
|
||||
# If you add another job to this workflow, give it its own explicit
|
||||
# `permissions:` block rather than relying on the workflow-level default
|
||||
@@ -40,7 +40,7 @@ jobs:
|
||||
# build runs against the same versions CI tests against.
|
||||
- name: Install pinned build dependencies
|
||||
run: pip install -r requirements-ci.txt
|
||||
- run: pip install build
|
||||
- run: pip install build==1.6.0
|
||||
# wheel is build-time only (not in requirements-ci.txt) — install the
|
||||
# same pinned version [build-system] declares so --no-isolation works.
|
||||
- run: pip install wheel==0.48.0
|
||||
@@ -63,11 +63,29 @@ jobs:
|
||||
|
||||
print("Explorer frontend is packaged")
|
||||
PY
|
||||
- name: Verify PyPI long-description will render
|
||||
run: |
|
||||
pip install twine==7.0.0
|
||||
twine check dist/*
|
||||
- name: Attest build provenance
|
||||
uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4
|
||||
with:
|
||||
subject-path: 'dist/*'
|
||||
- uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3
|
||||
# attest-build-provenance publishes to the GH attestations API only, which
|
||||
# OpenSSF Scorecard's Signed-Releases check does not inspect - it looks for
|
||||
# signature files attached as release assets. Sign here too so
|
||||
# `dist/*.sigstore.json` bundles ship alongside the wheel/sdist on the
|
||||
# GitHub Release itself.
|
||||
- name: Sign artifacts with Sigstore
|
||||
uses: sigstore/gh-action-sigstore-python@790bc6befb9d733738f18d8f895854b453640ec9 # v3.5.0
|
||||
with:
|
||||
files: dist/*
|
||||
inputs: |
|
||||
dist/*.whl
|
||||
dist/*.tar.gz
|
||||
- uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3
|
||||
with:
|
||||
files: |
|
||||
dist/*.whl
|
||||
dist/*.tar.gz
|
||||
dist/*.sigstore.json
|
||||
- uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
name: Scorecard supply-chain security
|
||||
|
||||
permissions: read-all
|
||||
|
||||
on:
|
||||
branch_protection_rule:
|
||||
schedule:
|
||||
- cron: '30 1 * * 6' # weekly
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
analysis:
|
||||
name: Scorecard analysis
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
security-events: write # to upload SARIF results
|
||||
id-token: write # to publish results and get a badge
|
||||
contents: read
|
||||
actions: read # to detect GitHub Actions workflows
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Run analysis
|
||||
uses: ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc # v2.4.4
|
||||
with:
|
||||
results_file: results.sarif
|
||||
results_format: sarif
|
||||
publish_results: true
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: SARIF file
|
||||
path: results.sarif
|
||||
retention-days: 5
|
||||
|
||||
- name: Upload to code-scanning
|
||||
uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
|
||||
with:
|
||||
sarif_file: results.sarif
|
||||
@@ -52,7 +52,7 @@ jobs:
|
||||
# Tooling AFTER the pinned set: installing safety/bandit/semgrep/jq
|
||||
# first lets the pinned requirements overwrite their transitive deps
|
||||
# (e.g. rich), which breaks the safety CLI at runtime.
|
||||
pip install safety bandit semgrep jq
|
||||
pip install safety==3.8.1 bandit==1.9.4 semgrep==1.175.0 jq==1.12.0
|
||||
|
||||
- name: Run Safety Check (Package Vulnerabilities)
|
||||
run: |
|
||||
|
||||
@@ -37,6 +37,6 @@ jobs:
|
||||
# pyproject.toml changes under review. The schedule/workflow_dispatch
|
||||
# runs stay non-blocking until a full pass over pre-existing findings
|
||||
# across the whole [all] tree has been done.
|
||||
- run: pip install pip-audit
|
||||
- run: pip install pip-audit==2.10.1
|
||||
- run: pip-audit -r requirements-ci.txt
|
||||
continue-on-error: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -213,6 +213,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Fixed
|
||||
|
||||
- **RETE engine matched every fact against every rule — `AlphaNode._matches()` and `BetaNode._can_join()` were placeholder stubs that always returned `True`** (closes #300)
|
||||
- `semantica/reasoning/rete_engine.py` shipped a Rete network whose per-condition alpha test and cross-condition beta join were both `return True` stubs, so `match_patterns()` fired every rule for every fact regardless of predicate, arity, or shared-variable consistency
|
||||
- New module-level `unify_condition()` reuses the regex-based approach from `Reasoner._match_pattern()`: a condition pattern like `Person(?x)` / `Parent(?x, ?y)` is compiled against a fact's `predicate(arg, ...)` string, `?var` becomes a named capture group, and a variable seen twice within one condition (e.g. `Loves(?x, ?x)`) becomes a backreference, so it only unifies when both positions hold the same value. Returns the bindings dict or `None`
|
||||
- Reworked propagation to carry partial-match **tokens** instead of bare facts: a new `Token` dataclass bundles the accumulated `facts` with the consistent `bindings`. `AlphaNode` emits a single-fact token per match; `BetaNode.join()` merges a left token with a right token, concatenating their facts in condition order and returning the merged token only when shared variables agree (conflicting values → `None`, no join). Terminal activations carry the full fact list and accumulated bindings through to the emitted match
|
||||
- This fixes a P1 chained-join defect: rules with three or more conditions (e.g. `Person(?x)`, `Parent(?x, ?y)`, `Located(?y, ?z)`) previously lost bindings and accumulated wrong facts at the third join, and a conflicting third condition could spuriously fire. Beta nodes now keep both `left_tokens` and `right_tokens` memories and join each new token against every token on the opposite side, so deep chains stay binding-consistent and third-level conflicts are correctly suppressed
|
||||
- Fixed an adjacent network-topology bug surfaced by the above: newly created beta nodes were never appended to their input nodes' `children`, so tokens could not propagate; propagation was reworked to support chained joins and to thread bindings end-to-end
|
||||
- Reconciled with the rule-actions/provenance layer (#1096) merged after this fix was opened: `execute_matches()` still dedupes and fires `Rule.actions`/legacy `handler` through a bound `Reasoner` via `_make_activation_key`, now sourced from the Token model's own `bindings` instead of the interim `_bindings_for_rule()` regex re-extraction, which is removed as redundant
|
||||
- New `tests/reasoning/test_rete_engine.py`: `unify_condition` unit cases (single/multi variable, literal args, predicate mismatch, repeated-variable equality), alpha match/reject, beta consistent-join vs conflict-reject, end-to-end rules (single-condition fires only the matching fact; multi-condition join fires only on consistent bindings), and a `TestThreeConditionChain` suite (valid three-condition match, third-level conflict suppression, insertion-order independence, `Match.facts` complete and in condition order, multiple left tokens joining one right fact, parity against `Reasoner._match_rule()`, and `reset()` clearing all token memory)
|
||||
|
||||
- **KG provenance tests asserted on generated ID strings instead of stored records, and `kg_provenance.py` was missed by the `utcnow` sweep** (closes #946) by @pravit-amp
|
||||
- The KG workflow and integration suites checked that a tracker call returned an ID matching a prefix (`assert cent_id.startswith("centrality_")`) without ever reading the record back, so an ID generator that returned a well-formed string and wrote nothing would have passed. Worse, some of those calls named tracker methods that do not exist anywhere in `semantica/` (`track_layer_analysis`, `track_centrality_score`), so the assertions were satisfied with no real interaction behind them
|
||||
- Those tests now read provenance back through `get_provenance()` and assert on algorithm metadata, and call the methods that actually persist records. Verified by mutation rather than by a green run alone: neutering the manager's storage write (`self.storage.store(...)` → no-op) fails 10 tests
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
cff-version: 1.2.0
|
||||
message: "If you use this software, please cite it as below."
|
||||
title: "Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems"
|
||||
type: software
|
||||
authors:
|
||||
- name: "Semantica"
|
||||
repository-code: "https://github.com/semantica-agi/semantica"
|
||||
url: "https://getsemantica.ai"
|
||||
license: MIT
|
||||
version: 0.6.7
|
||||
date-released: 2026-08-28
|
||||
keywords:
|
||||
- knowledge-graph
|
||||
- context-graph
|
||||
- ai-agents
|
||||
- llm
|
||||
- decision-intelligence
|
||||
- provenance
|
||||
- explainability
|
||||
- graph-rag
|
||||
+30
-4
@@ -1,5 +1,5 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
FROM node:26-alpine AS frontend-builder
|
||||
FROM node:26-alpine@sha256:2d984a15c9b54fd0aeb608b8e0d0d83529eb34d2966db27a1fb4f1edc3d298a3 AS frontend-builder
|
||||
|
||||
WORKDIR /app
|
||||
COPY explorer/package*.json ./explorer/
|
||||
@@ -9,7 +9,18 @@ RUN npm ci
|
||||
COPY explorer/ ./
|
||||
RUN mkdir -p /app/semantica && npm run build
|
||||
|
||||
FROM python:3.13-slim AS runtime
|
||||
# CVE-2026-14456 (OpenSSL QUIC-server DoS, flagged against this base image's
|
||||
# openssl/libssl3t64/openssl-provider-legacy): the Debian fix
|
||||
# (3.5.7-1~deb13u2) is only in trixie-proposed-updates as of this writing,
|
||||
# not yet promoted to trixie-security, so there's no package to pin here
|
||||
# today. Deliberately NOT running `apt-get upgrade` to chase it - that
|
||||
# breaks build reproducibility (terrascan AC_DOCKER_0052) and still
|
||||
# wouldn't reach a proposed-updates-only package. Once Debian ships the fix
|
||||
# and rebuilds this tag, the docker Dependabot ecosystem in
|
||||
# .github/dependabot.yml opens a PR bumping the digest pin above. Also: this
|
||||
# image only serves plain HTTP via uvicorn and never opens a QUIC listener,
|
||||
# so the bug isn't reachable here regardless.
|
||||
FROM python:3.13-slim@sha256:7ce4b6dfe35e55397b7cda544f8a13f191b7ae28dc5aad71fe664dbc9bc2623f AS runtime
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
@@ -22,12 +33,27 @@ WORKDIR /app
|
||||
RUN groupadd --system semantica \
|
||||
&& useradd --system --gid semantica --home-dir /app --shell /usr/sbin/nologin semantica
|
||||
|
||||
COPY pyproject.toml README.md LICENSE MANIFEST.in ./
|
||||
COPY pyproject.toml README.md LICENSE MANIFEST.in requirements-ci.txt ./
|
||||
COPY semantica/ ./semantica/
|
||||
COPY integrations/ ./integrations/
|
||||
COPY --from=frontend-builder /app/semantica/static ./semantica/static
|
||||
|
||||
RUN pip install --no-cache-dir ".[explorer]" \
|
||||
# The base image ships an outdated setuptools (CVE-2025-47273); upgrade it
|
||||
# explicitly since nothing in our own dependency tree otherwise pulls a
|
||||
# newer copy. Pinned to the exact version requirements-ci.txt/pyproject.toml
|
||||
# already build against, rather than a floor, per terrascan AC_DOCKER_0010.
|
||||
# requirements-ci.txt itself carries the audited, CVE-checked pins for every
|
||||
# transitive dependency (see security-scan.yml / security.yml) - feed them
|
||||
# in as an unhashed constraints file (pip's hash-checking mode rejects the
|
||||
# unhashable local source directory this installs) so the image lands on
|
||||
# the same patched versions CI verified, e.g. msgpack>=1.2.1, rather than
|
||||
# letting pip freely re-resolve and pick up an unpatched transitive version.
|
||||
# (Extracted with Python's re module rather than sed/grep so there's no
|
||||
# line-continuation-backslash stripping to get subtly wrong.)
|
||||
RUN pip install --no-cache-dir "setuptools==84.0.0" \
|
||||
&& python -c "import re, pathlib; pins = re.findall(r'^([A-Za-z0-9._-]+==\S+)', pathlib.Path('requirements-ci.txt').read_text(), re.M); pathlib.Path('/tmp/constraints.txt').write_text('\n'.join(pins))" \
|
||||
&& pip install --no-cache-dir -c /tmp/constraints.txt ".[explorer]" \
|
||||
&& rm -f /tmp/constraints.txt requirements-ci.txt \
|
||||
&& chown -R semantica:semantica /app
|
||||
|
||||
USER semantica
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
# Growth & Distribution Playbook
|
||||
|
||||
North star: **10,000 developers who actually use Semantica in real projects**, not a raw PyPI download number. Downloads are a lagging indicator of distribution, not a target to optimize directly.
|
||||
|
||||
```
|
||||
GitHub stars → Website visitors → PyPI installs → Weekly active users → Production deployments → Enterprise customers
|
||||
```
|
||||
The last two matter far more than the download count.
|
||||
|
||||
## Guardrails — do not do this
|
||||
|
||||
- No fake/looping CI jobs that repeatedly `pip install semantica` purely to inflate the graph. It's detectable, it produces zero real users, and it damages credibility with anyone doing diligence (investors, enterprise buyers, security reviewers).
|
||||
- No package-splitting purely to multiply install counts — only split into `semantica-*` packages when there's a real architectural reason.
|
||||
- No meaningless Docker pulls or notebook launches with no real content behind them.
|
||||
- Every item below should get someone from "installed it" to "used it for something real." If a channel can't do that, it's not worth building.
|
||||
|
||||
## 30-day priority sprint
|
||||
|
||||
Ordered by leverage-to-effort ratio; do these first.
|
||||
|
||||
| # | Initiative | Target |
|
||||
| - | ---------- | ------ |
|
||||
| 1 | ✅ GitHub Actions example + reusable `setup-semantica` composite action + install-matrix badge | done |
|
||||
| 2 | Google Colab notebooks | 10 |
|
||||
| 3 | Docker images (RAG, Graph, Agent, API) | 4-5 |
|
||||
| 4 | Hugging Face Spaces demos | 3-4 |
|
||||
| 5 | LangChain integration + example | 1 |
|
||||
| 6 | LlamaIndex integration + example | 1 |
|
||||
| 7 | Vector/graph DB integrations (Qdrant, Weaviate, Neo4j) | 3 |
|
||||
| 8 | MCP server + example | 1 (already have `mcp/` — package as a distributable example) |
|
||||
| 9 | Production-quality starter repos (FastAPI, Streamlit, Gradio) | 3 |
|
||||
| 10 | `awesome-rag` / `awesome-llm` / `awesome-knowledge-graph` list submissions | 3+ PRs |
|
||||
|
||||
Push everything through: GitHub → Discord (`sV34vps5hH`) → X (`@BuildSemantica`) → GitHub Discussions → Reddit → Hacker News → relevant newsletters.
|
||||
|
||||
## Full channel checklist
|
||||
|
||||
### CI/CD (highest-intent distribution — installs tied to real pipelines)
|
||||
|
||||
- [x] GitHub Actions example in `examples/ci/github-actions.yml`
|
||||
- [x] Reusable composite GitHub Action — [`.github/actions/setup-semantica`](.github/actions/setup-semantica/action.yml), modeled on `actions/setup-python`; usable by any repo as `uses: semantica-agi/semantica/.github/actions/setup-semantica@main`
|
||||
- [x] "pip install" status badge in the README, backed by [`.github/workflows/install-matrix.yml`](.github/workflows/install-matrix.yml) — verifies the *published* package installs cleanly on Ubuntu/macOS/Windows across Python 3.9-3.12, weekly + on every release
|
||||
- [x] GitLab CI template — `examples/ci/gitlab-ci.yml`
|
||||
- [x] CircleCI template — `examples/ci/circleci-config.yml`
|
||||
- [ ] Jenkins, Azure DevOps, Bitbucket Pipelines, Buildkite, Travis CI equivalents
|
||||
|
||||
### Release pipeline hardening (already had Trusted Publishing/OIDC + SLSA attestation — this rounds it out to match top-tier OSS release practice)
|
||||
|
||||
- [x] `twine check` gate in `.github/workflows/release.yml` before publish — catches a broken PyPI long-description render before it goes live instead of after (a malformed README on the live PyPI page is a silent conversion killer)
|
||||
- [x] `CITATION.cff` (see Academic & research below)
|
||||
- [x] OpenSSF Scorecard (see Discoverability below)
|
||||
- [ ] Considered and deliberately skipped: Release Drafter / auto-generated changelogs — this repo hand-curates `CHANGELOG.md` with far more detail (PR numbers, contributors, phase-1 limitations) than a bot would produce. Don't introduce this without checking with maintainers first.
|
||||
- [ ] Renovate / Dependabot config templates that auto-bump the `semantica` version in downstream repos — real recurring CI runs on real adopters
|
||||
- [ ] Nightly scheduled workflow template that tests a downstream project against `semantica@latest`
|
||||
|
||||
### Containers & dev environments
|
||||
|
||||
- [ ] Official Docker images: RAG, Graph, Agent, API, `+Postgres`, `+Neo4j`, `+Qdrant`
|
||||
- [ ] `docker-compose` examples (repo already has `docker-compose.dev.yml` / `docker-compose.yml` as a base)
|
||||
- [ ] `.devcontainer/devcontainer.json` for one-click "Reopen in Container"
|
||||
- [ ] GitHub Codespaces-ready config
|
||||
- [ ] Gitpod config
|
||||
- [ ] "Use this template" GitHub repo button so new projects start with `semantica` in `requirements.txt`
|
||||
|
||||
### Notebooks & hosted demos
|
||||
|
||||
- [ ] 10-20 Google Colab notebooks (Graph RAG, agent memory, entity resolution, semantic search, document intelligence)
|
||||
- [ ] Kaggle Notebooks/Kernels
|
||||
- [ ] Binder / mybinder.org config for instant repo launch
|
||||
- [ ] SageMaker Studio Lab / Databricks Community Edition / Paperspace Gradient examples
|
||||
- [ ] Hugging Face Spaces (Streamlit/Gradio) demos with `semantica` in `requirements.txt`
|
||||
- [ ] Public hosted playground (source on GitHub, install visible)
|
||||
|
||||
### Framework & data-store integrations
|
||||
|
||||
- [x] LangChain integration — `integrations/langchain/` (`SemanticaRetriever`, `SemanticaVectorStore`, `SemanticaKGTool`/`SemanticaDecisionTool`), `pip install semantica[langchain]`, shipped in 0.6.7
|
||||
- [ ] LlamaIndex integration + example
|
||||
- [ ] LangGraph example
|
||||
- [ ] Neo4j integration/example (docs already list it as a supported graph store — turn into a runnable example repo)
|
||||
- [ ] Vector DB examples: Qdrant, Weaviate, Milvus, Pinecone, Chroma, FAISS, pgvector, OpenSearch/Elasticsearch (FAISS/Pinecone/Weaviate/Qdrant/Milvus/PgVector already supported per `docs/community-projects.md` — package each as a standalone example)
|
||||
- [ ] LLM provider quickstarts: OpenAI, Anthropic, Gemini, Groq, Ollama, HuggingFace, DeepSeek, LiteLLM (already-supported providers per docs — each gets its own copy-paste quickstart)
|
||||
- [ ] CrewAI / Agno integration examples (already documented under `docs/integrations/`) — promote as standalone repos, not just docs pages
|
||||
|
||||
### Package managers & installers
|
||||
|
||||
- [ ] conda-forge feedstock
|
||||
- [ ] Homebrew formula for the CLI
|
||||
- [ ] Nix/nixpkgs packaging
|
||||
- [ ] Chocolatey / Scoop (Windows)
|
||||
- [ ] Document `uv add semantica` and `poetry add semantica` explicitly alongside `pip install`
|
||||
|
||||
### Downstream packages & CLI
|
||||
|
||||
- [ ] Genuinely useful `semantica-*` packages only where warranted (e.g. `semantica-rag`, `semantica-connectors`) — each pulls `semantica` as a real dependency
|
||||
- [ ] Make sure `semantica init / ingest / index / query / serve` CLI flows are the default onboarding path in every tutorial
|
||||
- [ ] VS Code extension wrapping the CLI (scaffold + run commands from the command palette)
|
||||
- [ ] JetBrains plugin equivalent
|
||||
|
||||
### Templates & starters
|
||||
|
||||
- [ ] Cookiecutter templates: `cookiecutter-semantic-rag`, `cookiecutter-ai-agent`, `cookiecutter-enterprise-rag`
|
||||
- [ ] Starter repos: FastAPI, Streamlit, Gradio, Next.js frontend + Semantica backend
|
||||
- [ ] Cloud deploy templates: AWS, GCP, Azure, Modal, Railway, Render, Fly.io (repo already has `deploy/azure`, `deploy/gcp`, `deploy/fly`, `deploy/railway`, `deploy/render`, `deploy/kubernetes`, `deploy/helm` — link these prominently from the README/quickstart, they're already-built distribution surface)
|
||||
- [ ] Terraform / Pulumi / Helm modules published to their respective registries
|
||||
|
||||
### Discoverability & curation
|
||||
|
||||
- [ ] Submit to `awesome-rag`, `awesome-llm`, `awesome-knowledge-graph`, `awesome-python`
|
||||
- [ ] Pitch newsletters with engaged Python/AI audiences (Python Weekly, Import AI, TLDR AI, etc.)
|
||||
- [x] PyPI trove classifiers/keywords and `project.urls` (Homepage/Docs/Repository/Changelog/Bug Tracker) — already complete in `pyproject.toml`
|
||||
- [ ] Get listed on Papers With Code for any retrieval/graph-RAG benchmark work
|
||||
- [x] [OpenSSF Scorecard](https://scorecard.dev/viewer/?uri=github.com/semantica-agi/semantica) badge + weekly workflow (`.github/workflows/scorecard.yml`) — a concrete trust signal security/procurement teams check before greenlighting adoption, which gates real (non-CI-bot) install growth at enterprises
|
||||
|
||||
### Academic & research
|
||||
|
||||
- [x] `CITATION.cff` at repo root — enables GitHub's native "Cite this repository" button, feeds Google Scholar/academic tooling; complements `docs/citation.md` (still needs a real Zenodo DOI to replace the `XXXXXXX` placeholder in both places once one is minted)
|
||||
- [ ] arXiv paper if there's real architectural novelty to describe
|
||||
- [ ] Zenodo DOI for citability (`docs/citation.md` already exists — make sure it points to a real DOI)
|
||||
- [ ] Workshop/tutorial sessions at PyData/ODSC-style events with hands-on install steps
|
||||
- [ ] University course material / bootcamp adoption outreach
|
||||
|
||||
### Content
|
||||
|
||||
- [ ] Reproducible benchmark repos (Graph RAG vs vector RAG, retrieval@k, enterprise-scale retrieval) with `pip install semantica && python benchmark.py`
|
||||
- [ ] 20-30 real-world example applications (RAG, enterprise document intelligence, financial entity graphs, code knowledge graphs, research discovery, agent memory)
|
||||
- [ ] Blog/tutorial posts on Dev.to, Medium, personal blogs — always with runnable code, not just prose
|
||||
- [ ] Contribute integrations/PRs to other projects building RAG/agents/knowledge graphs — "I implemented Semantica support" beats "please use Semantica"
|
||||
|
||||
## Tracking
|
||||
|
||||
Don't just watch the raw PyPI number — use download analytics (e.g. PePy) to separate CI/bot traffic from real installs, and track the funnel above end-to-end where possible (stars → site visits → installs → weekly actives).
|
||||
@@ -26,7 +26,7 @@
|
||||
|
||||
#### Built for High-Stakes, Regulated Domains
|
||||
|
||||
[](https://github.com/semantica-agi/semantica) [](https://github.com/semantica-agi/semantica/network/members) [](https://github.com/semantica-agi/semantica/graphs/contributors) [](https://pypi.org/project/semantica/) [](https://pepy.tech/project/semantica) [](https://www.python.org/) [](https://opensource.org/licenses/MIT) [](https://github.com/semantica-agi/semantica/actions) [](https://deepwiki.com/semantica-agi/semantica)
|
||||
[](https://github.com/semantica-agi/semantica) [](https://github.com/semantica-agi/semantica/network/members) [](https://github.com/semantica-agi/semantica/graphs/contributors) [](https://pypi.org/project/semantica/) [](https://pepy.tech/project/semantica) [](https://www.python.org/) [](https://opensource.org/licenses/MIT) [](https://github.com/semantica-agi/semantica/actions) [](https://github.com/semantica-agi/semantica/actions/workflows/install-matrix.yml) [](https://scorecard.dev/viewer/?uri=github.com/semantica-agi/semantica) [](https://deepwiki.com/semantica-agi/semantica)
|
||||
|
||||
[](https://getsemantica.ai/) [](https://docs.getsemantica.ai/) [](https://discord.gg/sV34vps5hH) [](https://x.com/BuildSemantica) [](https://www.youtube.com/watch?v=QfnNZg4-dZA) [](CHANGELOG.md)
|
||||
|
||||
@@ -1534,6 +1534,20 @@ git clone https://github.com/semantica-agi/semantica.git
|
||||
cd semantica && pip install -e ".[dev]" && pytest tests/
|
||||
```
|
||||
|
||||
### CI & Deployment
|
||||
|
||||
Wiring `semantica` into your own CI is a two-minute job. On GitHub Actions, use the reusable composite action:
|
||||
|
||||
```yaml
|
||||
- uses: semantica-agi/semantica/.github/actions/setup-semantica@main
|
||||
with:
|
||||
python-version: '3.11'
|
||||
```
|
||||
|
||||
Copy-paste starting templates for GitHub Actions, GitLab CI, and CircleCI live in [examples/ci/](examples/ci/). The published package itself is verified installable across Ubuntu/macOS/Windows and Python 3.9-3.12 every week by the [Install Matrix workflow](.github/workflows/install-matrix.yml).
|
||||
|
||||
Ready-made deployment configs for AWS, GCP, Azure, Fly.io, Railway, Render, Kubernetes, and Helm are in [deploy/](deploy/).
|
||||
|
||||
---
|
||||
|
||||
## Enterprise
|
||||
|
||||
+2
-1
@@ -106,7 +106,8 @@
|
||||
"integrations/langchain",
|
||||
"integrations/docling",
|
||||
"integrations/snowflake",
|
||||
"integrations/databricks"
|
||||
"integrations/databricks",
|
||||
"integrations/salesforce"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
+152
-24
@@ -28,6 +28,7 @@ The `semantica.llms` module provides a unified interface for connecting to Large
|
||||
## When To Use / When Not To Use
|
||||
|
||||
**Use LLM integrations for:**
|
||||
|
||||
- Text generation, summarization, and question-answering tasks
|
||||
- Complex reasoning that requires natural language understanding
|
||||
- Structured data extraction from unstructured text
|
||||
@@ -35,6 +36,7 @@ The `semantica.llms` module provides a unified interface for connecting to Large
|
||||
- Tasks where context, ambiguity, or domain knowledge matter
|
||||
|
||||
**Deterministic tools may be better for:**
|
||||
|
||||
- Pattern matching that regular expressions can handle
|
||||
- Simple rule-based classification with clear criteria
|
||||
- Mathematical calculations or statistical analysis
|
||||
@@ -42,6 +44,7 @@ The `semantica.llms` module provides a unified interface for connecting to Large
|
||||
- Data transformations with known logic
|
||||
|
||||
**A full LLM may be unnecessary for:**
|
||||
|
||||
- Simple keyword search or exact string matching
|
||||
- Deterministic workflows with predefined decision trees
|
||||
- High-frequency, low-latency operations where inference overhead matters
|
||||
@@ -59,7 +62,7 @@ Four factors drive provider selection, each optimized for different use cases:
|
||||
|
||||
**Accuracy** matters most in high-stakes decisions: clinical contraindication checks, credit committee reasoning, and legal document analysis. Frontier models like Claude or GPT-4 available through `LiteLLM` provide the strongest reasoning capabilities.
|
||||
|
||||
**Data residency** constraints eliminate cloud providers for classified or HIPAA-regulated workloads. `HuggingFaceLLM` with local model paths enables fully air-gapped deployments without network calls.
|
||||
**Data residency** constraints eliminate cloud providers for classified or HIPAA-regulated workloads. `HuggingFaceLLM` with local model paths, or `Ollama` pointed at a local server, both enable fully air-gapped deployments without network calls.
|
||||
|
||||
**Cost at scale** favors high-throughput providers like Novita AI for bulk extraction pipelines processing thousands of documents per hour where per-token costs accumulate quickly.
|
||||
|
||||
@@ -143,6 +146,131 @@ risk_data = oai.generate_structured(
|
||||
|
||||
The default model `gpt-3.5-turbo` is fine for classification and light extraction. Switch to `gpt-4o` for complex multi-step regulatory reasoning or document understanding.
|
||||
|
||||
## Anthropic — Complex Reasoning and Structured Extraction
|
||||
|
||||
**Anthropic** provides the Claude model family, built with an emphasis on careful, instruction-following behavior and strong performance on multi-step reasoning, long-document analysis, and code-related tasks. Claude models tend to be more cautious about ambiguous instructions than other providers. That matters when the cost of a confidently wrong answer is high.
|
||||
|
||||
The `Anthropic` provider wraps the Claude API. Reach for it when the task involves reasoning through several dependent steps (not just single-turn extraction), when you're processing long source documents that need to stay in context, or when you need schema-validated structured output rather than best-effort JSON.
|
||||
|
||||
Install with `pip install "semantica[llm-anthropic]"` (or just `pip install anthropic`) before using this provider.
|
||||
|
||||
```python
|
||||
from semantica.llms import Anthropic
|
||||
|
||||
claude = Anthropic(model="claude-sonnet-4-6", api_key="YOUR_ANTHROPIC_KEY")
|
||||
# api_key falls back to the ANTHROPIC_API_KEY environment variable
|
||||
|
||||
# is_available() only confirms a client was constructed from some key.
|
||||
# It does not validate the key or check network reachability - an
|
||||
# invalid or expired key still passes this check and fails at generate().
|
||||
if not claude.is_available():
|
||||
raise RuntimeError("Anthropic provider not configured - set ANTHROPIC_API_KEY")
|
||||
|
||||
# Plain generation - multi-step reasoning over a contract clause
|
||||
verdict = claude.generate(
|
||||
"A vendor contract has a 30-day termination-for-convenience clause "
|
||||
"but a 90-day data-return obligation that survives termination. "
|
||||
"If the customer terminates on day 1, when must vendor-held data "
|
||||
"be returned? Answer with the date basis only.",
|
||||
temperature=0.1,
|
||||
)
|
||||
print(verdict)
|
||||
# "Day 120 from termination notice. The 90-day return period runs from
|
||||
# the termination date (day 30), not from the notice date."
|
||||
|
||||
# Structured, schema-validated output
|
||||
from pydantic import BaseModel
|
||||
|
||||
class ContractRisk(BaseModel):
|
||||
clause: str
|
||||
risk_level: str
|
||||
days_to_deadline: int
|
||||
|
||||
risk = claude.generate_typed(
|
||||
"Extract the termination clause risk from: vendor contract, "
|
||||
"30-day termination for convenience, 90-day post-termination "
|
||||
"data return obligation.",
|
||||
schema=ContractRisk,
|
||||
)
|
||||
print(risk.risk_level, risk.days_to_deadline)
|
||||
# "medium" 90
|
||||
```
|
||||
|
||||
Model selection follows the same tier structure as the other providers: a Haiku model for high-volume classification where cost matters more than depth, a Sonnet model as the default for most extraction and reasoning tasks, an Opus model when a task genuinely needs the deepest reasoning available and latency/cost are secondary. Check Anthropic's docs for the current model identifiers, since they're versioned and change over time.
|
||||
|
||||
## Gemini — Long Context and Multimodal Input
|
||||
|
||||
**Gemini** is Google's model family, with a context window large enough to hold entire codebases or long regulatory filings in a single call, and native support for image and document input alongside text. Reach for it when a task needs to reference a large amount of source material at once, or when the input isn't plain text.
|
||||
|
||||
The `Gemini` provider tries the newer `google-genai` SDK first and falls back to the older `google-generativeai` package if that's what's installed. Install with `pip install "semantica[llm-gemini]"` (or `pip install google-genai`) before using this provider.
|
||||
|
||||
```python
|
||||
from semantica.llms import Gemini
|
||||
|
||||
gemini = Gemini(model="gemini-pro", api_key="YOUR_GEMINI_KEY")
|
||||
# api_key falls back to the GEMINI_API_KEY environment variable
|
||||
|
||||
if not gemini.is_available():
|
||||
raise RuntimeError("Gemini provider not configured - set GEMINI_API_KEY")
|
||||
|
||||
response = gemini.generate(
|
||||
"Summarize the key obligations in a standard NDA in three bullet points."
|
||||
)
|
||||
print(response)
|
||||
|
||||
data = gemini.generate_structured(
|
||||
"Extract the party names and effective date from: "
|
||||
"This Agreement is entered into between Acme Corp and Globex LLC, "
|
||||
"effective January 1, 2026."
|
||||
)
|
||||
print(data)
|
||||
```
|
||||
|
||||
## Ollama — Local, Air-Gapped Inference
|
||||
|
||||
**Ollama** runs models entirely on your own machine, with no API key and no outbound network call. It's the right choice for air-gapped environments, offline development, or any workload where the source data can't leave the local network.
|
||||
|
||||
Unlike the other providers here, `Ollama` takes a `base_url` instead of an `api_key`. It talks to a local Ollama server over HTTP. Start the server with `ollama serve` and pull a model with `ollama pull llama2` before using this provider. Install the Python client with `pip install "semantica[llm-ollama]"` (or `pip install ollama`).
|
||||
|
||||
```python
|
||||
from semantica.llms import Ollama
|
||||
|
||||
llm = Ollama(model="llama2", base_url="http://localhost:11434")
|
||||
|
||||
if not llm.is_available():
|
||||
raise RuntimeError("Ollama provider not configured - is 'ollama serve' running?")
|
||||
|
||||
response = llm.generate("Explain the difference between a hash map and a tree map.")
|
||||
print(response)
|
||||
```
|
||||
|
||||
`is_available()` for Ollama does a real connectivity check (it calls the server's `list()` endpoint), unlike the API-key-based providers above, so a `False` here usually means the server isn't running rather than a missing credential.
|
||||
|
||||
## DeepSeek — Budget Reasoning at Scale
|
||||
|
||||
**DeepSeek** exposes an OpenAI-compatible API at a fraction of the cost of the larger US providers, with reasoning quality that holds up well for extraction and classification work. It's a reasonable default when you're processing a large volume of documents and don't need the deepest reasoning tier.
|
||||
|
||||
Install with `pip install "semantica[llm-deepseek]"` (or `pip install openai`, since DeepSeek is accessed through the OpenAI client pointed at a different base URL).
|
||||
|
||||
```python
|
||||
from semantica.llms import DeepSeek
|
||||
|
||||
llm = DeepSeek(model="deepseek-chat", api_key="YOUR_DEEPSEEK_KEY")
|
||||
# api_key falls back to the DEEPSEEK_API_KEY environment variable
|
||||
|
||||
if not llm.is_available():
|
||||
raise RuntimeError("DeepSeek provider not configured - set DEEPSEEK_API_KEY")
|
||||
|
||||
response = llm.generate("List three risks of using a floating IP in a Kubernetes ingress.")
|
||||
print(response)
|
||||
|
||||
data = llm.generate_structured(
|
||||
"Extract the CVE ID and affected product from: "
|
||||
"CVE-2024-3400 affects PAN-OS GlobalProtect gateways."
|
||||
)
|
||||
print(data)
|
||||
```
|
||||
|
||||
## LiteLLM — One Interface, 100+ Providers
|
||||
|
||||
**LiteLLM** is a universal adapter that provides a single interface to over 100 different LLM providers, including Anthropic Claude, Azure OpenAI, AWS Bedrock, Google Vertex AI, and local Ollama instances. It acts as a translation layer, converting your unified API calls into provider-specific requests, enabling easy switching between providers without code changes.
|
||||
@@ -306,30 +434,32 @@ for t in triplets:
|
||||
|
||||
## Novita AI — Cost-Efficient Bulk Extraction
|
||||
|
||||
Novita AI exposes an OpenAI-compatible API and is available as a built-in provider for the extraction layer. It is accessed differently from the `semantica.llms` classes — through `create_provider` from `semantica.semantic_extract.providers` — making it the right choice for high-volume NER pipelines where per-call cost matters.
|
||||
**Novita AI** exposes an OpenAI-compatible API at low per-call cost, making it a reasonable choice for high-volume NER pipelines where cost matters more than getting the single best answer.
|
||||
|
||||
Install with `pip install "semantica[llm-novita]"` (or `pip install openai`, since Novita is accessed through the OpenAI client pointed at a different base URL).
|
||||
|
||||
```python
|
||||
from semantica.llms import Novita
|
||||
|
||||
llm = Novita(model="deepseek/deepseek-v3.2", api_key="YOUR_NOVITA_KEY")
|
||||
# api_key falls back to the NOVITA_API_KEY environment variable
|
||||
|
||||
if not llm.is_available():
|
||||
raise RuntimeError("Novita provider not configured - set NOVITA_API_KEY")
|
||||
|
||||
response = llm.generate("Summarize the Basel III leverage ratio requirement.")
|
||||
|
||||
data = llm.generate_structured(
|
||||
"Extract drug names and dosages from: "
|
||||
"Patient received warfarin 5mg daily, aspirin 75mg daily, metformin 500mg twice daily."
|
||||
)
|
||||
```
|
||||
|
||||
Novita is also reachable as a provider name string for the NER interface, without going through the `Novita` class directly:
|
||||
|
||||
```python
|
||||
from semantica.semantic_extract.providers import create_provider
|
||||
from semantica.semantic_extract import NamedEntityRecognizer
|
||||
|
||||
# create_provider pools instances — same key reuses the same object
|
||||
provider = create_provider(
|
||||
"novita",
|
||||
api_key="YOUR_NOVITA_KEY", # or set NOVITA_API_KEY env var
|
||||
model="deepseek/deepseek-v3.2", # default model
|
||||
)
|
||||
|
||||
if provider.is_available():
|
||||
# Plain generation
|
||||
response = provider.generate("Summarise the Basel III leverage ratio requirement.")
|
||||
|
||||
# Structured extraction — returns parsed dict
|
||||
data = provider.generate_structured(
|
||||
"Extract drug names and dosages from: "
|
||||
"Patient received warfarin 5mg daily, aspirin 75mg daily, metformin 500mg twice daily."
|
||||
)
|
||||
|
||||
# Use Novita through the NER interface — provider name as string
|
||||
ner = NamedEntityRecognizer(
|
||||
methods=["llm"],
|
||||
provider="novita",
|
||||
@@ -339,11 +469,9 @@ entities = ner.extract_entities(
|
||||
"CVE-2024-3400 is exploited by UNC3886 targeting PAN-OS GlobalProtect."
|
||||
)
|
||||
for e in entities:
|
||||
print("{} ({}) — conf={:.2f}".format(e.text, e.label, e.confidence))
|
||||
print("{} ({}) conf={:.2f}".format(e.text, e.label, e.confidence))
|
||||
```
|
||||
|
||||
Novita requires the `openai` Python client under the hood — install with `pip install "semantica[llm-openai]"` or `pip install openai`.
|
||||
|
||||
## Domain Examples
|
||||
|
||||
<Tabs>
|
||||
|
||||
@@ -0,0 +1,376 @@
|
||||
---
|
||||
title: "Salesforce Integration"
|
||||
description: "Ingest CRM records from Salesforce sObjects and SOQL queries into Semantica's KG pipeline."
|
||||
icon: "cloud"
|
||||
---
|
||||
|
||||
> Extract Accounts, Contacts, Opportunities, and custom objects from Salesforce into Semantica with username/password/security-token, JWT bearer, or session-based authentication.
|
||||
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
# Install with Salesforce support
|
||||
pip install "semantica[db-salesforce]"
|
||||
|
||||
# Or install the connector separately
|
||||
pip install simple-salesforce>=1.12.0
|
||||
```
|
||||
|
||||
|
||||
## Basic Usage
|
||||
|
||||
```python
|
||||
from semantica.ingest import SalesforceIngestor
|
||||
import os
|
||||
|
||||
ingestor = SalesforceIngestor(
|
||||
username=os.getenv("SALESFORCE_USERNAME"),
|
||||
password=os.getenv("SALESFORCE_PASSWORD"),
|
||||
security_token=os.getenv("SALESFORCE_SECURITY_TOKEN"),
|
||||
domain=os.getenv("SALESFORCE_DOMAIN", "login"), # "test" for sandbox
|
||||
)
|
||||
|
||||
data = ingestor.ingest_sobject("Account", fields=["Id", "Name", "Industry"], limit=1000)
|
||||
print(f"Retrieved {data.row_count} of {data.total_size} matching records")
|
||||
print(f"Columns: {data.columns}")
|
||||
```
|
||||
|
||||
<Tip>
|
||||
Use environment variables (or a `.env` file with `python-dotenv`) to keep credentials out of source code. `SalesforceIngestor()` with no arguments reads from `SALESFORCE_*` environment variables automatically.
|
||||
</Tip>
|
||||
|
||||
|
||||
## Authentication Methods
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Username / Password / Security Token">
|
||||
```python
|
||||
import os
|
||||
from semantica.ingest import SalesforceIngestor
|
||||
|
||||
ingestor = SalesforceIngestor(
|
||||
username=os.getenv("SALESFORCE_USERNAME"),
|
||||
password=os.getenv("SALESFORCE_PASSWORD"),
|
||||
security_token=os.getenv("SALESFORCE_SECURITY_TOKEN"),
|
||||
domain="login", # production; use "test" for sandbox
|
||||
)
|
||||
```
|
||||
Set the required environment variables before running:
|
||||
```bash
|
||||
export SALESFORCE_USERNAME="your-username@example.com"
|
||||
export SALESFORCE_PASSWORD="your-password"
|
||||
export SALESFORCE_SECURITY_TOKEN="your-security-token"
|
||||
```
|
||||
The standard server-side flow. The security token is appended to the
|
||||
password during Salesforce SOAP login. Generate or reset it under
|
||||
**Settings → My Personal Information → Reset My Security Token**.
|
||||
</Tab>
|
||||
<Tab title="JWT Bearer (Recommended for CI/CD)">
|
||||
```python
|
||||
import os
|
||||
from semantica.ingest import SalesforceIngestor
|
||||
|
||||
ingestor = SalesforceIngestor(
|
||||
username=os.getenv("SALESFORCE_USERNAME"),
|
||||
consumer_key=os.getenv("SALESFORCE_CONSUMER_KEY"),
|
||||
privatekey_file=os.getenv("SALESFORCE_PRIVATE_KEY_FILE"),
|
||||
domain="login", # or "test" for sandbox
|
||||
)
|
||||
```
|
||||
```bash
|
||||
export SALESFORCE_USERNAME="your-username@example.com"
|
||||
export SALESFORCE_CONSUMER_KEY="your-connected-app-consumer-key"
|
||||
export SALESFORCE_PRIVATE_KEY_FILE="/path/to/server.key"
|
||||
```
|
||||
The JWT bearer flow authenticates with a signed token — no password
|
||||
is transmitted. Ideal for server-to-server integrations and CI/CD
|
||||
pipelines. Requires a Salesforce connected app configured with
|
||||
**Use digital signatures** and the pre-authorised user listed under
|
||||
**Manage → Profiles / Permission Sets**.
|
||||
|
||||
If you prefer to pass the key material as a string instead of a file
|
||||
path, use `SALESFORCE_PRIVATE_KEY` (the PEM contents) in place of
|
||||
`SALESFORCE_PRIVATE_KEY_FILE`.
|
||||
</Tab>
|
||||
<Tab title="Session ID + Instance URL">
|
||||
```python
|
||||
ingestor = SalesforceIngestor(
|
||||
session_id=os.getenv("SALESFORCE_SESSION_ID"),
|
||||
instance_url=os.getenv("SALESFORCE_INSTANCE_URL"),
|
||||
)
|
||||
```
|
||||
Use this when your environment already manages the OAuth token
|
||||
lifecycle (e.g. a connected app obtaining tokens via the web-server
|
||||
or device flow). Pass the access token as `session_id` and the full
|
||||
instance URL (e.g. `https://myorg.my.salesforce.com`) as
|
||||
`instance_url`.
|
||||
</Tab>
|
||||
<Tab title="Sandbox">
|
||||
```python
|
||||
import os
|
||||
from semantica.ingest import SalesforceIngestor
|
||||
|
||||
ingestor = SalesforceIngestor(
|
||||
username=os.getenv("SALESFORCE_USERNAME"),
|
||||
password=os.getenv("SALESFORCE_PASSWORD"),
|
||||
security_token=os.getenv("SALESFORCE_SECURITY_TOKEN"),
|
||||
domain="test", # routes to test.salesforce.com
|
||||
)
|
||||
```
|
||||
```bash
|
||||
export SALESFORCE_USERNAME="your-sandbox-username@example.com.sandbox"
|
||||
export SALESFORCE_PASSWORD="your-password"
|
||||
export SALESFORCE_SECURITY_TOKEN="your-security-token"
|
||||
export SALESFORCE_DOMAIN="test"
|
||||
```
|
||||
Replace `domain="login"` with `domain="test"` (or set
|
||||
`SALESFORCE_DOMAIN=test` in your environment) to connect to a
|
||||
developer or full sandbox.
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
### Environment variables
|
||||
|
||||
All constructor parameters have environment-variable fallbacks:
|
||||
|
||||
| Variable | Parameter | Default |
|
||||
|---|---|---|
|
||||
| `SALESFORCE_USERNAME` | `username` | — |
|
||||
| `SALESFORCE_PASSWORD` | `password` | — |
|
||||
| `SALESFORCE_SECURITY_TOKEN` | `security_token` | — |
|
||||
| `SALESFORCE_DOMAIN` | `domain` | `"login"` |
|
||||
| `SALESFORCE_INSTANCE_URL` | `instance_url` | — |
|
||||
| `SALESFORCE_SESSION_ID` | `session_id` | — |
|
||||
| `SALESFORCE_CONSUMER_KEY` | `consumer_key` | — |
|
||||
| `SALESFORCE_PRIVATE_KEY_FILE` | `privatekey_file` | — |
|
||||
| `SALESFORCE_PRIVATE_KEY` | `privatekey` | — |
|
||||
| `SALESFORCE_API_VERSION` | `api_version` | library default (`59.0`) |
|
||||
|
||||
|
||||
## Object Ingestion
|
||||
|
||||
### Ingest a standard object
|
||||
|
||||
```python
|
||||
data = ingestor.ingest_sobject(
|
||||
"Account",
|
||||
fields=["Id", "Name", "Industry", "AnnualRevenue", "BillingCity"],
|
||||
where="Type = 'Customer' AND AnnualRevenue > 1000000",
|
||||
order_by="Name ASC",
|
||||
limit=5000,
|
||||
)
|
||||
print(f"Retrieved {data.row_count} of {data.total_size} matching records")
|
||||
```
|
||||
|
||||
<Note>
|
||||
`data.row_count` is the number of records in `data.data` (i.e. what was actually returned after any `limit`). `data.total_size` is Salesforce's `totalSize` — the number of records matching the query *before* the limit. Compare them to know whether you got all results.
|
||||
</Note>
|
||||
|
||||
### Ingest a custom object
|
||||
|
||||
Custom objects end with `__c` in their API name:
|
||||
|
||||
```python
|
||||
data = ingestor.ingest_sobject(
|
||||
"My_Custom_Object__c",
|
||||
fields=["Id", "Name", "Custom_Field__c"],
|
||||
)
|
||||
```
|
||||
|
||||
Relationship traversal fields (`Owner.Name`) are also supported:
|
||||
|
||||
```python
|
||||
data = ingestor.ingest_sobject(
|
||||
"Contact",
|
||||
fields=["Id", "Name", "Email", "Account.Name", "Owner.Name"],
|
||||
limit=10000,
|
||||
)
|
||||
```
|
||||
|
||||
### Let Semantica choose the fields
|
||||
|
||||
When `fields` is omitted, all selectable fields are fetched via `describe()`
|
||||
(one extra API call). Compound address and geolocation fields (`type=address`,
|
||||
`type=location`) are automatically excluded — select their components
|
||||
(`BillingStreet`, `BillingCity`, `Location__Latitude__s`, …) individually if
|
||||
you need them.
|
||||
|
||||
```python
|
||||
data = ingestor.ingest_sobject("Opportunity")
|
||||
```
|
||||
|
||||
|
||||
## Raw SOQL Ingestion
|
||||
|
||||
Pass any valid SOQL query verbatim — pagination is handled automatically:
|
||||
|
||||
```python
|
||||
data = ingestor.ingest_query("""
|
||||
SELECT Id, Name, StageName, Amount, CloseDate,
|
||||
Account.Name, Owner.Name
|
||||
FROM Opportunity
|
||||
WHERE IsClosed = false
|
||||
ORDER BY CloseDate ASC
|
||||
""")
|
||||
print(f"Open opportunities: {data.row_count}")
|
||||
```
|
||||
|
||||
The query is passed to the Salesforce REST API unchanged. The caller is
|
||||
responsible for SOQL correctness and safety.
|
||||
|
||||
<Warning>
|
||||
`ingest_query` does not validate or sanitise the SOQL string. Use
|
||||
`ingest_sobject` (which validates sObject names, field names, and WHERE/ORDER
|
||||
BY fragments) when building queries from application-controlled inputs.
|
||||
</Warning>
|
||||
|
||||
|
||||
## Document Export
|
||||
|
||||
Convert ingested records to the Semantica document format for use with
|
||||
`GraphBuilder`:
|
||||
|
||||
```python
|
||||
documents = ingestor.export_as_documents(
|
||||
data,
|
||||
id_field="Id", # default; Salesforce 18-char record Id
|
||||
text_fields=["Name", "Description"], # omit to join all string fields
|
||||
)
|
||||
|
||||
print(f"Created {len(documents)} documents")
|
||||
# Each document:
|
||||
# {
|
||||
# "id": "001xx000003GYk2AAG",
|
||||
# "text": "Acme Corp Enterprise software company",
|
||||
# "metadata": {
|
||||
# "source": "salesforce",
|
||||
# "sobject": "Account",
|
||||
# "instance_url": "https://myorg.my.salesforce.com",
|
||||
# "row_data": { ... full cleaned record ... }
|
||||
# }
|
||||
# }
|
||||
```
|
||||
|
||||
Feed the documents directly into `GraphBuilder`:
|
||||
|
||||
```python
|
||||
from semantica.kg import GraphBuilder
|
||||
|
||||
builder = GraphBuilder()
|
||||
kg = builder.build(documents)
|
||||
```
|
||||
|
||||
|
||||
## Object and Schema Discovery
|
||||
|
||||
```python
|
||||
# List all accessible sObjects
|
||||
sobject_names = ingestor.list_sobjects()
|
||||
print(sobject_names[:10]) # ["Account", "Case", "Contact", ...]
|
||||
|
||||
# Inspect fields for a specific sObject
|
||||
schema = ingestor.get_sobject_schema("Account")
|
||||
for field in schema["fields"]:
|
||||
print(f"{field['name']}: {field['type']} (nillable={field['nillable']})")
|
||||
```
|
||||
|
||||
|
||||
## Context Manager
|
||||
|
||||
Prefer the context manager for long-running jobs — it opens one connection on
|
||||
entry and closes it on exit, so every ingestion call inside the `with` block
|
||||
reuses the same authenticated session:
|
||||
|
||||
```python
|
||||
with SalesforceIngestor(
|
||||
username=os.getenv("SALESFORCE_USERNAME"),
|
||||
password=os.getenv("SALESFORCE_PASSWORD"),
|
||||
security_token=os.getenv("SALESFORCE_SECURITY_TOKEN"),
|
||||
) as sf:
|
||||
accounts = sf.ingest_sobject("Account", limit=10000)
|
||||
contacts = sf.ingest_sobject("Contact", limit=10000)
|
||||
sobjects = sf.list_sobjects()
|
||||
```
|
||||
|
||||
|
||||
## Convenience Function
|
||||
|
||||
Use `ingest_salesforce()` for one-liner ingestion:
|
||||
|
||||
```python
|
||||
from semantica.ingest import ingest_salesforce
|
||||
|
||||
# Fetch records
|
||||
data = ingest_salesforce(
|
||||
method="sobject",
|
||||
sobject_name="Account",
|
||||
fields=["Id", "Name", "Industry"],
|
||||
limit=500,
|
||||
)
|
||||
|
||||
# Execute raw SOQL (credentials from environment variables)
|
||||
data = ingest_salesforce(
|
||||
method="query",
|
||||
soql="SELECT Id, Name FROM Contact WHERE IsActive = true",
|
||||
)
|
||||
|
||||
# Ingest + export to documents in one step
|
||||
docs = ingest_salesforce(
|
||||
method="documents",
|
||||
sobject_name="Account",
|
||||
text_fields=["Name", "Description"],
|
||||
limit=1000,
|
||||
)
|
||||
|
||||
# List accessible sObjects
|
||||
sobject_names = ingest_salesforce(method="list_sobjects")
|
||||
```
|
||||
|
||||
Or use the unified `ingest()` dispatcher:
|
||||
|
||||
```python
|
||||
from semantica.ingest import ingest
|
||||
|
||||
result = ingest(
|
||||
None,
|
||||
source_type="salesforce",
|
||||
method="sobject",
|
||||
sobject_name="Account",
|
||||
fields=["Id", "Name"],
|
||||
limit=500,
|
||||
)
|
||||
data = result["data"] # SalesforceData
|
||||
```
|
||||
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
```python
|
||||
import os
|
||||
from semantica.ingest import SalesforceConnector
|
||||
|
||||
connector = SalesforceConnector(
|
||||
username=os.getenv("SALESFORCE_USERNAME"),
|
||||
password=os.getenv("SALESFORCE_PASSWORD"),
|
||||
security_token=os.getenv("SALESFORCE_SECURITY_TOKEN"),
|
||||
)
|
||||
if not connector.test_connection():
|
||||
print("Connection failed: check username, password, security token, and domain")
|
||||
```
|
||||
|
||||
Common causes of authentication failures:
|
||||
|
||||
- **Wrong domain**: production orgs use `domain="login"`; sandboxes use `domain="test"`.
|
||||
- **Stale security token**: reset it under **Settings → Reset My Security Token**. The new token is emailed to you.
|
||||
- **IP restriction**: your org's trusted IP ranges may block the originating IP. Check **Setup → Network Access**.
|
||||
- **API access disabled**: ensure the connected profile has the **API Enabled** permission.
|
||||
|
||||
|
||||
## See Also
|
||||
|
||||
- [Ingest Module](../reference/ingest) — Full `SalesforceIngestor` API and all other ingestors.
|
||||
- [Snowflake Integration](snowflake) — Relational warehouse connector with a similar design.
|
||||
- [Databricks Integration](databricks) — Lakehouse connector.
|
||||
- [Installation](../installation) — All optional dependency extras.
|
||||
- [Knowledge Graph](../reference/kg) — Build a KG from ingested Salesforce data.
|
||||
@@ -0,0 +1,36 @@
|
||||
# CI templates
|
||||
|
||||
Copy-paste starting points for wiring `semantica` into your own project's CI. Each file is a
|
||||
complete, working config — rename it into your project (see the comment at the top of each file
|
||||
for the target path) and swap the smoke-test / test step for whatever your project does with
|
||||
Semantica. Each template installs `semantica` unconditionally and your own project's dependencies
|
||||
only if a `requirements.txt` is present; if your project uses `pyproject.toml`, Poetry, or Pipenv
|
||||
instead, adjust the marked install line (each file calls it out inline).
|
||||
|
||||
| File | Target path in your repo |
|
||||
| ---- | ------------------------- |
|
||||
| [`github-actions.yml`](github-actions.yml) | `.github/workflows/semantica.yml` |
|
||||
| [`gitlab-ci.yml`](gitlab-ci.yml) | `.gitlab-ci.yml` |
|
||||
| [`circleci-config.yml`](circleci-config.yml) | `.circleci/config.yml` |
|
||||
|
||||
If your own project is hosted on GitHub, you can skip the setup boilerplate entirely and use
|
||||
Semantica's reusable composite action instead:
|
||||
|
||||
```yaml
|
||||
- uses: semantica-agi/semantica/.github/actions/setup-semantica@main
|
||||
with:
|
||||
python-version: '3.11'
|
||||
# extras: 'explorer,all' # optional
|
||||
# version: '==0.6.7' # optional, pin an exact release
|
||||
# cache: 'pip' # optional, only if your repo has a requirements.txt/pyproject.toml/etc.
|
||||
```
|
||||
|
||||
`@main` always tracks this repo's default branch, which is convenient but — like any mutable
|
||||
ref — can change out from under you between runs. For production CI, pin it to a commit SHA
|
||||
instead (find one via `git rev-parse` against a tagged release, or the commit history for
|
||||
[`.github/actions/setup-semantica/`](../../.github/actions/setup-semantica/)) and update the pin
|
||||
deliberately when you want to pick up changes, the same way this repo's own workflows are pinned
|
||||
(see [`verify-action-pins.yml`](../../.github/workflows/verify-action-pins.yml)).
|
||||
|
||||
It installs Python, installs `semantica`, and verifies the import (pip caching is opt-in via `cache: 'pip'`, since not every caller repo has a requirements file to key the cache on) — see
|
||||
[`.github/actions/setup-semantica/action.yml`](../../.github/actions/setup-semantica/action.yml).
|
||||
@@ -0,0 +1,40 @@
|
||||
# Drop this in as .circleci/config.yml in your own project.
|
||||
version: 2.1
|
||||
|
||||
jobs:
|
||||
test:
|
||||
docker:
|
||||
- image: cimg/python:3.11
|
||||
steps:
|
||||
- checkout
|
||||
# A content-hashed cache key (e.g. `{{ checksum "requirements.txt" }}`)
|
||||
# is more precise but breaks if that exact file doesn't exist in your
|
||||
# project - swap in one matched to however you declare dependencies
|
||||
# once you've adjusted the install step below.
|
||||
- restore_cache:
|
||||
keys:
|
||||
- pip-cache-v1
|
||||
- run:
|
||||
name: Install dependencies
|
||||
command: |
|
||||
pip install --upgrade pip
|
||||
pip install semantica
|
||||
# Install your own project's dependencies however your project
|
||||
# declares them - adjust this to match, e.g. `pip install -e .`
|
||||
# for pyproject.toml / setup.cfg, or `poetry install`.
|
||||
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
|
||||
- save_cache:
|
||||
key: pip-cache-v1
|
||||
paths:
|
||||
- ~/.cache/pip
|
||||
- run:
|
||||
name: Smoke test
|
||||
command: python -c "import semantica; print('semantica', semantica.__version__)"
|
||||
- run:
|
||||
name: Run tests
|
||||
command: pytest
|
||||
|
||||
workflows:
|
||||
test:
|
||||
jobs:
|
||||
- test
|
||||
@@ -0,0 +1,44 @@
|
||||
# Drop this in as .github/workflows/semantica.yml in your own project.
|
||||
#
|
||||
# Installs Semantica and runs a smoke import + your test suite. Swap the
|
||||
# smoke-test step for whatever your project actually does with Semantica
|
||||
# (build a context graph, run an ingest pipeline, etc.).
|
||||
#
|
||||
# Third-party actions below are pinned to a commit SHA rather than a mutable
|
||||
# tag - a moved tag can silently swap in different code. Update the pin (and
|
||||
# the trailing "# vX" comment) deliberately when you want a newer version;
|
||||
# see semantica-agi/semantica's own .github/workflows/verify-action-pins.yml
|
||||
# for one way to keep pins honest automatically.
|
||||
name: Semantica
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
|
||||
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install semantica
|
||||
# Install your own project's dependencies however your project
|
||||
# declares them - adjust this to match. Examples:
|
||||
# pip install -r requirements.txt
|
||||
# pip install -e . # pyproject.toml / setup.cfg
|
||||
# pip install -e ".[dev]"
|
||||
# poetry install
|
||||
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
|
||||
|
||||
- name: Run tests
|
||||
run: pytest
|
||||
@@ -0,0 +1,20 @@
|
||||
# Drop this in as .gitlab-ci.yml in your own project.
|
||||
semantica-test:
|
||||
image: python:3.11-slim
|
||||
cache:
|
||||
paths:
|
||||
- .cache/pip
|
||||
variables:
|
||||
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
|
||||
script:
|
||||
- pip install --upgrade pip
|
||||
- pip install semantica
|
||||
# Install your own project's dependencies however your project declares
|
||||
# them - adjust this to match, e.g. `pip install -e .` for pyproject.toml
|
||||
# / setup.cfg, or `poetry install`.
|
||||
- if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
|
||||
- python -c "import semantica; print('semantica', semantica.__version__)"
|
||||
- pytest
|
||||
rules:
|
||||
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
|
||||
- if: '$CI_COMMIT_BRANCH == "main"'
|
||||
@@ -0,0 +1,195 @@
|
||||
"""
|
||||
Deterministic Explorer Rendering E2E Example.
|
||||
|
||||
Demonstrates building, serializing, and reloading a deterministic 4-node,
|
||||
3-edge knowledge graph baseline for visual inspection in Semantica Explorer (#1037).
|
||||
|
||||
Graph topology:
|
||||
Alice (Person, #63E6FF) --WORKS_AT--> Acme (Organization, #A78BFA)
|
||||
Bob (Person, #63E6FF) --KNOWS--> Alice (Person, #63E6FF)
|
||||
Acme (Organization, #A78BFA) --LOCATED_IN--> New York (Location, #34D399)
|
||||
|
||||
Clean Checkout Prerequisites:
|
||||
1. Python backend dependencies:
|
||||
pip install -e ".[explorer]"
|
||||
2. Frontend workspace dependencies:
|
||||
cd explorer && npm install && cd ..
|
||||
|
||||
Usage:
|
||||
# 1. Generate the deterministic graph baseline:
|
||||
python examples/explorer_deterministic_rendering_example.py
|
||||
|
||||
# 2. Launch Explorer with local dev authentication (Option A - Dev mode):
|
||||
# Terminal 1 (Backend API):
|
||||
SEMANTICA_ALLOW_ANONYMOUS=true python -m semantica.explorer --graph explorer_e2e_test_graph.json --port 8000 --no-browser
|
||||
# Terminal 2 (Frontend UI):
|
||||
cd explorer && npm run dev
|
||||
# Open http://localhost:5173
|
||||
|
||||
# 2. Launch Explorer (Option B - Standalone CLI server):
|
||||
SEMANTICA_ALLOW_ANONYMOUS=true python -m semantica.explorer --graph explorer_e2e_test_graph.json --port 8000
|
||||
# Open http://localhost:8000
|
||||
|
||||
# Secure authentication alternative:
|
||||
export SEMANTICA_API_KEY="your-secret-api-key"
|
||||
python -m semantica.explorer --graph explorer_e2e_test_graph.json --port 8000
|
||||
# Send HTTP header: X-API-Key: your-secret-api-key
|
||||
|
||||
Verification Checklist:
|
||||
- Exactly 4 nodes visible on canvas:
|
||||
* Alice (Person, #63E6FF)
|
||||
* Bob (Person, #63E6FF)
|
||||
* Acme (Organization, #A78BFA)
|
||||
* New York (Location, #34D399)
|
||||
- Exactly 3 directed edges with canonical relationship labels:
|
||||
* Alice -> Acme (WORKS_AT)
|
||||
* Bob -> Alice (KNOWS)
|
||||
* Acme -> New York (LOCATED_IN)
|
||||
- Zoom behavior:
|
||||
* Zoom in to Inspection tier (ratio <= 0.5): directional arrows and node labels scale clearly.
|
||||
* Zoom out to Overview tier (ratio > 1.2): layout remains stable and non-colliding.
|
||||
- Hover & Selection interactions:
|
||||
* Hover over 'Alice': node halo triggers; incident edges (WORKS_AT, KNOWS) highlight in local context.
|
||||
* Click an edge: Inspector panel confirms edgeType ('WORKS_AT', 'KNOWS', or 'LOCATED_IN').
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
from semantica.explorer.session import GraphSession
|
||||
|
||||
|
||||
def build_deterministic_graph() -> ContextGraph:
|
||||
"""Build the exact 4-node, 3-edge graph specified in #1037."""
|
||||
graph = ContextGraph(advanced_analytics=False)
|
||||
|
||||
# 1. Add exactly 4 nodes
|
||||
graph.add_node(
|
||||
"alice",
|
||||
node_type="Person",
|
||||
content="Alice",
|
||||
color="#63E6FF",
|
||||
)
|
||||
graph.add_node(
|
||||
"bob",
|
||||
node_type="Person",
|
||||
content="Bob",
|
||||
color="#63E6FF",
|
||||
)
|
||||
graph.add_node(
|
||||
"acme",
|
||||
node_type="Organization",
|
||||
content="Acme",
|
||||
color="#A78BFA",
|
||||
)
|
||||
graph.add_node(
|
||||
"new_york",
|
||||
node_type="Location",
|
||||
content="New York",
|
||||
color="#34D399",
|
||||
)
|
||||
|
||||
# 2. Add exactly 3 directed edges
|
||||
graph.add_edge("alice", "acme", edge_type="WORKS_AT", weight=1.0)
|
||||
graph.add_edge("bob", "alice", edge_type="KNOWS", weight=1.0)
|
||||
graph.add_edge("acme", "new_york", edge_type="LOCATED_IN", weight=1.0)
|
||||
|
||||
return graph
|
||||
|
||||
|
||||
def main() -> None:
|
||||
print("=" * 75)
|
||||
print("Semantica Explorer Deterministic Graph Generator (#1037)")
|
||||
print("=" * 75)
|
||||
|
||||
print("1. Building deterministic ContextGraph...")
|
||||
graph = build_deterministic_graph()
|
||||
print(
|
||||
f" ✓ Graph built with {len(graph.nodes)} nodes "
|
||||
f"and {len(graph.edges)} edges."
|
||||
)
|
||||
|
||||
output_path = Path("explorer_e2e_test_graph.json").resolve()
|
||||
print(f"2. Persisting graph to '{output_path.name}'...")
|
||||
graph.save_to_file(str(output_path))
|
||||
print(f" ✓ Graph saved to {output_path}")
|
||||
|
||||
# Verify JSON format
|
||||
with open(output_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
assert len(data.get("nodes", [])) == 4
|
||||
assert len(data.get("edges", [])) == 3
|
||||
|
||||
print("3. Verifying reload via GraphSession.from_file()...")
|
||||
session = GraphSession.from_file(str(output_path))
|
||||
stats = session.get_stats()
|
||||
nodes, total_nodes = session.get_nodes()
|
||||
edges, total_edges = session.get_edges()
|
||||
|
||||
assert stats["node_count"] == 4
|
||||
assert stats["edge_count"] == 3
|
||||
assert total_nodes == 4
|
||||
assert total_edges == 3
|
||||
|
||||
print(
|
||||
f" ✓ Graph reloaded successfully without mutation "
|
||||
f"(nodes: {total_nodes}, edges: {total_edges}).\n"
|
||||
)
|
||||
|
||||
print("=" * 75)
|
||||
print("Clean Checkout Prerequisites:")
|
||||
print("=" * 75)
|
||||
print(" pip install -e '.[explorer]'")
|
||||
print(" cd explorer && npm install && cd ..\n")
|
||||
|
||||
print("=" * 75)
|
||||
print("Reproduction instructions to view in Semantica Explorer:")
|
||||
print("=" * 75)
|
||||
print("Option A (Frontend dev server + API backend — recommended for development):")
|
||||
print(
|
||||
f" 1. Backend: SEMANTICA_ALLOW_ANONYMOUS=true python -m semantica.explorer "
|
||||
f"--graph {output_path} --port 8000 --no-browser"
|
||||
)
|
||||
print(" 2. Frontend: cd explorer && npm run dev")
|
||||
print(" 3. Open http://localhost:5173 to inspect the graph canvas.\n")
|
||||
|
||||
print("Option B (Standalone Explorer CLI server):")
|
||||
print(
|
||||
f" SEMANTICA_ALLOW_ANONYMOUS=true python -m semantica.explorer "
|
||||
f"--graph {output_path} --port 8000"
|
||||
)
|
||||
print(" Open http://localhost:8000\n")
|
||||
|
||||
print("Secure Authentication Alternative:")
|
||||
print(" export SEMANTICA_API_KEY='your-secret-api-key'")
|
||||
print(
|
||||
f" python -m semantica.explorer --graph {output_path} --port 8000"
|
||||
)
|
||||
print(" Send header: 'X-API-Key: your-secret-api-key'\n")
|
||||
|
||||
print("=" * 75)
|
||||
print("Verification Checklist:")
|
||||
print("=" * 75)
|
||||
print(" 1. Nodes (4 total):")
|
||||
print(" - Alice (Person, #63E6FF)")
|
||||
print(" - Bob (Person, #63E6FF)")
|
||||
print(" - Acme (Organization, #A78BFA)")
|
||||
print(" - New York (Location, #34D399)")
|
||||
print(" 2. Directed Edges & Canonical Labels (3 total):")
|
||||
print(" - Alice -> Acme [WORKS_AT]")
|
||||
print(" - Bob -> Alice [KNOWS]")
|
||||
print(" - Acme -> New York [LOCATED_IN]")
|
||||
print(" 3. Zoom Interactions:")
|
||||
print(" - Inspection tier (zoom in): directional arrows & labels remain legible.")
|
||||
print(" - Overview tier (zoom out): nodes and edges maintain layout integrity.")
|
||||
print(" 4. Hover & Selection Interactions:")
|
||||
print(" - Hover Alice: node halo triggers and incident edges (WORKS_AT, KNOWS) highlight.")
|
||||
print(" - Click edge: Inspector panel displays edgeType label ('WORKS_AT', 'KNOWS', 'LOCATED_IN').")
|
||||
print("=" * 75)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Generated
+6
-6
@@ -2083,9 +2083,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/brace-expansion": {
|
||||
"version": "5.0.8",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.8.tgz",
|
||||
"integrity": "sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg==",
|
||||
"version": "5.0.9",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz",
|
||||
"integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -4250,9 +4250,9 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/nanoid": {
|
||||
"version": "3.3.16",
|
||||
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz",
|
||||
"integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==",
|
||||
"version": "3.3.18",
|
||||
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz",
|
||||
"integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
|
||||
@@ -9,7 +9,8 @@
|
||||
"lint": "eslint .",
|
||||
"preview": "vite preview",
|
||||
"test:graph-store": "node --test tests/graphStore.multi-edge.test.mjs",
|
||||
"test:graph-workspace": "node --import tsx --test tests/markdownContentViewer.test.ts tests/graphSceneState.display.test.ts tests/temporalLifecycle.test.ts",
|
||||
"test:graph-workspace": "node --import tsx --test tests/markdownContentViewer.test.ts tests/graphSceneState.display.test.ts tests/temporalLifecycle.test.ts tests/deterministicExplorerRendering.test.ts",
|
||||
"test:deterministic-e2e": "node --import tsx --test tests/deterministicExplorerRendering.e2e.ts",
|
||||
"test:plugin-registry": "node --import tsx --test tests/pluginRegistry.temporal.test.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
|
||||
@@ -41,6 +41,7 @@ import {
|
||||
} from "./plugins";
|
||||
import { explorationEffectsShouldLoad, neighborhoodPanelShouldLoad, temporalOverlayShouldLoad } from "./pluginRegistryPredicates";
|
||||
import { shouldFetchTemporalBounds, shouldFetchTemporalSnapshot } from "./temporalLifecyclePredicates";
|
||||
import { createTemporalSnapshotGuards, type TemporalSnapshotResponse } from "./temporalSnapshotGuards";
|
||||
import type { LinkPrediction, PathResponse } from "./GraphInspectorPanel";
|
||||
import type { GraphSceneHandle, GraphSceneRuntime } from "./scene";
|
||||
import type {
|
||||
@@ -1479,6 +1480,23 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
summary?.edgeCount,
|
||||
]);
|
||||
|
||||
// Guards the snapshot lifecycle: at most one in-flight request per scrubber
|
||||
// position (identical-`at` polls are deduplicated, breaking the idle/play
|
||||
// polling loop), applied snapshots are cached and re-applied on revisit, and
|
||||
// a response applies only while the scrubber is still on its position
|
||||
// (out-of-order responses cannot clobber the active-node count).
|
||||
const temporalSnapshotGuardsRef = useRef<ReturnType<typeof createTemporalSnapshotGuards> | null>(null);
|
||||
if (temporalSnapshotGuardsRef.current === null) {
|
||||
temporalSnapshotGuardsRef.current = createTemporalSnapshotGuards();
|
||||
}
|
||||
const temporalSnapshotGuards = temporalSnapshotGuardsRef.current;
|
||||
|
||||
// A new graph summary means the graph data was replaced (reload/retry);
|
||||
// snapshots cached against the previous graph are stale, so reset all state.
|
||||
useEffect(() => {
|
||||
temporalSnapshotGuards.reset();
|
||||
}, [summary]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!canFetchTemporalSnapshot) {
|
||||
return;
|
||||
@@ -1488,37 +1506,67 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
return;
|
||||
}
|
||||
|
||||
const atMs = debouncedTime.getTime();
|
||||
const { seq, cached } = temporalSnapshotGuards.begin(atMs);
|
||||
if (seq === null) {
|
||||
// An identical request is already in flight: one request per position.
|
||||
return;
|
||||
}
|
||||
|
||||
let cancelled = false;
|
||||
|
||||
const applyData = (data: TemporalSnapshotResponse) => {
|
||||
const nextActiveIds = new Set(data.active_node_ids);
|
||||
requestAnimationFrame(() => {
|
||||
if (cancelled) return;
|
||||
if (!temporalSnapshotGuards.shouldApply(atMs, seq)) {
|
||||
// The scrubber moved on (or this request was superseded): release the
|
||||
// position so a return to it refetches instead of stalling.
|
||||
temporalSnapshotGuards.finish(atMs, seq);
|
||||
return;
|
||||
}
|
||||
const previous = prevActiveIdsRef.current;
|
||||
previous.forEach((id) => {
|
||||
if (!nextActiveIds.has(id) && graph.hasNode(id)) {
|
||||
graph.setNodeAttribute(id, "hidden", true);
|
||||
}
|
||||
});
|
||||
nextActiveIds.forEach((id) => {
|
||||
if (graph.hasNode(id)) {
|
||||
graph.setNodeAttribute(id, "hidden", false);
|
||||
}
|
||||
});
|
||||
prevActiveIdsRef.current = nextActiveIds;
|
||||
setActiveNodeCount(data.active_node_count);
|
||||
setGraphVersion((current) => current + 1);
|
||||
sceneRef.current?.getRuntime()?.requestRender();
|
||||
temporalSnapshotGuards.apply(atMs, seq, data);
|
||||
});
|
||||
};
|
||||
|
||||
if (cached) {
|
||||
// Returning to a position whose snapshot was already applied: re-apply
|
||||
// the cached result without a network request.
|
||||
applyData(cached);
|
||||
return;
|
||||
}
|
||||
|
||||
const applySnapshot = async () => {
|
||||
try {
|
||||
const at = debouncedTime.toISOString();
|
||||
const response = await fetch(`/api/temporal/snapshot?at=${encodeURIComponent(at)}`);
|
||||
if (!response.ok || cancelled) return;
|
||||
|
||||
const data: { active_node_ids: string[]; active_node_count: number } = await response.json();
|
||||
if (!response.ok) {
|
||||
// A failed request must be retryable if the scrubber returns.
|
||||
if (!cancelled) temporalSnapshotGuards.finish(atMs, seq);
|
||||
return;
|
||||
}
|
||||
if (cancelled) return;
|
||||
|
||||
const nextActiveIds = new Set(data.active_node_ids);
|
||||
requestAnimationFrame(() => {
|
||||
if (cancelled) return;
|
||||
const previous = prevActiveIdsRef.current;
|
||||
previous.forEach((id) => {
|
||||
if (!nextActiveIds.has(id) && graph.hasNode(id)) {
|
||||
graph.setNodeAttribute(id, "hidden", true);
|
||||
}
|
||||
});
|
||||
nextActiveIds.forEach((id) => {
|
||||
if (graph.hasNode(id)) {
|
||||
graph.setNodeAttribute(id, "hidden", false);
|
||||
}
|
||||
});
|
||||
prevActiveIdsRef.current = nextActiveIds;
|
||||
setActiveNodeCount(data.active_node_count);
|
||||
setGraphVersion((current) => current + 1);
|
||||
sceneRef.current?.getRuntime()?.requestRender();
|
||||
});
|
||||
const data: TemporalSnapshotResponse = await response.json();
|
||||
if (cancelled) return;
|
||||
applyData(data);
|
||||
} catch (fetchError) {
|
||||
temporalSnapshotGuards.finish(atMs, seq);
|
||||
if (!cancelled) {
|
||||
console.error("[Temporal] Snapshot fetch failed", fetchError);
|
||||
}
|
||||
@@ -1528,6 +1576,8 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
applySnapshot();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
// A cancelled request must be retryable when its position is revisited.
|
||||
temporalSnapshotGuards.finish(atMs, seq);
|
||||
};
|
||||
}, [
|
||||
canFetchTemporalSnapshot,
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
/**
|
||||
* Guards for the temporal snapshot fetch/apply lifecycle.
|
||||
*
|
||||
* The snapshot effect previously fetched /api/temporal/snapshot with no
|
||||
* idempotency or ordering protection. Upstream churn (timeline recreation
|
||||
* while bounds settle, play ticks resetting the playhead, drag events) could
|
||||
* re-request the same `at` repeatedly, and responses could arrive after the
|
||||
* scrubber had moved on.
|
||||
*
|
||||
* The guards enforce:
|
||||
* - at most one in-flight request per scrubber position (identical `at`
|
||||
* values are deduplicated while a request is pending, breaking the
|
||||
* idle/play polling loop);
|
||||
* - successful snapshots are cached per position and re-applied when the
|
||||
* scrubber returns (play wrap-around, back-scrubbing) without a refetch;
|
||||
* - a response is applied only while the scrubber is still on its position,
|
||||
* so out-of-order responses cannot clobber a newer position's count;
|
||||
* - failed, cancelled, or superseded requests release their position so it
|
||||
* can be fetched again on the next visit;
|
||||
* - `reset()` drops all state when the underlying graph data is replaced
|
||||
* (reload/retry), because cached snapshots describe the previous graph.
|
||||
*
|
||||
* `createTemporalSnapshotGuards()` is stateful by design.
|
||||
*/
|
||||
|
||||
export interface TemporalSnapshotResponse {
|
||||
active_node_ids: string[];
|
||||
active_node_count: number;
|
||||
}
|
||||
|
||||
export interface TemporalSnapshotRequest {
|
||||
/** null when the request was deduplicated because one is already in flight. */
|
||||
seq: number | null;
|
||||
/** The snapshot previously applied for this position, when revisiting it. */
|
||||
cached: TemporalSnapshotResponse | null;
|
||||
}
|
||||
|
||||
export interface TemporalSnapshotGuards {
|
||||
/** Begin (or dedupe) a request for `atMs`; marks it as the current position. */
|
||||
begin(atMs: number): TemporalSnapshotRequest;
|
||||
/** True when the response for `atMs`/`seq` may be applied (scrubber still on `atMs`). */
|
||||
shouldApply(atMs: number, seq: number): boolean;
|
||||
/** Record a successful application and cache its snapshot for revisits. */
|
||||
apply(atMs: number, seq: number, data: TemporalSnapshotResponse): void;
|
||||
/** Release a position whose request failed, was cancelled, or was superseded. */
|
||||
finish(atMs: number, seq: number): void;
|
||||
/** Drop all state; call when the underlying graph data is replaced (reload). */
|
||||
reset(): void;
|
||||
}
|
||||
|
||||
interface SnapshotEntry {
|
||||
seq: number;
|
||||
/** null while the request is in flight (or before the first success). */
|
||||
data: TemporalSnapshotResponse | null;
|
||||
}
|
||||
|
||||
/** Upper bound on cached positions so long scrubbing sessions stay bounded. */
|
||||
const MAX_CACHED_POSITIONS = 256;
|
||||
|
||||
export function createTemporalSnapshotGuards(): TemporalSnapshotGuards {
|
||||
const entries = new Map<number, SnapshotEntry>();
|
||||
let latestRequestSeq = 0;
|
||||
let currentAtMs: number | null = null;
|
||||
|
||||
const evictOldest = () => {
|
||||
while (entries.size > MAX_CACHED_POSITIONS) {
|
||||
const oldestAtMs = entries.keys().next().value;
|
||||
if (oldestAtMs === undefined) return;
|
||||
entries.delete(oldestAtMs);
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
begin(atMs) {
|
||||
const existing = entries.get(atMs);
|
||||
if (existing && existing.data === null) {
|
||||
// Identical request already in flight: dedupe, but the scrubber is here now.
|
||||
currentAtMs = atMs;
|
||||
return { seq: null, cached: null };
|
||||
}
|
||||
latestRequestSeq += 1;
|
||||
const seq = latestRequestSeq;
|
||||
entries.set(atMs, { seq, data: existing?.data ?? null });
|
||||
currentAtMs = atMs;
|
||||
evictOldest();
|
||||
return { seq, cached: existing?.data ?? null };
|
||||
},
|
||||
|
||||
shouldApply(atMs, seq) {
|
||||
return atMs === currentAtMs && entries.get(atMs)?.seq === seq;
|
||||
},
|
||||
|
||||
apply(atMs, seq, data) {
|
||||
const entry = entries.get(atMs);
|
||||
if (entry && entry.seq === seq) {
|
||||
entry.data = data;
|
||||
}
|
||||
},
|
||||
|
||||
finish(atMs, seq) {
|
||||
const entry = entries.get(atMs);
|
||||
if (entry && entry.seq === seq && entry.data === null) {
|
||||
entries.delete(atMs);
|
||||
}
|
||||
},
|
||||
|
||||
reset() {
|
||||
entries.clear();
|
||||
latestRequestSeq = 0;
|
||||
currentAtMs = null;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -318,6 +318,20 @@ interface EdgeListResponse {
|
||||
|
||||
const PAGE_LIMIT = 1000;
|
||||
|
||||
/** Surface the server's `detail` message (e.g. auth/setup guidance) on non-OK responses. */
|
||||
async function fetchErrorDetail(response: Response): Promise<string> {
|
||||
try {
|
||||
const body: unknown = await response.json();
|
||||
const detail = (body as { detail?: unknown } | null)?.detail;
|
||||
if (typeof detail === "string" && detail.trim()) {
|
||||
return ` — ${detail.trim()}`;
|
||||
}
|
||||
} catch {
|
||||
// Non-JSON or unreadable body: fall back to the status-only message.
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
async function fetchAllNodes(
|
||||
signal: AbortSignal,
|
||||
onProgress?: (progress: GraphLoadProgress) => void,
|
||||
@@ -335,7 +349,7 @@ async function fetchAllNodes(
|
||||
|
||||
const response = await fetch(url.toString(), { signal });
|
||||
if (!response.ok) {
|
||||
throw new Error(`Fetch failed: ${response.status}`);
|
||||
throw new Error(`Fetch failed: ${response.status}${await fetchErrorDetail(response)}`);
|
||||
}
|
||||
|
||||
const data: NodeListResponse = await response.json();
|
||||
@@ -390,7 +404,7 @@ async function fetchAllEdges(
|
||||
|
||||
const response = await fetch(url.toString(), { signal });
|
||||
if (!response.ok) {
|
||||
throw new Error(`Fetch failed: ${response.status}`);
|
||||
throw new Error(`Fetch failed: ${response.status}${await fetchErrorDetail(response)}`);
|
||||
}
|
||||
|
||||
const data: EdgeListResponse = await response.json();
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawn, type ChildProcess } from "node:child_process";
|
||||
import { existsSync } from "node:fs";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import test from "node:test";
|
||||
import { chromium, type Page } from "playwright";
|
||||
|
||||
const PORT = 4173;
|
||||
const BASE_URL = `http://127.0.0.1:${PORT}`;
|
||||
|
||||
const nodes = [
|
||||
{ id: "alice", type: "Person", content: "Alice", properties: {} },
|
||||
{ id: "bob", type: "Person", content: "Bob", properties: {} },
|
||||
{ id: "acme", type: "Organization", content: "Acme", properties: {} },
|
||||
{ id: "new_york", type: "Location", content: "New York", properties: {} },
|
||||
];
|
||||
|
||||
const edges = [
|
||||
{ id: "edge_alice_acme", familyId: "edge_alice_acme", source: "alice", target: "acme", type: "WORKS_AT", weight: 1, properties: {} },
|
||||
{ id: "edge_bob_alice", familyId: "edge_bob_alice", source: "bob", target: "alice", type: "KNOWS", weight: 1, properties: {} },
|
||||
{ id: "edge_acme_new_york", familyId: "edge_acme_new_york", source: "acme", target: "new_york", type: "LOCATED_IN", weight: 1, properties: {} },
|
||||
];
|
||||
|
||||
let server: ChildProcess | undefined;
|
||||
|
||||
async function startVite(): Promise<void> {
|
||||
server = spawn("npm", ["run", "dev", "--", "--host", "127.0.0.1", "--port", String(PORT)], {
|
||||
cwd: process.cwd(),
|
||||
stdio: "ignore",
|
||||
});
|
||||
|
||||
for (let attempt = 0; attempt < 50; attempt += 1) {
|
||||
try {
|
||||
const response = await fetch(BASE_URL);
|
||||
if (response.ok) return;
|
||||
} catch {
|
||||
// Vite is still starting.
|
||||
}
|
||||
await delay(100);
|
||||
}
|
||||
throw new Error("Vite did not become ready");
|
||||
}
|
||||
|
||||
async function installApiFixture(page: Page): Promise<void> {
|
||||
await page.route("**/api/graph/**", async (route) => {
|
||||
const pathname = new URL(route.request().url()).pathname;
|
||||
if (pathname === "/api/graph/stats") {
|
||||
await route.fulfill({ json: { node_count: 4, edge_count: 3 } });
|
||||
} else if (pathname === "/api/graph/nodes") {
|
||||
await route.fulfill({ json: { nodes, total: nodes.length, skip: 0, limit: 1000, next_cursor: null } });
|
||||
} else if (pathname === "/api/graph/edges") {
|
||||
await route.fulfill({ json: { edges, total: edges.length, skip: 0, limit: 1000, next_cursor: null } });
|
||||
} else {
|
||||
await route.continue();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test("real Explorer loading path hydrates and renders API edge labels", async (t) => {
|
||||
await startVite();
|
||||
t.after(async () => {
|
||||
server?.kill();
|
||||
});
|
||||
|
||||
const browser = await chromium.launch({
|
||||
headless: true,
|
||||
executablePath: process.env.CHROMIUM_PATH || (existsSync("/usr/bin/chromium") ? "/usr/bin/chromium" : undefined),
|
||||
});
|
||||
t.after(() => browser.close());
|
||||
const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||
|
||||
await page.addInitScript(() => {
|
||||
const captured = (window as Window & { __capturedCanvasText?: string[] }).__capturedCanvasText = [];
|
||||
const originalFillText = CanvasRenderingContext2D.prototype.fillText;
|
||||
CanvasRenderingContext2D.prototype.fillText = function (text: string, ...args: [number, number, number?, number?]) {
|
||||
captured.push(String(text));
|
||||
return originalFillText.call(this, text, ...args);
|
||||
};
|
||||
});
|
||||
await installApiFixture(page);
|
||||
await page.goto(BASE_URL);
|
||||
await page.getByRole("button", { name: /Open Semantica Explorer/ }).click();
|
||||
|
||||
await page.locator("canvas").nth(0).waitFor({ state: "attached" });
|
||||
await page.waitForFunction(() => document.querySelectorAll("canvas").length >= 2);
|
||||
await page.waitForFunction(() => {
|
||||
const labels = (window as Window & { __capturedCanvasText?: string[] }).__capturedCanvasText ?? [];
|
||||
return ["WORKS_AT", "KNOWS", "LOCATED_IN"].every((label) => labels.includes(label));
|
||||
}, undefined, { timeout: 10_000 });
|
||||
|
||||
const capturedLabels = await page.evaluate(() => (window as Window & { __capturedCanvasText?: string[] }).__capturedCanvasText ?? []);
|
||||
for (const label of ["WORKS_AT", "KNOWS", "LOCATED_IN"]) {
|
||||
assert.ok(capturedLabels.includes(label), `Expected rendered edge label ${label}`);
|
||||
}
|
||||
assert.ok(capturedLabels.includes("Alice"));
|
||||
|
||||
await page.getByRole("button", { name: "Zoom In" }).click();
|
||||
await page.waitForTimeout(250);
|
||||
const labelsAfterZoom = await page.evaluate(() => (window as Window & { __capturedCanvasText?: string[] }).__capturedCanvasText ?? []);
|
||||
for (const label of ["WORKS_AT", "KNOWS", "LOCATED_IN"]) {
|
||||
assert.ok(labelsAfterZoom.includes(label), `Expected edge label ${label} after zoom`);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,508 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import {
|
||||
batchMergeEdges,
|
||||
batchMergeNodes,
|
||||
clearGraph,
|
||||
graph,
|
||||
} from "../src/store/graphStore.ts";
|
||||
import {
|
||||
buildStructuralDistanceSnapshot,
|
||||
classifyFullGraphEdge,
|
||||
resolveDisplayGraph,
|
||||
resolveEdgeElementStyle,
|
||||
resolveEdgeVisualState,
|
||||
resolveNodeElementStyle,
|
||||
resolveNodeVisualState,
|
||||
shouldForceNodeLabel,
|
||||
} from "../src/workspaces/GraphWorkspace/graphSceneState.ts";
|
||||
import { GRAPH_THEME, type GraphZoomTier } from "../src/workspaces/GraphWorkspace/graphTheme.ts";
|
||||
|
||||
test.beforeEach(() => {
|
||||
clearGraph();
|
||||
});
|
||||
|
||||
test.after(() => {
|
||||
clearGraph();
|
||||
});
|
||||
|
||||
/**
|
||||
* Loads the canonical 4-node, 3-edge deterministic test graph (Semantica #1037).
|
||||
*
|
||||
* Graph structure:
|
||||
* Alice (Person) --WORKS_AT--> Acme (Organization)
|
||||
* Bob (Person) --KNOWS--> Alice (Person)
|
||||
* Acme (Organization) --LOCATED_IN--> New York (Location)
|
||||
*/
|
||||
function loadDeterministicTestGraph() {
|
||||
batchMergeNodes([
|
||||
{
|
||||
id: "alice",
|
||||
attributes: {
|
||||
label: "Alice",
|
||||
content: "Alice",
|
||||
x: 0,
|
||||
y: 0,
|
||||
size: 8,
|
||||
color: "#63E6FF",
|
||||
baseColor: "#63E6FF",
|
||||
nodeType: "Person",
|
||||
semanticGroup: "Person",
|
||||
properties: {},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "bob",
|
||||
attributes: {
|
||||
label: "Bob",
|
||||
content: "Bob",
|
||||
x: -50,
|
||||
y: 0,
|
||||
size: 8,
|
||||
color: "#63E6FF",
|
||||
baseColor: "#63E6FF",
|
||||
nodeType: "Person",
|
||||
semanticGroup: "Person",
|
||||
properties: {},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "acme",
|
||||
attributes: {
|
||||
label: "Acme",
|
||||
content: "Acme",
|
||||
x: 50,
|
||||
y: 0,
|
||||
size: 8,
|
||||
color: "#A78BFA",
|
||||
baseColor: "#A78BFA",
|
||||
nodeType: "Organization",
|
||||
semanticGroup: "Organization",
|
||||
properties: {},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "new_york",
|
||||
attributes: {
|
||||
label: "New York",
|
||||
content: "New York",
|
||||
x: 100,
|
||||
y: 0,
|
||||
size: 8,
|
||||
color: "#34D399",
|
||||
baseColor: "#34D399",
|
||||
nodeType: "Location",
|
||||
semanticGroup: "Location",
|
||||
properties: {},
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
batchMergeEdges([
|
||||
{
|
||||
id: "edge_alice_acme",
|
||||
source: "alice",
|
||||
target: "acme",
|
||||
attributes: {
|
||||
edgeId: "edge_alice_acme",
|
||||
edgeType: "WORKS_AT",
|
||||
weight: 1.0,
|
||||
visualPriority: 0.8,
|
||||
baseSize: 0.8,
|
||||
properties: {},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "edge_bob_alice",
|
||||
source: "bob",
|
||||
target: "alice",
|
||||
attributes: {
|
||||
edgeId: "edge_bob_alice",
|
||||
edgeType: "KNOWS",
|
||||
weight: 1.0,
|
||||
visualPriority: 0.8,
|
||||
baseSize: 0.8,
|
||||
properties: {},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "edge_acme_new_york",
|
||||
source: "acme",
|
||||
target: "new_york",
|
||||
attributes: {
|
||||
edgeId: "edge_acme_new_york",
|
||||
edgeType: "LOCATED_IN",
|
||||
weight: 1.0,
|
||||
visualPriority: 0.8,
|
||||
baseSize: 0.8,
|
||||
properties: {},
|
||||
},
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
test("deterministic graph contains exactly 4 nodes and 3 edges in store", () => {
|
||||
loadDeterministicTestGraph();
|
||||
|
||||
assert.equal(graph.order, 4, "Expected exactly 4 nodes");
|
||||
assert.equal(graph.size, 3, "Expected exactly 3 edges");
|
||||
|
||||
// Verify node identities and labels
|
||||
const alice = graph.getNodeAttributes("alice");
|
||||
const bob = graph.getNodeAttributes("bob");
|
||||
const acme = graph.getNodeAttributes("acme");
|
||||
const newYork = graph.getNodeAttributes("new_york");
|
||||
|
||||
assert.equal(alice.label, "Alice");
|
||||
assert.equal(alice.nodeType, "Person");
|
||||
assert.equal(alice.color, "#63E6FF");
|
||||
|
||||
assert.equal(bob.label, "Bob");
|
||||
assert.equal(bob.nodeType, "Person");
|
||||
assert.equal(bob.color, "#63E6FF");
|
||||
|
||||
assert.equal(acme.label, "Acme");
|
||||
assert.equal(acme.nodeType, "Organization");
|
||||
assert.equal(acme.color, "#A78BFA");
|
||||
|
||||
assert.equal(newYork.label, "New York");
|
||||
assert.equal(newYork.nodeType, "Location");
|
||||
assert.equal(newYork.color, "#34D399");
|
||||
|
||||
// Verify edge connectivity and canonical edgeType labels
|
||||
const edgeAliceAcme = graph.getEdgeAttributes("edge_alice_acme");
|
||||
const edgeBobAlice = graph.getEdgeAttributes("edge_bob_alice");
|
||||
const edgeAcmeNewYork = graph.getEdgeAttributes("edge_acme_new_york");
|
||||
|
||||
assert.equal(edgeAliceAcme.edgeType, "WORKS_AT");
|
||||
assert.equal(graph.source("edge_alice_acme"), "alice");
|
||||
assert.equal(graph.target("edge_alice_acme"), "acme");
|
||||
|
||||
assert.equal(edgeBobAlice.edgeType, "KNOWS");
|
||||
assert.equal(graph.source("edge_bob_alice"), "bob");
|
||||
assert.equal(graph.target("edge_bob_alice"), "alice");
|
||||
|
||||
assert.equal(edgeAcmeNewYork.edgeType, "LOCATED_IN");
|
||||
assert.equal(graph.source("edge_acme_new_york"), "acme");
|
||||
assert.equal(graph.target("edge_acme_new_york"), "new_york");
|
||||
});
|
||||
|
||||
test("display graph resolution preserves all 4 nodes and 3 edges in full view", () => {
|
||||
loadDeterministicTestGraph();
|
||||
|
||||
const { graph: displayGraph } = resolveDisplayGraph("", [], [], "full", { aggregationEnabled: false });
|
||||
|
||||
assert.equal(displayGraph.order, 4);
|
||||
assert.equal(displayGraph.size, 3);
|
||||
assert.ok(displayGraph.hasNode("alice"));
|
||||
assert.ok(displayGraph.hasNode("bob"));
|
||||
assert.ok(displayGraph.hasNode("acme"));
|
||||
assert.ok(displayGraph.hasNode("new_york"));
|
||||
assert.ok(displayGraph.hasEdge("edge_alice_acme"));
|
||||
assert.ok(displayGraph.hasEdge("edge_bob_alice"));
|
||||
assert.ok(displayGraph.hasEdge("edge_acme_new_york"));
|
||||
});
|
||||
|
||||
test("structural distance calculation resolves correct hop counts across the 3-edge chain", () => {
|
||||
loadDeterministicTestGraph();
|
||||
|
||||
// From Bob: Bob (0) -> Alice (1) -> Acme (2) -> New York (3)
|
||||
const distances = buildStructuralDistanceSnapshot(graph, "bob", 3);
|
||||
|
||||
assert.equal(distances.bob, 0);
|
||||
assert.equal(distances.alice, 1);
|
||||
assert.equal(distances.acme, 2);
|
||||
assert.equal(distances.new_york, 3);
|
||||
});
|
||||
|
||||
test("edge rendering and canonical edge labels remain legible across zoom tiers and inspection modes", () => {
|
||||
loadDeterministicTestGraph();
|
||||
|
||||
const canonicalEdges = [
|
||||
{ id: "edge_alice_acme", source: "alice", target: "acme", label: "WORKS_AT" },
|
||||
{ id: "edge_bob_alice", source: "bob", target: "alice", label: "KNOWS" },
|
||||
{ id: "edge_acme_new_york", source: "acme", target: "new_york", label: "LOCATED_IN" },
|
||||
];
|
||||
|
||||
// 1. Edge attributes preserve canonical edgeType labels in graph store:
|
||||
for (const item of canonicalEdges) {
|
||||
const attrs = graph.getEdgeAttributes(item.id);
|
||||
assert.equal(attrs.edgeType, item.label, `Edge ${item.id} must have edgeType ${item.label}`);
|
||||
assert.equal(graph.source(item.id), item.source);
|
||||
assert.equal(graph.target(item.id), item.target);
|
||||
}
|
||||
|
||||
// 2. In active context / neighbor state across all zoom tiers (overview, structure, inspection):
|
||||
const allTiers: GraphZoomTier[] = ["overview", "structure", "inspection"];
|
||||
for (const tier of allTiers) {
|
||||
for (const item of canonicalEdges) {
|
||||
const attrs = graph.getEdgeAttributes(item.id);
|
||||
const contextStyle = resolveEdgeElementStyle(
|
||||
GRAPH_THEME,
|
||||
tier,
|
||||
"neighbor",
|
||||
attrs,
|
||||
item.source,
|
||||
item.target,
|
||||
"full",
|
||||
item.id,
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
contextStyle.hidden,
|
||||
false,
|
||||
`Edge ${item.id} (${item.label}) in context state 'neighbor' must be visible in zoom tier '${tier}'`,
|
||||
);
|
||||
assert.ok(
|
||||
contextStyle.size !== undefined && contextStyle.size > 0,
|
||||
`Edge ${item.id} (${item.label}) must have positive render size in zoom tier '${tier}'`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. In selected state in inspection zoom tier (close examination of edge details and label):
|
||||
for (const item of canonicalEdges) {
|
||||
const attrs = graph.getEdgeAttributes(item.id);
|
||||
const selectedStyle = resolveEdgeElementStyle(
|
||||
GRAPH_THEME,
|
||||
"inspection",
|
||||
"selected",
|
||||
attrs,
|
||||
item.source,
|
||||
item.target,
|
||||
"full",
|
||||
item.id,
|
||||
"selected",
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
selectedStyle.hidden,
|
||||
false,
|
||||
`Selected edge ${item.id} (${item.label}) must be visible in inspection zoom tier`,
|
||||
);
|
||||
assert.ok(
|
||||
selectedStyle.size !== undefined && selectedStyle.size > 0,
|
||||
`Selected edge ${item.id} (${item.label}) must have positive render size`,
|
||||
);
|
||||
}
|
||||
|
||||
// 4. Verify inspection zoom tier camera and arrow rendering settings
|
||||
assert.equal(GRAPH_THEME.zoomTiers.inspection.showContextualArrows, true);
|
||||
assert.equal(GRAPH_THEME.zoomTiers.inspection.showCurves, true);
|
||||
});
|
||||
|
||||
test("node hover interaction preserves edge visibility and highlights canonical incident edge types", () => {
|
||||
loadDeterministicTestGraph();
|
||||
|
||||
// Scenario 1: Hover Alice
|
||||
// Incident edges: Alice -> Acme (WORKS_AT) and Bob -> Alice (KNOWS)
|
||||
const aliceAttrs = graph.getNodeAttributes("alice");
|
||||
const aliceVisual = resolveNodeVisualState("alice", "structure", "alice", "", "", new Set(), new Set(), new Set());
|
||||
assert.equal(aliceVisual, "hovered");
|
||||
|
||||
const aliceStyle = resolveNodeElementStyle(GRAPH_THEME, "structure", "hovered", aliceAttrs, "Alice");
|
||||
assert.equal(aliceStyle.forceLabel, true, "Hovered Alice must force-render label");
|
||||
assert.equal(aliceStyle.label, "Alice");
|
||||
assert.equal(aliceStyle.showHalo, true, "Hovered Alice must show interactive halo");
|
||||
|
||||
const aliceIncidentEdges = new Set(["edge_alice_acme", "edge_bob_alice"]);
|
||||
|
||||
// Edge Alice -> Acme (WORKS_AT) under Alice hover
|
||||
const aliceAcmeAttrs = graph.getEdgeAttributes("edge_alice_acme");
|
||||
assert.equal(aliceAcmeAttrs.edgeType, "WORKS_AT");
|
||||
const aliceAcmeState = resolveEdgeVisualState(
|
||||
"edge_alice_acme",
|
||||
"alice",
|
||||
"acme",
|
||||
"structure",
|
||||
"alice",
|
||||
"",
|
||||
"",
|
||||
new Set(),
|
||||
new Set(),
|
||||
aliceIncidentEdges,
|
||||
);
|
||||
assert.equal(aliceAcmeState, "hovered");
|
||||
const aliceAcmeStyle = resolveEdgeElementStyle(
|
||||
GRAPH_THEME,
|
||||
"structure",
|
||||
"hovered",
|
||||
aliceAcmeAttrs,
|
||||
"alice",
|
||||
"acme",
|
||||
"full",
|
||||
"edge_alice_acme",
|
||||
);
|
||||
assert.equal(aliceAcmeStyle.hidden, false, "Incident edge WORKS_AT must remain visible on hover");
|
||||
assert.ok(aliceAcmeStyle.size !== undefined && aliceAcmeStyle.size > 0);
|
||||
|
||||
// Edge Bob -> Alice (KNOWS) under Alice hover
|
||||
const bobAliceAttrs = graph.getEdgeAttributes("edge_bob_alice");
|
||||
assert.equal(bobAliceAttrs.edgeType, "KNOWS");
|
||||
const bobAliceState = resolveEdgeVisualState(
|
||||
"edge_bob_alice",
|
||||
"bob",
|
||||
"alice",
|
||||
"structure",
|
||||
"alice",
|
||||
"",
|
||||
"",
|
||||
new Set(),
|
||||
new Set(),
|
||||
aliceIncidentEdges,
|
||||
);
|
||||
assert.equal(bobAliceState, "hovered");
|
||||
const bobAliceStyle = resolveEdgeElementStyle(
|
||||
GRAPH_THEME,
|
||||
"structure",
|
||||
"hovered",
|
||||
bobAliceAttrs,
|
||||
"bob",
|
||||
"alice",
|
||||
"full",
|
||||
"edge_bob_alice",
|
||||
);
|
||||
assert.equal(bobAliceStyle.hidden, false, "Incident edge KNOWS must remain visible on hover");
|
||||
|
||||
// Non-incident edge Acme -> New York (LOCATED_IN) under Alice hover
|
||||
const acmeNyAttrs = graph.getEdgeAttributes("edge_acme_new_york");
|
||||
assert.equal(acmeNyAttrs.edgeType, "LOCATED_IN");
|
||||
const acmeNyState = resolveEdgeVisualState(
|
||||
"edge_acme_new_york",
|
||||
"acme",
|
||||
"new_york",
|
||||
"structure",
|
||||
"alice",
|
||||
"",
|
||||
"",
|
||||
new Set(),
|
||||
new Set(),
|
||||
aliceIncidentEdges,
|
||||
);
|
||||
assert.equal(acmeNyState, "muted");
|
||||
|
||||
// Scenario 2: Hover Acme
|
||||
// Incident edges: Alice -> Acme (WORKS_AT) and Acme -> New York (LOCATED_IN)
|
||||
const acmeAttrs = graph.getNodeAttributes("acme");
|
||||
const acmeStyle = resolveNodeElementStyle(GRAPH_THEME, "structure", "hovered", acmeAttrs, "Acme");
|
||||
assert.equal(acmeStyle.forceLabel, true);
|
||||
assert.equal(acmeStyle.label, "Acme");
|
||||
|
||||
const acmeIncidentEdges = new Set(["edge_alice_acme", "edge_acme_new_york"]);
|
||||
const acmeNyHoverState = resolveEdgeVisualState(
|
||||
"edge_acme_new_york",
|
||||
"acme",
|
||||
"new_york",
|
||||
"structure",
|
||||
"acme",
|
||||
"",
|
||||
"",
|
||||
new Set(),
|
||||
new Set(),
|
||||
acmeIncidentEdges,
|
||||
);
|
||||
assert.equal(acmeNyHoverState, "hovered");
|
||||
const acmeNyHoverStyle = resolveEdgeElementStyle(
|
||||
GRAPH_THEME,
|
||||
"structure",
|
||||
"hovered",
|
||||
acmeNyAttrs,
|
||||
"acme",
|
||||
"new_york",
|
||||
"full",
|
||||
"edge_acme_new_york",
|
||||
);
|
||||
assert.equal(acmeNyHoverStyle.hidden, false, "Incident edge LOCATED_IN must remain visible on hover");
|
||||
|
||||
// Scenario 3: Hover Bob
|
||||
// Incident edge: Bob -> Alice (KNOWS)
|
||||
const bobAttrs = graph.getNodeAttributes("bob");
|
||||
const bobStyle = resolveNodeElementStyle(GRAPH_THEME, "structure", "hovered", bobAttrs, "Bob");
|
||||
assert.equal(bobStyle.forceLabel, true);
|
||||
assert.equal(bobStyle.label, "Bob");
|
||||
|
||||
const bobIncidentEdges = new Set(["edge_bob_alice"]);
|
||||
const bobAliceHoverState = resolveEdgeVisualState(
|
||||
"edge_bob_alice",
|
||||
"bob",
|
||||
"alice",
|
||||
"structure",
|
||||
"bob",
|
||||
"",
|
||||
"",
|
||||
new Set(),
|
||||
new Set(),
|
||||
bobIncidentEdges,
|
||||
);
|
||||
assert.equal(bobAliceHoverState, "hovered");
|
||||
});
|
||||
|
||||
test("edge selection maintains canonical edge type labels and active visual state", () => {
|
||||
loadDeterministicTestGraph();
|
||||
|
||||
const edgeCases = [
|
||||
{ id: "edge_alice_acme", source: "alice", target: "acme", label: "WORKS_AT" },
|
||||
{ id: "edge_bob_alice", source: "bob", target: "alice", label: "KNOWS" },
|
||||
{ id: "edge_acme_new_york", source: "acme", target: "new_york", label: "LOCATED_IN" },
|
||||
];
|
||||
|
||||
for (const { id, source, target, label } of edgeCases) {
|
||||
const attrs = graph.getEdgeAttributes(id);
|
||||
assert.equal(attrs.edgeType, label);
|
||||
|
||||
const visualState = resolveEdgeVisualState(
|
||||
id,
|
||||
source,
|
||||
target,
|
||||
"inspection",
|
||||
null,
|
||||
"",
|
||||
id, // selected edge
|
||||
new Set(),
|
||||
new Set(),
|
||||
);
|
||||
assert.equal(visualState, "selected", `Selected edge ${id} must resolve to 'selected' state`);
|
||||
|
||||
const style = resolveEdgeElementStyle(
|
||||
GRAPH_THEME,
|
||||
"inspection",
|
||||
"selected",
|
||||
attrs,
|
||||
source,
|
||||
target,
|
||||
"full",
|
||||
id,
|
||||
"selected",
|
||||
);
|
||||
|
||||
assert.equal(style.hidden, false, `Selected edge ${id} (${label}) must not be hidden`);
|
||||
assert.ok(
|
||||
style.size !== undefined && style.size > 0,
|
||||
`Selected edge ${id} (${label}) must have positive render size`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("node labels remain forced visible during hover, selection, and inspection zoom tier", () => {
|
||||
loadDeterministicTestGraph();
|
||||
|
||||
const nodes = ["alice", "bob", "acme", "new_york"];
|
||||
|
||||
for (const nid of nodes) {
|
||||
const attrs = graph.getNodeAttributes(nid);
|
||||
|
||||
// Hover state forces label visibility
|
||||
const hoverForcesLabel = shouldForceNodeLabel(GRAPH_THEME, "structure", "hovered", attrs, 0);
|
||||
assert.equal(hoverForcesLabel, true, `Node ${nid} label must force visible on hover`);
|
||||
|
||||
// Selected state forces label visibility
|
||||
const selectForcesLabel = shouldForceNodeLabel(GRAPH_THEME, "structure", "selected", attrs, 0);
|
||||
assert.equal(selectForcesLabel, true, `Node ${nid} label must force visible on selection`);
|
||||
|
||||
// Resolved style emits actual string label
|
||||
const style = resolveNodeElementStyle(GRAPH_THEME, "inspection", "hovered", attrs, attrs.label);
|
||||
assert.equal(style.forceLabel, true);
|
||||
assert.equal(style.label, attrs.label);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,150 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { createTemporalSnapshotGuards } from "../src/workspaces/GraphWorkspace/temporalSnapshotGuards.ts";
|
||||
|
||||
const POSITION_1 = new Date("2023-07-02T00:00:00Z").getTime();
|
||||
const POSITION_2 = new Date("2024-01-02T00:00:00Z").getTime();
|
||||
const POSITION_3 = new Date("2024-07-02T00:00:00Z").getTime();
|
||||
|
||||
const SNAPSHOT = { active_node_ids: ["n1", "n2"], active_node_count: 2 };
|
||||
|
||||
// ── begin: one request per scrubber position ─────────────────────────────────
|
||||
|
||||
test("begin: a new position returns a fresh request sequence", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
assert.deepEqual(guards.begin(POSITION_1), { seq: 1, cached: null });
|
||||
});
|
||||
|
||||
test("begin: an identical in-flight request is deduplicated (no duplicate fetch)", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
guards.begin(POSITION_1);
|
||||
assert.deepEqual(guards.begin(POSITION_1), { seq: null, cached: null });
|
||||
});
|
||||
|
||||
test("begin: distinct positions request independently", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
assert.equal(guards.begin(POSITION_1).seq, 1);
|
||||
assert.equal(guards.begin(POSITION_2).seq, 2);
|
||||
});
|
||||
|
||||
test("begin: revisiting an applied position returns its cached snapshot", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
guards.apply(POSITION_1, seq, SNAPSHOT);
|
||||
const revisit = guards.begin(POSITION_1);
|
||||
assert.equal(revisit.seq, 2);
|
||||
assert.deepEqual(revisit.cached, SNAPSHOT);
|
||||
});
|
||||
|
||||
test("begin: a failed position (finished) can be requested again", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
guards.finish(POSITION_1, seq);
|
||||
const retry = guards.begin(POSITION_1);
|
||||
assert.equal(retry.seq, 2);
|
||||
assert.equal(retry.cached, null);
|
||||
});
|
||||
|
||||
test("finish: does not clear a position whose snapshot was already applied", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
guards.apply(POSITION_1, seq, SNAPSHOT);
|
||||
guards.finish(POSITION_1, seq);
|
||||
assert.deepEqual(guards.begin(POSITION_1).cached, SNAPSHOT);
|
||||
});
|
||||
|
||||
test("finish: a stale sequence cannot release a newer request's position", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const first = guards.begin(POSITION_1);
|
||||
guards.finish(POSITION_1, first.seq);
|
||||
guards.begin(POSITION_1); // seq 2, in flight again
|
||||
guards.finish(POSITION_1, first.seq); // stale seq: must not release seq 2
|
||||
assert.deepEqual(guards.begin(POSITION_1), { seq: null, cached: null });
|
||||
});
|
||||
|
||||
// ── shouldApply: applied only while the scrubber is on that position ─────────
|
||||
|
||||
test("shouldApply: the current position's response is applied", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
assert.equal(guards.shouldApply(POSITION_1, seq), true);
|
||||
});
|
||||
|
||||
test("shouldApply: a response for a position the scrubber left is discarded", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq: seq1 } = guards.begin(POSITION_1);
|
||||
guards.begin(POSITION_2);
|
||||
assert.equal(guards.shouldApply(POSITION_1, seq1), false);
|
||||
assert.equal(guards.shouldApply(POSITION_2, 2), true);
|
||||
});
|
||||
|
||||
test("shouldApply: a late response for the position the scrubber returned to is applied", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq: seq1 } = guards.begin(POSITION_1);
|
||||
const { seq: seq2 } = guards.begin(POSITION_2);
|
||||
guards.begin(POSITION_1); // back to 1: deduplicated, no new request
|
||||
assert.equal(guards.shouldApply(POSITION_1, seq1), true);
|
||||
assert.equal(guards.shouldApply(POSITION_2, seq2), false);
|
||||
});
|
||||
|
||||
test("shouldApply: an unknown sequence is discarded", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
guards.begin(POSITION_1);
|
||||
assert.equal(guards.shouldApply(POSITION_1, 99), false);
|
||||
});
|
||||
|
||||
test("shouldApply: after a reset no pre-reset response applies", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
guards.reset();
|
||||
assert.equal(guards.shouldApply(POSITION_1, seq), false);
|
||||
});
|
||||
|
||||
// ── apply: caching for revisits ─────────────────────────────────────────────
|
||||
|
||||
test("apply: stores the snapshot so a revisit re-applies it without a request", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
guards.apply(POSITION_1, seq, SNAPSHOT);
|
||||
guards.begin(POSITION_2);
|
||||
assert.deepEqual(guards.begin(POSITION_1).cached, SNAPSHOT);
|
||||
});
|
||||
|
||||
test("apply: play wrap-around re-applies the wrapped-to position's snapshot", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
guards.apply(POSITION_1, seq, SNAPSHOT);
|
||||
guards.begin(POSITION_2);
|
||||
guards.begin(POSITION_3);
|
||||
const wrap = guards.begin(POSITION_1);
|
||||
assert.deepEqual(wrap.cached, SNAPSHOT);
|
||||
assert.equal(guards.shouldApply(POSITION_1, wrap.seq), true);
|
||||
});
|
||||
|
||||
// ── reset: graph reload ─────────────────────────────────────────────────────
|
||||
|
||||
test("reset: clears requested and cached state so positions refetch", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const { seq } = guards.begin(POSITION_1);
|
||||
guards.apply(POSITION_1, seq, SNAPSHOT);
|
||||
guards.reset();
|
||||
const fresh = guards.begin(POSITION_1);
|
||||
assert.equal(fresh.seq, 1);
|
||||
assert.equal(fresh.cached, null);
|
||||
});
|
||||
|
||||
// ── cache bound ─────────────────────────────────────────────────────────────
|
||||
|
||||
test("cache: oldest positions are evicted when the cache is full", () => {
|
||||
const guards = createTemporalSnapshotGuards();
|
||||
const count = 300;
|
||||
for (let i = 0; i < count; i++) {
|
||||
const { seq } = guards.begin(POSITION_1 + i * 1000);
|
||||
guards.apply(POSITION_1 + i * 1000, seq, SNAPSHOT);
|
||||
}
|
||||
const oldest = guards.begin(POSITION_1);
|
||||
assert.equal(oldest.cached, null); // evicted: must refetch on revisit
|
||||
const newest = guards.begin(POSITION_1 + (count - 1) * 1000);
|
||||
assert.deepEqual(newest.cached, SNAPSHOT); // still cached
|
||||
});
|
||||
+12
-3
@@ -49,7 +49,14 @@ dependencies = [
|
||||
"scipy>=1.13.1",
|
||||
"scikit-learn>=1.7.2",
|
||||
"umap-learn>=0.5.12",
|
||||
"spacy>=3.4.0",
|
||||
# thinc (spacy's core dep) dropped Python 3.9 wheels at 8.3.10, and later
|
||||
# spacy patch releases (3.8.8+) require thinc>=8.3.9-only-on-3.10+ ranges,
|
||||
# which forces a source build that fails outright on 3.9 (see Install
|
||||
# Matrix run history). Capping both keeps 3.9 on the last wheel-compatible
|
||||
# pair; 3.10+ is left unconstrained to always get the latest spacy/thinc.
|
||||
"spacy>=3.4.0,<3.8.8; python_version < '3.10'",
|
||||
"spacy>=3.4.0; python_version >= '3.10'",
|
||||
"thinc<8.3.5; python_version < '3.10'",
|
||||
"transformers>=4.20.0",
|
||||
"torch>=1.13.1",
|
||||
"sentence-transformers>=2.2.0",
|
||||
@@ -107,11 +114,12 @@ llm-gemini = ["google-genai>=0.1.0"]
|
||||
llm-anthropic = ["anthropic>=0.122.0"]
|
||||
llm-ollama = ["ollama>=0.1.0"]
|
||||
llm-deepseek = ["openai>=1.0.0"]
|
||||
llm-novita = ["openai>=1.0.0"]
|
||||
llm-litellm = ["litellm>=1.83.9"]
|
||||
llm-instructor = ["instructor>=1.15.3"]
|
||||
|
||||
llm-all = [
|
||||
"semantica[llm-openai,llm-groq,llm-gemini,llm-anthropic,llm-ollama,llm-deepseek,llm-litellm,llm-instructor]"
|
||||
"semantica[llm-openai,llm-groq,llm-gemini,llm-anthropic,llm-ollama,llm-deepseek,llm-novita,llm-litellm,llm-instructor]"
|
||||
]
|
||||
|
||||
# ---- Document Parsing ----
|
||||
@@ -124,12 +132,13 @@ shacl = ["pyshacl>=0.25.0"]
|
||||
db-snowflake = ["snowflake-connector-python>=4.6.0", "cryptography>=49.0.0"]
|
||||
db-databricks = ["databricks-sdk>=0.60.0", "databricks-sql-connector>=4.0.0"]
|
||||
db-arrow = ["pyarrow>=24.0.0"]
|
||||
db-salesforce = ["simple-salesforce>=1.12.0"]
|
||||
ingest-parquet = ["pyarrow>=24.0.0"]
|
||||
ingest-arrow = ["pyarrow>=24.0.0"]
|
||||
ingest-sap = ["requests>=2.28.0"]
|
||||
|
||||
db-all = [
|
||||
"semantica[db-snowflake,db-databricks,db-arrow]"
|
||||
"semantica[db-snowflake,db-databricks,db-salesforce,db-arrow]"
|
||||
]
|
||||
|
||||
# ---- Embedding / Models ----
|
||||
|
||||
+114
-9
@@ -3714,19 +3714,61 @@ def store_stats(cli_ctx: CLIContext, backend: str, fmt: str, local_json: bool) -
|
||||
_run_with_error_handling(_action)
|
||||
|
||||
|
||||
_MIGRATE_SUPPORTED_BACKENDS = {"faiss", "sqlite", "pgvector"}
|
||||
_MIGRATE_BATCH_SIZE = 500
|
||||
|
||||
|
||||
def _migrate_backend_config(vs_cfg: Dict[str, Any], backend: str) -> Dict[str, Any]:
|
||||
"""Resolve per-backend config out of the vector_store config section.
|
||||
|
||||
Supports both a per-backend nested shape (``vector_store.faiss.dimension``)
|
||||
and the common flat single-backend shape (``vector_store.backend`` +
|
||||
sibling keys), since either can appear depending on how many backends a
|
||||
user has configured.
|
||||
"""
|
||||
nested = vs_cfg.get(backend)
|
||||
if isinstance(nested, dict):
|
||||
return dict(nested)
|
||||
if vs_cfg.get("backend") == backend:
|
||||
return {k: v for k, v in vs_cfg.items() if k != "backend"}
|
||||
return {}
|
||||
|
||||
|
||||
def _require_faiss_index_path(cfg: Dict[str, Any], role: str) -> str:
|
||||
"""FAISS has no server to hold state between commands: a fresh FAISSStore
|
||||
starts empty and nothing outside the process persists it, so migration
|
||||
needs an explicit on-disk index to read from or write to."""
|
||||
index_path = cfg.get("index_path")
|
||||
if not index_path:
|
||||
raise click.ClickException(
|
||||
f"faiss as migration {role} requires 'index_path' in the vector_store "
|
||||
f"config (vector_store.faiss.index_path or vector_store.index_path "
|
||||
f"when faiss is the configured backend)."
|
||||
)
|
||||
return index_path
|
||||
|
||||
|
||||
@store.command("migrate")
|
||||
@click.option("--from", "from_backend", required=True)
|
||||
@click.option("--to", "to_backend", required=True)
|
||||
@click.option("--namespace", default=None)
|
||||
@click.option("--dry-run", "local_dry", is_flag=True, default=False)
|
||||
@click.option("--json", "local_json", is_flag=True, default=False)
|
||||
@click.pass_obj
|
||||
def store_migrate(cli_ctx: CLIContext, from_backend: str, to_backend: str,
|
||||
namespace: Optional[str], local_dry: bool) -> None:
|
||||
namespace: Optional[str], local_dry: bool, local_json: bool) -> None:
|
||||
"""Migrate data between backends.
|
||||
|
||||
Direct migration is only wired up between faiss, sqlite, and pgvector -
|
||||
these are the backends whose storage contract supports paging through
|
||||
every stored vector. Migrating to or from qdrant, pinecone, milvus, or
|
||||
weaviate still needs the export/reindex workaround below, since each of
|
||||
those needs its own enumeration design (Qdrant scroll, Pinecone list,
|
||||
etc.) that hasn't been built yet.
|
||||
|
||||
\b
|
||||
Example:
|
||||
semantica store migrate --from faiss --to qdrant --namespace production --dry-run
|
||||
semantica store migrate --from faiss --to sqlite --namespace production --dry-run
|
||||
"""
|
||||
cli_ctx = _require_ctx(cli_ctx)
|
||||
|
||||
@@ -3734,13 +3776,76 @@ def store_migrate(cli_ctx: CLIContext, from_backend: str, to_backend: str,
|
||||
if _is_dry(cli_ctx, local_dry):
|
||||
_dry(cli_ctx, "migrate", from_backend=from_backend, to_backend=to_backend)
|
||||
return
|
||||
raise click.ClickException(
|
||||
f"Direct backend migration ({from_backend} → {to_backend}) is not yet supported "
|
||||
"by the vector store layer. To migrate, export your data first:\n"
|
||||
" semantica export --format parquet --output dump.parquet\n"
|
||||
f" semantica embed index dump.parquet --store {to_backend}"
|
||||
+ (f" --namespace {namespace}" if namespace else "")
|
||||
)
|
||||
|
||||
if from_backend not in _MIGRATE_SUPPORTED_BACKENDS or to_backend not in _MIGRATE_SUPPORTED_BACKENDS:
|
||||
raise click.ClickException(
|
||||
f"Direct backend migration ({from_backend} → {to_backend}) is only supported "
|
||||
f"between {', '.join(sorted(_MIGRATE_SUPPORTED_BACKENDS))}. To migrate involving "
|
||||
"another backend, export your data first:\n"
|
||||
" semantica export --format parquet --output dump.parquet\n"
|
||||
f" semantica embed index dump.parquet --store {to_backend}"
|
||||
+ (f" --namespace {namespace}" if namespace else "")
|
||||
)
|
||||
|
||||
from .vector_store import VectorStore
|
||||
|
||||
vs_cfg = cli_ctx.config.to_dict().get("vector_store", {}) or {}
|
||||
source_cfg = _migrate_backend_config(vs_cfg, from_backend)
|
||||
dest_cfg = _migrate_backend_config(vs_cfg, to_backend)
|
||||
|
||||
source_index_path = None
|
||||
if from_backend == "faiss":
|
||||
source_index_path = _require_faiss_index_path(source_cfg, "source")
|
||||
dest_index_path = None
|
||||
if to_backend == "faiss":
|
||||
dest_index_path = _require_faiss_index_path(dest_cfg, "destination")
|
||||
|
||||
source = VectorStore(backend=from_backend, config=source_cfg)
|
||||
if source_index_path:
|
||||
source._backend_store.load_index(source_index_path)
|
||||
|
||||
source_dimension = getattr(source._backend_store, "dimension", None)
|
||||
if source_dimension and "dimension" not in dest_cfg:
|
||||
dest_cfg["dimension"] = source_dimension
|
||||
|
||||
dest = VectorStore(backend=to_backend, config=dest_cfg)
|
||||
if dest_index_path and Path(dest_index_path).exists():
|
||||
dest._backend_store.load_index(dest_index_path)
|
||||
|
||||
migrated = 0
|
||||
vectors_batch: List[Any] = []
|
||||
metadata_batch: List[Dict[str, Any]] = []
|
||||
ids_batch: List[str] = []
|
||||
|
||||
def _flush() -> None:
|
||||
nonlocal migrated
|
||||
if not vectors_batch:
|
||||
return
|
||||
dest.store_vectors(list(vectors_batch), list(metadata_batch), ids=list(ids_batch))
|
||||
migrated += len(vectors_batch)
|
||||
vectors_batch.clear()
|
||||
metadata_batch.clear()
|
||||
ids_batch.clear()
|
||||
|
||||
for item in source.iter_vectors(batch_size=_MIGRATE_BATCH_SIZE):
|
||||
meta = dict(item.get("metadata") or {})
|
||||
if namespace and "namespace" not in meta:
|
||||
meta["namespace"] = namespace
|
||||
vectors_batch.append(item["vector"])
|
||||
metadata_batch.append(meta)
|
||||
ids_batch.append(item["id"])
|
||||
if len(vectors_batch) >= _MIGRATE_BATCH_SIZE:
|
||||
_flush()
|
||||
_flush()
|
||||
|
||||
if dest_index_path and migrated:
|
||||
dest._backend_store.save_index(dest_index_path)
|
||||
|
||||
result = {"from": from_backend, "to": to_backend, "migrated": migrated}
|
||||
if _is_json(cli_ctx, local_json):
|
||||
_jecho(result)
|
||||
else:
|
||||
_ok(cli_ctx, f"Migrated {migrated} vectors from {from_backend} to {to_backend}")
|
||||
|
||||
_run_with_error_handling(_action)
|
||||
|
||||
|
||||
@@ -1286,13 +1286,19 @@ class AgentMemory:
|
||||
"""
|
||||
return self.retrieve(content, max_results=limit, **kwargs)
|
||||
|
||||
def find_by_entity(self, entity_id: str, limit: int = 10) -> List[Dict[str, Any]]:
|
||||
def find_by_entity(
|
||||
self, entity_id: str, limit: Optional[int] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Find by entity.
|
||||
|
||||
Args:
|
||||
entity_id: Entity ID to search for
|
||||
limit: Maximum results (default: 10)
|
||||
limit: Maximum results. None (the default) returns ALL matches.
|
||||
The previous default of 10 silently truncated results — an
|
||||
erasure workflow computing "what references this entity"
|
||||
from a truncated page would leave the remainder live
|
||||
(#1018). Callers that want pagination pass an explicit limit.
|
||||
|
||||
Returns:
|
||||
List of memory dicts containing the entity
|
||||
@@ -1308,9 +1314,9 @@ class AgentMemory:
|
||||
if mem_dict:
|
||||
results.append(mem_dict)
|
||||
break
|
||||
if len(results) >= limit:
|
||||
if limit is not None and len(results) >= limit:
|
||||
break
|
||||
return results[:limit]
|
||||
return results if limit is None else results[:limit]
|
||||
|
||||
def find_by_relationship(
|
||||
self, relationship_type: str, limit: int = 10
|
||||
|
||||
@@ -306,10 +306,10 @@ class JSONExporter:
|
||||
|
||||
self.logger.debug(f"Exporting {len(entities)} entity(ies) to JSON")
|
||||
|
||||
# Build JSON data with JSON-LD context
|
||||
# Build JSON data with JSON-LD context. No @vocab: it would expand
|
||||
# every bare key in the caller's entity dicts into ns# (#1146).
|
||||
json_data = {
|
||||
"@context": {
|
||||
"@vocab": "https://semantica.dev/vocab/",
|
||||
"semantica": SEMANTICA_NS,
|
||||
"entities": {"@id": "semantica:entities", "@container": "@list"},
|
||||
},
|
||||
@@ -339,7 +339,6 @@ class JSONExporter:
|
||||
"""
|
||||
json_data = {
|
||||
"@context": {
|
||||
"@vocab": "https://semantica.dev/vocab/",
|
||||
"semantica": SEMANTICA_NS,
|
||||
"relationships": {
|
||||
"@id": "semantica:relationships",
|
||||
@@ -434,11 +433,14 @@ class JSONExporter:
|
||||
Returns:
|
||||
Dictionary in JSON-LD format with @context, @graph/@value, and metadata
|
||||
"""
|
||||
# Initialize JSON-LD structure with context
|
||||
# Initialize JSON-LD structure with context. No @vocab: for a generic
|
||||
# payload it turned whatever bare keys the caller happened to use into
|
||||
# ns# terms (#1146). Undeclared terms now simply expand to nothing,
|
||||
# which is standard JSON-LD behaviour for a context that does not
|
||||
# know them; the raw payload is still in the document.
|
||||
jsonld = {
|
||||
"@context": {
|
||||
"@vocab": "https://semantica.dev/vocab/",
|
||||
"semantica": "https://semantica.dev/ns#",
|
||||
"semantica": SEMANTICA_NS,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -598,13 +600,21 @@ class JSONExporter:
|
||||
Returns:
|
||||
Dictionary in JSON-LD format with @context, @id, @type, and graph data
|
||||
"""
|
||||
# Initialize JSON-LD structure with RDF context
|
||||
# Initialize JSON-LD structure with RDF context. No @vocab: it applied
|
||||
# to every bare term in caller data, so an extracted type like "ORG"
|
||||
# became ns#ORG and a metadata key like "source" collided with the
|
||||
# real sem:source object property (#1146). Only explicit semantica:
|
||||
# terms resolve now, and the caller's metadata dict is typed @json so
|
||||
# it survives as one rdf:JSON literal instead of expanding its keys.
|
||||
jsonld = {
|
||||
"@context": {
|
||||
"@vocab": "https://semantica.dev/vocab/",
|
||||
"semantica": "https://semantica.dev/ns#",
|
||||
"semantica": SEMANTICA_NS,
|
||||
"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
|
||||
"rdfs": "http://www.w3.org/2000/01/rdf-schema#",
|
||||
"semantica:metadata": {
|
||||
"@id": "semantica:metadata",
|
||||
"@type": "@json",
|
||||
},
|
||||
},
|
||||
# Minted from the graph's own content rather than the wall clock
|
||||
# (#1147): re-exporting an unchanged graph must produce the same
|
||||
@@ -664,14 +674,23 @@ class JSONExporter:
|
||||
entity_text = entity.get("text") or entity.get("label", "unknown")
|
||||
entity_id = entity.get("id") or mint_entity_iri(entity_text)
|
||||
|
||||
# The caller's type label is data, not a class we define: minting it
|
||||
# into @type expanded it through @vocab into ns#ORG and friends, terms
|
||||
# that look official but do not exist (#1146). The node is always a
|
||||
# semantica:Entity and the label travels as semantica:type, exactly
|
||||
# how _relationship_to_jsonld has always carried the relationship type.
|
||||
jsonld = {
|
||||
"@id": entity_id,
|
||||
"@type": entity.get("type") or "semantica:Entity",
|
||||
"@type": "semantica:Entity",
|
||||
"semantica:text": entity.get("text") or entity.get("label", ""),
|
||||
"semantica:confidence": entity.get("confidence", 1.0),
|
||||
}
|
||||
entity_type = entity.get("type")
|
||||
if entity_type:
|
||||
jsonld["semantica:type"] = entity_type
|
||||
|
||||
# Add metadata if present
|
||||
# Add metadata if present. The @json term definition on
|
||||
# semantica:metadata keeps the whole dict one rdf:JSON literal.
|
||||
if "metadata" in entity:
|
||||
jsonld["semantica:metadata"] = entity["metadata"]
|
||||
|
||||
|
||||
@@ -1226,11 +1226,14 @@ class RDFSerializer:
|
||||
metadata_terms = _resolve_metadata_terms(options.pop("metadata_terms", None))
|
||||
graph_uri: Optional[str] = options.pop("graph_uri", None)
|
||||
|
||||
# Initialize JSON-LD structure with context
|
||||
# Initialize JSON-LD structure with context. No @vocab: it applied to
|
||||
# every bare term in caller data, so an extracted type like "ORG"
|
||||
# became ns#ORG and a metadata key like "source" collided with the
|
||||
# real sem:source object property (#1146). Only explicit semantica:
|
||||
# terms resolve now.
|
||||
jsonld = {
|
||||
"@context": {
|
||||
"@vocab": "https://semantica.dev/vocab/",
|
||||
"semantica": "https://semantica.dev/ns#",
|
||||
"semantica": SEMANTICA_NS,
|
||||
"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
|
||||
"rdfs": "http://www.w3.org/2000/01/rdf-schema#",
|
||||
},
|
||||
@@ -1252,11 +1255,20 @@ class RDFSerializer:
|
||||
# and was dropped in full by a JSON-LD parser, silently.
|
||||
entity_id = entity.get("id") or mint_entity_iri(entity.get("text", ""))
|
||||
|
||||
# The caller's type label is data, not a class we define: minting
|
||||
# it into @type expanded it through @vocab into ns#ORG and
|
||||
# friends, terms that look official but do not exist (#1146).
|
||||
# The node is always a semantica:Entity and the label travels as
|
||||
# semantica:type, matching the relationship node below and
|
||||
# JSONExporter._entity_to_jsonld.
|
||||
node = {
|
||||
"@id": entity_id,
|
||||
"@type": entity.get("type", "semantica:Entity"),
|
||||
"@type": "semantica:Entity",
|
||||
"semantica:text": entity.get("text") or entity.get("label", ""),
|
||||
}
|
||||
entity_type = entity.get("type")
|
||||
if entity_type:
|
||||
node["semantica:type"] = entity_type
|
||||
confidence = normalize_confidence(entity.get("confidence", 1.0))
|
||||
if confidence is None:
|
||||
self.logger.warning(
|
||||
|
||||
@@ -130,7 +130,10 @@ Example Usage:
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
from typing import Any, Dict, Tuple
|
||||
from typing import TYPE_CHECKING, Any, Dict, Tuple
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .salesforce_ingestor import SalesforceConnector, SalesforceData, SalesforceIngestor
|
||||
|
||||
from .config import IngestConfig, ingest_config
|
||||
from .file_ingestor import (
|
||||
@@ -152,6 +155,7 @@ from .methods import (
|
||||
ingest_parquet,
|
||||
ingest_public_api,
|
||||
ingest_repository,
|
||||
ingest_salesforce,
|
||||
ingest_stream,
|
||||
ingest_web,
|
||||
ingest_xml,
|
||||
@@ -235,6 +239,10 @@ _LAZY_EXPORTS: Dict[str, Tuple[str, str]] = {
|
||||
# XML ingestion
|
||||
"XMLIngestor": (".xml_ingestor", "XMLIngestor"),
|
||||
"XMLIngestionData": (".xml_ingestor", "XMLIngestionData"),
|
||||
# Salesforce ingestion
|
||||
"SalesforceIngestor": (".salesforce_ingestor", "SalesforceIngestor"),
|
||||
"SalesforceData": (".salesforce_ingestor", "SalesforceData"),
|
||||
"SalesforceConnector": (".salesforce_ingestor", "SalesforceConnector"),
|
||||
}
|
||||
|
||||
_OPTIONAL_DEPENDENCY_MESSAGES = {
|
||||
@@ -262,6 +270,11 @@ _OPTIONAL_DEPENDENCY_MESSAGES = {
|
||||
"Arrow ingestion requires optional dependency 'pyarrow'. "
|
||||
"Install it before importing ArrowIngestor or using ingest_arrow()."
|
||||
),
|
||||
".salesforce_ingestor": (
|
||||
"Salesforce ingestion requires optional dependency 'simple-salesforce'. "
|
||||
"Install it with: pip install \"semantica[db-salesforce]\" "
|
||||
"or: pip install simple-salesforce>=1.12.0"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@@ -276,7 +289,7 @@ def __getattr__(name: str) -> Any:
|
||||
except ModuleNotFoundError as exc:
|
||||
message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)
|
||||
missing_name = getattr(exc, "name", None)
|
||||
if message and missing_name in {"git", "bs4", "pyarrow"}:
|
||||
if message and missing_name in {"git", "bs4", "pyarrow", "simple_salesforce"}:
|
||||
raise ImportError(message) from exc
|
||||
raise
|
||||
|
||||
@@ -366,6 +379,10 @@ __all__ = [
|
||||
# XML ingestion
|
||||
"XMLIngestor",
|
||||
"XMLIngestionData",
|
||||
# Salesforce ingestion
|
||||
"SalesforceIngestor",
|
||||
"SalesforceData",
|
||||
"SalesforceConnector",
|
||||
# Registry and Methods
|
||||
"MethodRegistry",
|
||||
"method_registry",
|
||||
@@ -377,6 +394,7 @@ __all__ = [
|
||||
"ingest_repository",
|
||||
"ingest_email",
|
||||
"ingest_database",
|
||||
"ingest_salesforce",
|
||||
"ingest_ontology",
|
||||
"ingest_arrow",
|
||||
"ingest_parquet",
|
||||
|
||||
@@ -186,8 +186,13 @@ class IngestConfig:
|
||||
self._method_configs[method] = config
|
||||
|
||||
def get_method_config(self, method: str) -> Dict:
|
||||
"""Get method-specific configuration."""
|
||||
return self._method_configs.get(method, {})
|
||||
"""Get method-specific configuration.
|
||||
|
||||
Returns a **copy** of the stored method configuration so callers can
|
||||
safely mutate it (e.g. to merge per-call options) without poisoning the
|
||||
global configuration for subsequent calls.
|
||||
"""
|
||||
return dict(self._method_configs.get(method, {}))
|
||||
|
||||
def get_all(self) -> Dict[str, Any]:
|
||||
"""Get all configuration."""
|
||||
|
||||
@@ -875,56 +875,95 @@ schema = connector.get_schema(engine)
|
||||
print(f" {table_name}: {[col['name'] for col in columns]}")
|
||||
```
|
||||
|
||||
## SAP OData Ingestion
|
||||
## Salesforce CRM Ingestion
|
||||
|
||||
`SAPIngestor` reads an Entity Set from a SAP OData service — S/4HANA Cloud,
|
||||
SuccessFactors, or an on-prem NetWeaver Gateway over its REST surface. It
|
||||
follows OData v2/v4 server-driven pagination and flattens each record into a
|
||||
document dict via `export_as_documents()`.
|
||||
Salesforce ingestion requires `simple-salesforce`:
|
||||
|
||||
Install with `pip install 'semantica[ingest-sap]'`.
|
||||
|
||||
### Connector Construction & Authentication
|
||||
|
||||
```python
|
||||
from semantica.ingest import SAPIngestor
|
||||
|
||||
# OAuth2 client-credentials (BTP / S/4HANA Cloud)
|
||||
ing = SAPIngestor(
|
||||
base_url="https://my-sap.example.com/sap/opu/odata/sap/API_BUSINESS_PARTNER",
|
||||
client_id="...", client_secret="...",
|
||||
token_url="https://my-sap.example.com/oauth/token",
|
||||
)
|
||||
# On-prem NetWeaver often uses Basic auth instead — swap the block above for:
|
||||
# ing = SAPIngestor(base_url="...", username="erp_user", password="...")
|
||||
```bash
|
||||
pip install "semantica[db-salesforce]"
|
||||
```
|
||||
|
||||
### Entity-Set Ingestion & Document Export
|
||||
### Basic Usage
|
||||
|
||||
```python
|
||||
# 1. Discover entity sets + field types from $metadata
|
||||
sets = ing.discover_service()
|
||||
from semantica.ingest import SalesforceIngestor
|
||||
import os
|
||||
|
||||
# 2. Page-walk an Entity Set (v2/v4 next links handled automatically)
|
||||
partners = ing.ingest_entity_set(
|
||||
entity_set="A_BusinessPartnerSet",
|
||||
select="BusinessPartner,BusinessPartnerFullName",
|
||||
top=1000,
|
||||
ingestor = SalesforceIngestor(
|
||||
username=os.getenv("SALESFORCE_USERNAME"),
|
||||
password=os.getenv("SALESFORCE_PASSWORD"),
|
||||
security_token=os.getenv("SALESFORCE_SECURITY_TOKEN"),
|
||||
domain="login", # "test" for sandbox
|
||||
)
|
||||
|
||||
# 3. Flatten to document dicts that GraphBuilder can consume directly
|
||||
docs = ing.export_as_documents(partners)
|
||||
# Ingest Account records
|
||||
data = ingestor.ingest_sobject(
|
||||
"Account",
|
||||
fields=["Id", "Name", "Industry", "BillingCity"],
|
||||
where="Type = 'Customer'",
|
||||
limit=5000,
|
||||
)
|
||||
print(f"Retrieved {data.row_count} of {data.total_size} matching records")
|
||||
```
|
||||
|
||||
- Use `expand="to_Item"` on a sales-order header set to pull nested line items
|
||||
in one request — handy for modeling order → line-item → material relations.
|
||||
- Every outbound request, including the OAuth2 token exchange, is routed through
|
||||
the SSRF guard, and pagination never follows a next link that points to a
|
||||
different host than the service root.
|
||||
`SalesforceIngestor()` with no arguments reads from `SALESFORCE_USERNAME`, `SALESFORCE_PASSWORD`, `SALESFORCE_SECURITY_TOKEN`, and `SALESFORCE_DOMAIN` environment variables automatically.
|
||||
|
||||
### Custom Objects and Raw SOQL
|
||||
|
||||
```python
|
||||
# Custom object (API name ends in __c)
|
||||
data = ingestor.ingest_sobject("My_Custom_Object__c", fields=["Id", "Name", "Custom_Field__c"])
|
||||
|
||||
# Raw SOQL GÇö pagination is handled automatically
|
||||
data = ingestor.ingest_query("""
|
||||
SELECT Id, Name, StageName, Amount
|
||||
FROM Opportunity
|
||||
WHERE IsClosed = false
|
||||
ORDER BY CloseDate ASC
|
||||
""")
|
||||
print(f"Open opportunities: {data.row_count}")
|
||||
```
|
||||
|
||||
### Document Export
|
||||
|
||||
```python
|
||||
documents = ingestor.export_as_documents(
|
||||
data,
|
||||
id_field="Id", # Salesforce 18-char record Id
|
||||
text_fields=["Name", "Description"],
|
||||
)
|
||||
# Each document: {"id": "001...", "text": "...", "metadata": {"source": "salesforce", ...}}
|
||||
```
|
||||
|
||||
### Convenience Function
|
||||
|
||||
```python
|
||||
from semantica.ingest import ingest_salesforce
|
||||
|
||||
# Fetch records
|
||||
data = ingest_salesforce(
|
||||
method="sobject",
|
||||
sobject_name="Account",
|
||||
fields=["Id", "Name"],
|
||||
limit=500,
|
||||
)
|
||||
|
||||
# Ingest and export as documents in one call
|
||||
docs = ingest_salesforce(
|
||||
method="documents",
|
||||
sobject_name="Account",
|
||||
text_fields=["Name", "Description"],
|
||||
)
|
||||
|
||||
# Using the unified dispatcher
|
||||
from semantica.ingest import ingest
|
||||
result = ingest(None, source_type="salesforce", method="sobject",
|
||||
sobject_name="Account", fields=["Id", "Name"])
|
||||
data = result["data"]
|
||||
```
|
||||
|
||||
See [Salesforce Integration](https://docs.getsemantica.ai/integrations/salesforce) for full documentation including sandbox, schema discovery, pagination details, and troubleshooting.
|
||||
|
||||
> **Security Note:** Never hardcode credentials (`client_secret`, `password`);
|
||||
> pass them via environment variables (`SAP_CLIENT_SECRET`, `SAP_PASSWORD`) or a
|
||||
> secrets manager.
|
||||
|
||||
## MCP Server Ingestion
|
||||
|
||||
@@ -1664,3 +1703,54 @@ for source_type, source_list in sources.items():
|
||||
for batch in process_in_batches(large_dataset, batch_size=1000):
|
||||
result = ingest(batch)
|
||||
```
|
||||
|
||||
## SAP OData Ingestion
|
||||
|
||||
`SAPIngestor` reads an Entity Set from a SAP OData service — S/4HANA Cloud,
|
||||
SuccessFactors, or an on-prem NetWeaver Gateway over its REST surface. It
|
||||
follows OData v2/v4 server-driven pagination and flattens each record into a
|
||||
document dict via `export_as_documents()`.
|
||||
|
||||
Install with `pip install 'semantica[ingest-sap]'`.
|
||||
|
||||
### Connector Construction & Authentication
|
||||
|
||||
```python
|
||||
from semantica.ingest import SAPIngestor
|
||||
|
||||
# OAuth2 client-credentials (BTP / S/4HANA Cloud)
|
||||
ing = SAPIngestor(
|
||||
base_url="https://my-sap.example.com/sap/opu/odata/sap/API_BUSINESS_PARTNER",
|
||||
client_id="...", client_secret="...",
|
||||
token_url="https://my-sap.example.com/oauth/token",
|
||||
)
|
||||
# On-prem NetWeaver often uses Basic auth instead — swap the block above for:
|
||||
# ing = SAPIngestor(base_url="...", username="erp_user", password="...")
|
||||
```
|
||||
|
||||
### Entity-Set Ingestion & Document Export
|
||||
|
||||
```python
|
||||
# 1. Discover entity sets + field types from $metadata
|
||||
sets = ing.discover_service()
|
||||
|
||||
# 2. Page-walk an Entity Set (v2/v4 next links handled automatically)
|
||||
partners = ing.ingest_entity_set(
|
||||
entity_set="A_BusinessPartnerSet",
|
||||
select="BusinessPartner,BusinessPartnerFullName",
|
||||
top=1000,
|
||||
)
|
||||
|
||||
# 3. Flatten to document dicts that GraphBuilder can consume directly
|
||||
docs = ing.export_as_documents(partners)
|
||||
```
|
||||
|
||||
- Use `expand="to_Item"` on a sales-order header set to pull nested line items
|
||||
in one request — handy for modeling order → line-item → material relations.
|
||||
- Every outbound request, including the OAuth2 token exchange, is routed through
|
||||
the SSRF guard, and pagination never follows a next link that points to a
|
||||
different host than the service root.
|
||||
|
||||
> **Security Note:** Never hardcode credentials (`client_secret`, `password`);
|
||||
> pass them via environment variables (`SAP_CLIENT_SECRET`, `SAP_PASSWORD`) or a
|
||||
> secrets manager.
|
||||
|
||||
@@ -203,6 +203,7 @@ if TYPE_CHECKING:
|
||||
from .ontology_ingestor import OntologyData
|
||||
from .parquet_ingestor import ParquetData
|
||||
from .public_api_ingestor import PublicAPIDetection
|
||||
from .salesforce_ingestor import SalesforceData
|
||||
from .stream_ingestor import StreamProcessor
|
||||
from .web_ingestor import WebContent
|
||||
from .xml_ingestor import XMLIngestionData
|
||||
@@ -1126,6 +1127,213 @@ def ingest_database(
|
||||
raise
|
||||
|
||||
|
||||
def ingest_salesforce(
|
||||
source: Optional[Dict[str, Any]] = None,
|
||||
method: str = "sobject",
|
||||
**kwargs,
|
||||
) -> Union["SalesforceData", List[Dict[str, Any]], Dict[str, Any]]:
|
||||
"""Ingest data from Salesforce CRM (convenience function).
|
||||
|
||||
A user-friendly wrapper around :class:`~semantica.ingest.SalesforceIngestor`
|
||||
that connects, ingests, and returns data in a single call.
|
||||
|
||||
Args:
|
||||
source: Optional credential/configuration dictionary. Keys mirror the
|
||||
:class:`~semantica.ingest.SalesforceConnector` constructor:
|
||||
``username``, ``password``, ``security_token``, ``domain``
|
||||
(``"login"`` for production, ``"test"`` for sandbox),
|
||||
``instance_url``, ``session_id``, ``api_version``.
|
||||
When ``None``, credentials are read from environment variables
|
||||
(``SALESFORCE_USERNAME`` / ``SALESFORCE_PASSWORD`` /
|
||||
``SALESFORCE_SECURITY_TOKEN`` etc.).
|
||||
method: Ingestion method:
|
||||
|
||||
* ``"sobject"`` *(default)* — fetch records from a named sObject
|
||||
(requires ``sobject_name`` kwarg).
|
||||
* ``"query"`` — execute a raw SOQL query string (requires
|
||||
``soql`` kwarg).
|
||||
* ``"list_sobjects"`` — return a sorted list of accessible sObject
|
||||
API names.
|
||||
* ``"schema"`` — return field metadata for a named sObject
|
||||
(requires ``sobject_name`` kwarg).
|
||||
* ``"documents"`` — ingest an sObject and convert to the Semantica
|
||||
document format in one step (requires ``sobject_name`` kwarg;
|
||||
optional ``text_fields`` and ``id_field`` kwargs).
|
||||
|
||||
**kwargs: Additional options forwarded to the ingestor method.
|
||||
Common kwargs for ``"sobject"`` / ``"documents"``:
|
||||
|
||||
* ``sobject_name`` — Salesforce sObject API name (e.g.
|
||||
``"Account"``, ``"My_Custom__c"``).
|
||||
* ``fields`` — list of field API names to select. When omitted
|
||||
all selectable fields are fetched via ``describe()``.
|
||||
* ``where`` — SOQL ``WHERE`` clause fragment (trusted input only).
|
||||
* ``order_by`` — SOQL ``ORDER BY`` clause fragment.
|
||||
* ``limit`` — maximum number of records.
|
||||
|
||||
For ``"query"``:
|
||||
|
||||
* ``soql`` — full SOQL query string.
|
||||
|
||||
For ``"schema"``:
|
||||
|
||||
* ``sobject_name`` — sObject to describe.
|
||||
|
||||
Returns:
|
||||
* ``"sobject"`` / ``"query"`` → :class:`~semantica.ingest.SalesforceData`
|
||||
* ``"documents"`` → ``List[Dict[str, Any]]`` (Semantica document format)
|
||||
* ``"list_sobjects"`` → ``List[str]``
|
||||
* ``"schema"`` → ``Dict[str, Any]``
|
||||
|
||||
Raises:
|
||||
:class:`~semantica.utils.exceptions.ConfigurationError`: If
|
||||
``simple-salesforce`` is not installed.
|
||||
:class:`~semantica.utils.exceptions.ValidationError`: If credentials
|
||||
are incomplete or an sObject / field name is invalid.
|
||||
:class:`~semantica.utils.exceptions.ProcessingError`: If the
|
||||
Salesforce API call fails.
|
||||
|
||||
Examples::
|
||||
|
||||
>>> from semantica.ingest import ingest_salesforce
|
||||
|
||||
>>> # Fetch Account records (credentials from env vars)
|
||||
>>> data = ingest_salesforce(
|
||||
... method="sobject",
|
||||
... sobject_name="Account",
|
||||
... fields=["Id", "Name", "Industry"],
|
||||
... limit=500,
|
||||
... )
|
||||
|
||||
>>> # Execute a raw SOQL query (credentials from environment variables)
|
||||
>>> data = ingest_salesforce(
|
||||
... method="query",
|
||||
... soql="SELECT Id, Name FROM Contact WHERE IsActive = true",
|
||||
... )
|
||||
|
||||
>>> # Ingest and export as documents for GraphBuilder in one step
|
||||
>>> docs = ingest_salesforce(
|
||||
... method="documents",
|
||||
... sobject_name="Account",
|
||||
... text_fields=["Name", "Description"],
|
||||
... limit=1000,
|
||||
... )
|
||||
|
||||
>>> # List all accessible sObjects in the connected org
|
||||
>>> sobject_names = ingest_salesforce(method="list_sobjects")
|
||||
|
||||
>>> # Use sandbox org
|
||||
>>> data = ingest_salesforce(
|
||||
... method="sobject",
|
||||
... sobject_name="Account",
|
||||
... ) # set SALESFORCE_DOMAIN=test in environment for sandbox
|
||||
"""
|
||||
# Registry hook — allows callers to register a custom "salesforce" method
|
||||
custom_method = method_registry.get("salesforce", method)
|
||||
if custom_method and custom_method != ingest_salesforce:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, source,
|
||||
fallback_on_custom_error=fallback, **kwargs,
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
from .salesforce_ingestor import SalesforceIngestor
|
||||
except ModuleNotFoundError as exc:
|
||||
if _is_missing_dependency(exc, "simple_salesforce"):
|
||||
raise _missing_optional_dependency(
|
||||
"Salesforce ingestion", "simple-salesforce"
|
||||
) from exc
|
||||
raise
|
||||
|
||||
# Unpack credential dict (if given); everything else stays in kwargs.
|
||||
creds: Dict[str, Any] = {}
|
||||
if source is not None:
|
||||
if not isinstance(source, dict):
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() source must be a credential dict or None. "
|
||||
"Pass sobject_name / soql as keyword arguments."
|
||||
)
|
||||
creds = dict(source)
|
||||
|
||||
# Merge any ingest_config method config under "salesforce".
|
||||
# get_method_config() now returns a copy, so this dict is safe to mutate.
|
||||
# We build the final connector config in order of increasing priority:
|
||||
# 1. base method config (lowest — global defaults set by operator)
|
||||
# 2. per-call credential dict supplied via `source`
|
||||
# 3. per-call connector params supplied as kwargs
|
||||
# Credentials are extracted from kwargs and removed so they don't also
|
||||
# flow into the ingest method call (which doesn't understand them).
|
||||
_CONNECTOR_PARAMS = frozenset({
|
||||
"username", "password", "security_token", "domain",
|
||||
"instance_url", "session_id", "api_version",
|
||||
})
|
||||
connector_kwargs = {k: v for k, v in kwargs.items() if k in _CONNECTOR_PARAMS}
|
||||
for k in _CONNECTOR_PARAMS:
|
||||
kwargs.pop(k, None)
|
||||
|
||||
# Build a fresh per-call config dict — never mutate the global store.
|
||||
config: Dict[str, Any] = {
|
||||
**ingest_config.get_method_config("salesforce"), # base (already a copy)
|
||||
**creds, # source dict credentials
|
||||
**connector_kwargs, # kwarg credentials
|
||||
}
|
||||
|
||||
ingestor = SalesforceIngestor(**config)
|
||||
|
||||
if method == "sobject":
|
||||
sobject_name = kwargs.pop("sobject_name", None)
|
||||
if not sobject_name:
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() with method='sobject' requires "
|
||||
"sobject_name keyword argument."
|
||||
)
|
||||
return ingestor.ingest_sobject(sobject_name, **kwargs)
|
||||
|
||||
elif method == "query":
|
||||
soql = kwargs.pop("soql", None)
|
||||
if not soql:
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() with method='query' requires "
|
||||
"soql keyword argument."
|
||||
)
|
||||
return ingestor.ingest_query(soql, **kwargs)
|
||||
|
||||
elif method == "list_sobjects":
|
||||
return ingestor.list_sobjects()
|
||||
|
||||
elif method == "schema":
|
||||
sobject_name = kwargs.pop("sobject_name", None)
|
||||
if not sobject_name:
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() with method='schema' requires "
|
||||
"sobject_name keyword argument."
|
||||
)
|
||||
return ingestor.get_sobject_schema(sobject_name)
|
||||
|
||||
elif method == "documents":
|
||||
sobject_name = kwargs.pop("sobject_name", None)
|
||||
if not sobject_name:
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() with method='documents' requires "
|
||||
"sobject_name keyword argument."
|
||||
)
|
||||
id_field = kwargs.pop("id_field", "Id")
|
||||
text_fields = kwargs.pop("text_fields", None)
|
||||
data = ingestor.ingest_sobject(sobject_name, **kwargs)
|
||||
return ingestor.export_as_documents(data, id_field=id_field,
|
||||
text_fields=text_fields)
|
||||
|
||||
else:
|
||||
raise ProcessingError(
|
||||
f"Unknown ingest_salesforce method: {method!r}. "
|
||||
"Valid methods: 'sobject', 'query', 'list_sobjects', 'schema', "
|
||||
"'documents'."
|
||||
)
|
||||
|
||||
|
||||
def ingest_mcp(
|
||||
source: Union[str, Dict[str, Any]],
|
||||
method: str = "resources",
|
||||
@@ -1306,6 +1514,7 @@ def ingest(
|
||||
- "ontology": Ontology ingestion
|
||||
- "parquet": Apache Parquet file or directory ingestion
|
||||
- "xml": XML file or directory ingestion
|
||||
- "salesforce": Salesforce CRM ingestion (pass credentials via kwargs)
|
||||
method: Optional specific ingestion method
|
||||
**kwargs: Additional options passed to ingestor
|
||||
|
||||
@@ -1428,6 +1637,9 @@ def ingest(
|
||||
return {"ontology": ingest_ontology(sources, method=method or "file", **kwargs)}
|
||||
elif source_type == "mcp":
|
||||
return {"data": ingest_mcp(sources, method=method or "resources", **kwargs)}
|
||||
elif source_type == "salesforce":
|
||||
return {"data": ingest_salesforce(sources,
|
||||
method=method or "sobject", **kwargs)}
|
||||
else:
|
||||
raise ProcessingError(f"Unknown source type: {source_type}")
|
||||
|
||||
@@ -1540,3 +1752,9 @@ method_registry.register("ontology", "file", ingest_ontology)
|
||||
method_registry.register("ontology", "directory", ingest_ontology)
|
||||
method_registry.register("ingest", "default", ingest)
|
||||
method_registry.register("ingest", "unified", ingest)
|
||||
method_registry.register("salesforce", "default", ingest_salesforce)
|
||||
method_registry.register("salesforce", "sobject", ingest_salesforce)
|
||||
method_registry.register("salesforce", "query", ingest_salesforce)
|
||||
method_registry.register("salesforce", "list_sobjects", ingest_salesforce)
|
||||
method_registry.register("salesforce", "schema", ingest_salesforce)
|
||||
method_registry.register("salesforce", "documents", ingest_salesforce)
|
||||
|
||||
@@ -66,6 +66,7 @@ class MethodRegistry:
|
||||
"parquet": {},
|
||||
"arrow": {},
|
||||
"xml": {},
|
||||
"salesforce": {},
|
||||
"ingest": {},
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -10,28 +10,53 @@ Supported Providers:
|
||||
- OpenAI: OpenAI API (GPT-3.5, GPT-4, etc.)
|
||||
- HuggingFaceLLM: HuggingFace Transformers for local LLM inference
|
||||
- LiteLLM: Unified interface to 100+ LLM providers (OpenAI, Anthropic, Groq, Azure, Bedrock, Vertex AI, etc.)
|
||||
- Anthropic: Anthropic Claude API (Claude sonnet, Opus, Haiku, etc.)
|
||||
- Gemini: Google Gemini API
|
||||
- Ollama: Local models served through Ollama
|
||||
- DeepSeek: DeepSeek's OpenAI-compatible API
|
||||
- Novita: Novita AI's OpenAI-compatible API
|
||||
|
||||
Example Usage:
|
||||
>>> from semantica.llms import Groq, OpenAI, HuggingFaceLLM, LiteLLM
|
||||
>>>
|
||||
>>> from semantica.llms import Groq, OpenAI, HuggingFaceLLM, LiteLLM, Anthropic
|
||||
>>>
|
||||
>>> # Groq provider
|
||||
>>> groq = Groq(model="llama-3.1-8b-instant", api_key="your-key")
|
||||
>>> response = groq.generate("Hello, world!")
|
||||
>>>
|
||||
>>>
|
||||
>>> # OpenAI provider
|
||||
>>> openai = OpenAI(model="gpt-4", api_key="your-key")
|
||||
>>> response = openai.generate("Hello, world!")
|
||||
>>>
|
||||
>>>
|
||||
>>> # HuggingFace LLM provider
|
||||
>>> hf = HuggingFaceLLM(model_name="gpt2")
|
||||
>>> response = hf.generate("Hello, world!")
|
||||
>>>
|
||||
>>>
|
||||
>>> # LiteLLM provider (supports 100+ LLMs)
|
||||
>>> llm = LiteLLM(model="openai/gpt-4o", api_key="your-key")
|
||||
>>> response = llm.generate("Hello, world!")
|
||||
>>> # Or use other providers via LiteLLM
|
||||
>>> llm = LiteLLM(model="anthropic/claude-sonnet-4-20250514")
|
||||
>>> response = llm.generate("Hello, world!")
|
||||
>>>
|
||||
>>> # Anthropic provider
|
||||
>>> claude = Anthropic(model="claude-sonnet-4-6", api_key="the-key")
|
||||
>>> response = claude.generate("Hello, world!")
|
||||
>>>
|
||||
>>> # Gemini provider
|
||||
>>> gemini = Gemini(model="gemini-pro", api_key="your-key")
|
||||
>>> response = gemini.generate("Hello, world!")
|
||||
>>>
|
||||
>>> # Ollama provider (local, no api_key)
|
||||
>>> ollama = Ollama(model="llama2")
|
||||
>>> response = ollama.generate("Hello, world!")
|
||||
>>>
|
||||
>>> # DeepSeek provider
|
||||
>>> deepseek = DeepSeek(model="deepseek-chat", api_key="your-key")
|
||||
>>> response = deepseek.generate("Hello, world!")
|
||||
>>>
|
||||
>>> # Novita provider
|
||||
>>> novita = Novita(model="deepseek/deepseek-v3.2", api_key="your-key")
|
||||
>>> response = novita.generate("Hello, world!")
|
||||
|
||||
Author: Semantica Contributors
|
||||
License: MIT
|
||||
@@ -41,6 +66,20 @@ from .groq import Groq
|
||||
from .openai import OpenAI
|
||||
from .huggingface import HuggingFaceLLM
|
||||
from .litellm import LiteLLM
|
||||
from .anthropic import Anthropic
|
||||
from .gemini import Gemini
|
||||
from .ollama import Ollama
|
||||
from .deepseek import DeepSeek
|
||||
from .novita import Novita
|
||||
|
||||
__all__ = ["Groq", "OpenAI", "HuggingFaceLLM", "LiteLLM"]
|
||||
|
||||
__all__ = [
|
||||
"Groq",
|
||||
"OpenAI",
|
||||
"HuggingFaceLLM",
|
||||
"LiteLLM",
|
||||
"Anthropic",
|
||||
"Gemini",
|
||||
"Ollama",
|
||||
"DeepSeek",
|
||||
"Novita",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
"""
|
||||
Anthropic LLM Provider
|
||||
|
||||
Wrapper for Anthropic Claude API provider with clean interface
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from ..semantic_extract.providers import AnthropicProvider
|
||||
from ..utils.exceptions import ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
|
||||
logger = get_logger("llms.anthropic")
|
||||
|
||||
|
||||
class Anthropic:
|
||||
"""
|
||||
Anthropic Claude LLM provider wrapper.
|
||||
|
||||
Provides clean interface to Anthropic's Claude API.
|
||||
|
||||
Example:
|
||||
>>> from semantica.llms import Anthropic
|
||||
>>> claude = Anthropic(model="claude-sonnet-4-6", api_key="the-key")
|
||||
>>> response = claude.generate("What is API key?")
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: str = "claude-sonnet-4-6",
|
||||
api_key: Optional[str] = None,
|
||||
**kwargs
|
||||
):
|
||||
"""
|
||||
Initialize Anthropic provider.
|
||||
|
||||
Args:
|
||||
model: Model name (default: claude-sonnet-4-6)
|
||||
api_key: Anthropic API key (default: from ANTHROPIC_API_KEY env var)
|
||||
**kwargs: Additional provider options
|
||||
"""
|
||||
self.provider = AnthropicProvider(api_key=api_key, model=model, **kwargs)
|
||||
self.model = model
|
||||
self.api_key = api_key
|
||||
|
||||
def is_available(self) -> bool:
|
||||
"""Check if Anthropic provider is available."""
|
||||
return self.provider.is_available()
|
||||
|
||||
def generate(self, prompt: str, **kwargs) -> str:
|
||||
"""
|
||||
Generate text from prompt.
|
||||
|
||||
Args:
|
||||
prompt: Input prompt text
|
||||
**kwargs: Generation options (temperature, max_tokens, etc.)
|
||||
|
||||
Returns:
|
||||
Generated text response
|
||||
|
||||
Raises:
|
||||
ProcessingError: If provider is not available or generation fails
|
||||
"""
|
||||
if not self.is_available():
|
||||
raise ProcessingError(
|
||||
"Anthropic provider not available. Set ANTHROPIC_API_KEY or pass api_key."
|
||||
)
|
||||
return self.provider.generate(prompt, **kwargs)
|
||||
|
||||
def generate_structured(self, prompt: str, **kwargs) -> Union[Dict[str, Any], List[Any]]:
|
||||
"""
|
||||
Generates structured JSON output.
|
||||
|
||||
Args:
|
||||
prompt: Input prompt text
|
||||
**kwargs: Generation options
|
||||
|
||||
Returns:
|
||||
Parsed JSON response. A dict for a top-level JSON object, or a
|
||||
list if the model returns a top-level JSON array.
|
||||
|
||||
Raises:
|
||||
ProcessingError: If provider is not available or generation fails
|
||||
"""
|
||||
if not self.is_available():
|
||||
raise ProcessingError(
|
||||
"Anthropic provider not available. Set ANTHROPIC_API_KEY or pass api_key."
|
||||
)
|
||||
return self.provider.generate_structured(prompt, **kwargs)
|
||||
|
||||
def generate_typed(self, prompt: str, schema: Any, max_retries: int = 3, **kwargs) -> Any:
|
||||
"""
|
||||
Generate output validated against a Pydantic schema.
|
||||
|
||||
Args:
|
||||
prompt: Input prompt text
|
||||
schema: Pydantic model class to validate the output against
|
||||
max_retries: Number of retries if validation fails (default: 3)
|
||||
**kwargs: Generation options
|
||||
|
||||
Returns:
|
||||
An instance of `schema`, populated from the model's response
|
||||
|
||||
Raises:
|
||||
ProcessingError: If provider is not available or generation fails
|
||||
"""
|
||||
if not self.is_available():
|
||||
raise ProcessingError(
|
||||
"Anthropic provider not available. Set ANTHROPIC_API_KEY or pass api_key."
|
||||
)
|
||||
return self.provider.generate_typed(prompt, schema, max_retries=max_retries, **kwargs)
|
||||
@@ -0,0 +1,111 @@
|
||||
"""
|
||||
DeepSeek LLM Provider
|
||||
|
||||
Wrapper for DeepSeek API provider with clean interface.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from ..semantic_extract.providers import DeepSeekProvider
|
||||
from ..utils.exceptions import ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
|
||||
logger = get_logger("llms.deepseek")
|
||||
|
||||
|
||||
class DeepSeek:
|
||||
"""
|
||||
DeepSeek LLM provider wrapper.
|
||||
|
||||
Provides clean interface to DeepSeek's OpenAI-compatible API.
|
||||
|
||||
Example:
|
||||
>>> from semantica.llms import DeepSeek
|
||||
>>> llm = DeepSeek(model="deepseek-chat", api_key="your-key")
|
||||
>>> response = llm.generate("What is AI?")
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: str = "deepseek-chat",
|
||||
api_key: Optional[str] = None,
|
||||
**kwargs
|
||||
):
|
||||
"""
|
||||
Initialize DeepSeek provider.
|
||||
|
||||
Args:
|
||||
model: Model name (default: "deepseek-chat")
|
||||
api_key: DeepSeek API key (default: from DEEPSEEK_API_KEY env var)
|
||||
**kwargs: Additional provider options
|
||||
"""
|
||||
self.provider = DeepSeekProvider(api_key=api_key, model=model, **kwargs)
|
||||
self.model = model
|
||||
self.api_key = api_key
|
||||
|
||||
def is_available(self) -> bool:
|
||||
"""Check if DeepSeek provider is available."""
|
||||
return self.provider.is_available()
|
||||
|
||||
def generate(self, prompt: str, **kwargs) -> str:
|
||||
"""
|
||||
Generate text from prompt.
|
||||
|
||||
Args:
|
||||
prompt: Input prompt text
|
||||
**kwargs: Generation options (temperature, max_tokens, etc.)
|
||||
|
||||
Returns:
|
||||
Generated text response
|
||||
|
||||
Raises:
|
||||
ProcessingError: If provider is not available or generation fails
|
||||
"""
|
||||
if not self.is_available():
|
||||
raise ProcessingError(
|
||||
"DeepSeek provider not available. Set DEEPSEEK_API_KEY or pass api_key."
|
||||
)
|
||||
return self.provider.generate(prompt, **kwargs)
|
||||
|
||||
def generate_structured(self, prompt: str, **kwargs) -> Union[Dict[str, Any], List[Any]]:
|
||||
"""
|
||||
Generate structured JSON output.
|
||||
|
||||
Args:
|
||||
prompt: Input prompt text
|
||||
**kwargs: Generation options
|
||||
|
||||
Returns:
|
||||
Parsed JSON response. A dict for a top-level JSON object, or a
|
||||
list if the model returns a top-level JSON array.
|
||||
|
||||
Raises:
|
||||
ProcessingError: If provider is not available or generation fails
|
||||
"""
|
||||
if not self.is_available():
|
||||
raise ProcessingError(
|
||||
"DeepSeek provider not available. Set DEEPSEEK_API_KEY or pass api_key."
|
||||
)
|
||||
return self.provider.generate_structured(prompt, **kwargs)
|
||||
|
||||
def generate_typed(self, prompt: str, schema: Any, max_retries: int = 3, **kwargs) -> Any:
|
||||
"""
|
||||
Generate output validated against a Pydantic schema.
|
||||
|
||||
Args:
|
||||
prompt: Input prompt text
|
||||
schema: Pydantic model class to validate the output against
|
||||
max_retries: Number of retries if validation fails (default: 3)
|
||||
**kwargs: Generation options
|
||||
|
||||
Returns:
|
||||
An instance of `schema`, populated from the model's response
|
||||
|
||||
Raises:
|
||||
ProcessingError: If provider is not available or generation fails
|
||||
"""
|
||||
if not self.is_available():
|
||||
raise ProcessingError(
|
||||
"DeepSeek provider not available. Set DEEPSEEK_API_KEY or pass api_key."
|
||||
)
|
||||
return self.provider.generate_typed(prompt, schema, max_retries=max_retries, **kwargs)
|
||||
@@ -0,0 +1,111 @@
|
||||
"""
|
||||
Gemini LLM Provider
|
||||
|
||||
Wrapper for Google Gemini API provider with clean interface.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from ..semantic_extract.providers import GeminiProvider
|
||||
from ..utils.exceptions import ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
|
||||
logger = get_logger("llms.gemini")
|
||||
|
||||
|
||||
class Gemini:
|
||||
"""
|
||||
Google Gemini LLM provider wrapper.
|
||||
|
||||
Provides clean interface to Google's Gemini API.
|
||||
|
||||
Example:
|
||||
>>> from semantica.llms import Gemini
|
||||
>>> gemini = Gemini(model="gemini-pro", api_key="your-key")
|
||||
>>> response = gemini.generate("What is AI?")
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: str = "gemini-pro",
|
||||
api_key: Optional[str] = None,
|
||||
**kwargs
|
||||
):
|
||||
"""
|
||||
Initialize Gemini provider.
|
||||
|
||||
Args:
|
||||
model: Model name (default: "gemini-pro")
|
||||
api_key: Gemini API key (default: from GEMINI_API_KEY env var)
|
||||
**kwargs: Additional provider options
|
||||
"""
|
||||
self.provider = GeminiProvider(api_key=api_key, model=model, **kwargs)
|
||||
self.model = model
|
||||
self.api_key = api_key
|
||||
|
||||
def is_available(self) -> bool:
|
||||
"""Check if Gemini provider is available."""
|
||||
return self.provider.is_available()
|
||||
|
||||
def generate(self, prompt: str, **kwargs) -> str:
|
||||
"""
|
||||
Generate text from prompt.
|
||||
|
||||
Args:
|
||||
prompt: Input prompt text
|
||||
**kwargs: Generation options (temperature, max_tokens, etc.)
|
||||
|
||||
Returns:
|
||||
Generated text response
|
||||
|
||||
Raises:
|
||||
ProcessingError: If provider is not available or generation fails
|
||||
"""
|
||||
if not self.is_available():
|
||||
raise ProcessingError(
|
||||
"Gemini provider not available. Set GEMINI_API_KEY or pass api_key."
|
||||
)
|
||||
return self.provider.generate(prompt, **kwargs)
|
||||
|
||||
def generate_structured(self, prompt: str, **kwargs) -> Union[Dict[str, Any], List[Any]]:
|
||||
"""
|
||||
Generate structured JSON output.
|
||||
|
||||
Args:
|
||||
prompt: Input prompt text
|
||||
**kwargs: Generation options
|
||||
|
||||
Returns:
|
||||
Parsed JSON response. A dict for a top-level JSON object, or a
|
||||
list if the model returns a top-level JSON array.
|
||||
|
||||
Raises:
|
||||
ProcessingError: If provider is not available or parsing fails
|
||||
"""
|
||||
if not self.is_available():
|
||||
raise ProcessingError(
|
||||
"Gemini provider not available. Set GEMINI_API_KEY or pass api_key."
|
||||
)
|
||||
return self.provider.generate_structured(prompt, **kwargs)
|
||||
|
||||
def generate_typed(self, prompt: str, schema: Any, max_retries: int = 3, **kwargs) -> Any:
|
||||
"""
|
||||
Generate output validated against a Pydantic schema.
|
||||
|
||||
Args:
|
||||
prompt: Input prompt text
|
||||
schema: Pydantic model class to validate the output against
|
||||
max_retries: Number of retries if validation fails (default: 3)
|
||||
**kwargs: Generation options
|
||||
|
||||
Returns:
|
||||
An instance of `schema`, populated from the model's response
|
||||
|
||||
Raises:
|
||||
ProcessingError: If provider is not available or generation fails
|
||||
"""
|
||||
if not self.is_available():
|
||||
raise ProcessingError(
|
||||
"Gemini provider not available. Set GEMINI_API_KEY or pass api_key."
|
||||
)
|
||||
return self.provider.generate_typed(prompt, schema, max_retries=max_retries, **kwargs)
|
||||
@@ -0,0 +1,111 @@
|
||||
"""
|
||||
Novita LLM Provider
|
||||
|
||||
Wrapper for Novita AI's OpenAI-compatible API with clean interface.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from ..semantic_extract.providers import NovitaProvider
|
||||
from ..utils.exceptions import ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
|
||||
logger = get_logger("llms.novita")
|
||||
|
||||
|
||||
class Novita:
|
||||
"""
|
||||
Novita AI LLM provider wrapper.
|
||||
|
||||
Provides clean interface to Novita's OpenAI-compatible API.
|
||||
|
||||
Example:
|
||||
>>> from semantica.llms import Novita
|
||||
>>> llm = Novita(model="deepseek/deepseek-v3.2", api_key="your-key")
|
||||
>>> response = llm.generate("What is AI?")
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: str = "deepseek/deepseek-v3.2",
|
||||
api_key: Optional[str] = None,
|
||||
**kwargs
|
||||
):
|
||||
"""
|
||||
Initialize Novita provider.
|
||||
|
||||
Args:
|
||||
model: Model name (default: "deepseek/deepseek-v3.2")
|
||||
api_key: Novita API key (default: from NOVITA_API_KEY env var)
|
||||
**kwargs: Additional provider options
|
||||
"""
|
||||
self.provider = NovitaProvider(api_key=api_key, model=model, **kwargs)
|
||||
self.model = model
|
||||
self.api_key = api_key
|
||||
|
||||
def is_available(self) -> bool:
|
||||
"""Check if Novita provider is available."""
|
||||
return self.provider.is_available()
|
||||
|
||||
def generate(self, prompt: str, **kwargs) -> str:
|
||||
"""
|
||||
Generate text from prompt.
|
||||
|
||||
Args:
|
||||
prompt: Input prompt text
|
||||
**kwargs: Generation options (temperature, max_tokens, etc.)
|
||||
|
||||
Returns:
|
||||
Generated text response
|
||||
|
||||
Raises:
|
||||
ProcessingError: If provider is not available or generation fails
|
||||
"""
|
||||
if not self.is_available():
|
||||
raise ProcessingError(
|
||||
"Novita provider not available. Set NOVITA_API_KEY or pass api_key."
|
||||
)
|
||||
return self.provider.generate(prompt, **kwargs)
|
||||
|
||||
def generate_structured(self, prompt: str, **kwargs) -> Union[Dict[str, Any], List[Any]]:
|
||||
"""
|
||||
Generate structured JSON output.
|
||||
|
||||
Args:
|
||||
prompt: Input prompt text
|
||||
**kwargs: Generation options
|
||||
|
||||
Returns:
|
||||
Parsed JSON response. A dict for a top-level JSON object, or a
|
||||
list if the model returns a top-level JSON array.
|
||||
|
||||
Raises:
|
||||
ProcessingError: If provider is not available or generation fails
|
||||
"""
|
||||
if not self.is_available():
|
||||
raise ProcessingError(
|
||||
"Novita provider not available. Set NOVITA_API_KEY or pass api_key."
|
||||
)
|
||||
return self.provider.generate_structured(prompt, **kwargs)
|
||||
|
||||
def generate_typed(self, prompt: str, schema: Any, max_retries: int = 3, **kwargs) -> Any:
|
||||
"""
|
||||
Generate output validated against a Pydantic schema.
|
||||
|
||||
Args:
|
||||
prompt: Input prompt text
|
||||
schema: Pydantic model class to validate the output against
|
||||
max_retries: Number of retries if validation fails (default: 3)
|
||||
**kwargs: Generation options
|
||||
|
||||
Returns:
|
||||
An instance of `schema`, populated from the model's response
|
||||
|
||||
Raises:
|
||||
ProcessingError: If provider is not available or generation fails
|
||||
"""
|
||||
if not self.is_available():
|
||||
raise ProcessingError(
|
||||
"Novita provider not available. Set NOVITA_API_KEY or pass api_key."
|
||||
)
|
||||
return self.provider.generate_typed(prompt, schema, max_retries=max_retries, **kwargs)
|
||||
@@ -0,0 +1,116 @@
|
||||
"""
|
||||
Ollama LLM Provider
|
||||
|
||||
Wrapper for local Ollama models with clean interface.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Union
|
||||
|
||||
from ..semantic_extract.providers import OllamaProvider
|
||||
from ..utils.exceptions import ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
|
||||
logger = get_logger("llms.ollama")
|
||||
|
||||
|
||||
class Ollama:
|
||||
"""
|
||||
Ollama LLM provider wrapper.
|
||||
|
||||
Provides clean interface to a local Ollama server. Unlike the other
|
||||
providers here, this one has no API key. It talks to an Ollama
|
||||
instance over HTTP, so make sure `ollama serve` is running first.
|
||||
|
||||
Example:
|
||||
>>> from semantica.llms import Ollama
|
||||
>>> llm = Ollama(model="llama2")
|
||||
>>> response = llm.generate("What is AI?")
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: str = "llama2",
|
||||
base_url: str = "http://localhost:11434",
|
||||
**kwargs
|
||||
):
|
||||
"""
|
||||
Initialize Ollama provider.
|
||||
|
||||
Args:
|
||||
model: Model name (default: "llama2")
|
||||
base_url: Ollama server URL (default: "http://localhost:11434")
|
||||
**kwargs: Additional provider options
|
||||
"""
|
||||
self.provider = OllamaProvider(base_url=base_url, model=model, **kwargs)
|
||||
self.model = model
|
||||
self.base_url = base_url
|
||||
|
||||
def is_available(self) -> bool:
|
||||
"""Check if Ollama provider is available."""
|
||||
return self.provider.is_available()
|
||||
|
||||
def generate(self, prompt: str, **kwargs) -> str:
|
||||
"""
|
||||
Generate text from prompt.
|
||||
|
||||
Args:
|
||||
prompt: Input prompt text
|
||||
**kwargs: Generation options (temperature, max_tokens, etc.)
|
||||
|
||||
Returns:
|
||||
Generated text response
|
||||
|
||||
Raises:
|
||||
ProcessingError: If provider is not available or generation fails
|
||||
"""
|
||||
if not self.is_available():
|
||||
raise ProcessingError(
|
||||
"Ollama provider not available. Make sure Ollama is running "
|
||||
"and reachable at the configured base_url."
|
||||
)
|
||||
return self.provider.generate(prompt, **kwargs)
|
||||
|
||||
def generate_structured(self, prompt: str, **kwargs) -> Union[Dict[str, Any], List[Any]]:
|
||||
"""
|
||||
Generate structured JSON output.
|
||||
|
||||
Args:
|
||||
prompt: Input prompt text
|
||||
**kwargs: Generation options
|
||||
|
||||
Returns:
|
||||
Parsed JSON response. A dict for a top-level JSON object, or a
|
||||
list if the model returns a top-level JSON array.
|
||||
|
||||
Raises:
|
||||
ProcessingError: If provider is not available or parsing fails
|
||||
"""
|
||||
if not self.is_available():
|
||||
raise ProcessingError(
|
||||
"Ollama provider not available. Make sure Ollama is running "
|
||||
"and reachable at the configured base_url."
|
||||
)
|
||||
return self.provider.generate_structured(prompt, **kwargs)
|
||||
|
||||
def generate_typed(self, prompt: str, schema: Any, max_retries: int = 3, **kwargs) -> Any:
|
||||
"""
|
||||
Generate output validated against a Pydantic schema.
|
||||
|
||||
Args:
|
||||
prompt: Input prompt text
|
||||
schema: Pydantic model class to validate the output against
|
||||
max_retries: Number of retries if validation fails (default: 3)
|
||||
**kwargs: Generation options
|
||||
|
||||
Returns:
|
||||
An instance of `schema`, populated from the model's response
|
||||
|
||||
Raises:
|
||||
ProcessingError: If provider is not available or generation fails
|
||||
"""
|
||||
if not self.is_available():
|
||||
raise ProcessingError(
|
||||
"Ollama provider not available. Make sure Ollama is running "
|
||||
"and reachable at the configured base_url."
|
||||
)
|
||||
return self.provider.generate_typed(prompt, schema, max_retries=max_retries, **kwargs)
|
||||
@@ -117,6 +117,8 @@ class PropertyGenerator:
|
||||
data_properties = self._infer_data_properties(entities, classes, **options)
|
||||
properties.extend(data_properties)
|
||||
|
||||
properties = self._coalesce_normalized_properties(properties)
|
||||
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
@@ -193,6 +195,67 @@ class PropertyGenerator:
|
||||
|
||||
return properties
|
||||
|
||||
def _coalesce_normalized_properties(
|
||||
self, properties: List[Dict[str, Any]]
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Merge same-kind properties that normalize to the same name."""
|
||||
property_kinds = defaultdict(set)
|
||||
for prop in properties:
|
||||
property_kinds[prop["name"]].add(prop.get("type"))
|
||||
|
||||
collisions = {
|
||||
name: sorted(kind for kind in kinds if kind is not None)
|
||||
for name, kinds in property_kinds.items()
|
||||
if len({kind for kind in kinds if kind is not None}) > 1
|
||||
}
|
||||
if collisions:
|
||||
raise ValidationError(
|
||||
"Normalized property names cannot be shared by object and "
|
||||
"data properties.",
|
||||
validation_context={"property_kind_collisions": collisions},
|
||||
)
|
||||
|
||||
merged: Dict[tuple, Dict[str, Any]] = {}
|
||||
result = []
|
||||
for prop in properties:
|
||||
key = (prop.get("type"), prop["name"])
|
||||
existing = merged.get(key)
|
||||
if existing is None:
|
||||
merged[key] = prop
|
||||
result.append(prop)
|
||||
continue
|
||||
|
||||
existing["domain"] = self._merge_property_values(
|
||||
existing.get("domain", []), prop.get("domain", [])
|
||||
)
|
||||
if prop.get("type") == "object":
|
||||
existing["range"] = self._merge_property_values(
|
||||
existing.get("range", []), prop.get("range", [])
|
||||
)
|
||||
existing_metadata = existing.setdefault("metadata", {})
|
||||
existing_metadata["occurrence_count"] = (
|
||||
existing_metadata.get("occurrence_count", 0)
|
||||
+ prop.get("metadata", {}).get("occurrence_count", 0)
|
||||
)
|
||||
elif existing.get("range") != prop.get("range"):
|
||||
existing["range"] = self._get_more_general_type(
|
||||
existing["range"], prop["range"]
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _merge_property_values(current: Any, incoming: Any) -> List[Any]:
|
||||
"""Merge scalar-or-list property values while preserving input order."""
|
||||
values = list(current) if isinstance(current, list) else [current]
|
||||
incoming_values = (
|
||||
incoming if isinstance(incoming, list) else [incoming]
|
||||
)
|
||||
for value in incoming_values:
|
||||
if value not in values:
|
||||
values.append(value)
|
||||
return [value for value in values if value is not None]
|
||||
|
||||
def _infer_data_properties(
|
||||
self, entities: List[Dict[str, Any]], classes: List[Dict[str, Any]], **options
|
||||
) -> List[Dict[str, Any]]:
|
||||
|
||||
@@ -70,7 +70,8 @@ sem:metadata a owl:AnnotationProperty ;
|
||||
rdfs:label "metadata" ;
|
||||
rdfs:comment """Free-form metadata carried through from extraction. An
|
||||
annotation property because its value is an arbitrary structure rather than a
|
||||
modelled one.""" ;
|
||||
modelled one; in the JSON-LD export the whole mapping is written as one
|
||||
rdf:JSON literal so caller keys never expand into this namespace (#1146).""" ;
|
||||
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||
|
||||
# ── Relationship terms (JSON-LD export) ──────────────────────────────────────
|
||||
@@ -96,10 +97,10 @@ sem:target a owl:ObjectProperty ;
|
||||
|
||||
sem:type a owl:DatatypeProperty ;
|
||||
rdfs:label "type" ;
|
||||
rdfs:comment """The relationship type as a label, as emitted in the JSON-LD
|
||||
export. Distinct from rdf:type, which relates a node to a class rather than to
|
||||
a string.""" ;
|
||||
rdfs:domain sem:Relationship ;
|
||||
rdfs:comment """The entity or relationship type as a label, as emitted in
|
||||
the JSON-LD export. Distinct from rdf:type, which relates a node to a class
|
||||
rather than to a string. Emitted for both entities and relationships, so the
|
||||
domain is left open rather than tied to sem:Relationship.""" ;
|
||||
rdfs:range xsd:string ;
|
||||
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||
|
||||
|
||||
@@ -41,69 +41,134 @@ from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from .reasoner import Fact, Rule, _make_activation_key
|
||||
|
||||
logger = get_logger("rete_engine")
|
||||
|
||||
def _extract_bindings(condition: Any, fact: Fact) -> Dict[str, Any]:
|
||||
"""Extract ``?var`` bindings by matching a condition pattern against a fact.
|
||||
|
||||
``condition`` is the pattern stored on the alpha node (typically a string
|
||||
like ``"Person(?x)"``); ``fact`` is the working-memory :class:`Fact`. The
|
||||
fact's canonical string form (``Predicate(arg1, arg2, ...)``) is matched
|
||||
against the pattern using the same ``?\\w+`` placeholder convention as the
|
||||
Reasoner, so downstream actions receive real bindings (e.g. ``{"x": "John"}``)
|
||||
instead of the empty dict that previously left ``?x`` placeholders
|
||||
unsubstituted.
|
||||
def _build_condition_regex(
|
||||
pattern: str,
|
||||
initial_bindings: Optional[Dict[str, str]] = None,
|
||||
) -> str:
|
||||
"""Build an anchored regex string for a condition pattern.
|
||||
|
||||
Returns an empty dict when the condition is not a string pattern or does
|
||||
not match -- callers treat that as "no bindings extracted".
|
||||
Splits the pattern on ``?var`` placeholders, escaping the literal
|
||||
segments so surrounding parentheses/commas match literally. Variables
|
||||
become named groups (or backreferences when repeated); variables already
|
||||
present in ``initial_bindings`` are inlined as their literal value.
|
||||
|
||||
Args:
|
||||
pattern: The condition pattern string (e.g. ``"Person(?x)"``).
|
||||
initial_bindings: Bindings already established upstream. Variables
|
||||
already bound are matched as literals rather than captured.
|
||||
|
||||
Returns:
|
||||
An anchored regex string (``^...$``) suitable for ``re.compile`` /
|
||||
``re.match``.
|
||||
"""
|
||||
if not isinstance(condition, str):
|
||||
return {}
|
||||
|
||||
segments = re.split(r"(\?\w+)", condition)
|
||||
bindings = initial_bindings or {}
|
||||
segments = re.split(r"(\?\w+)", pattern)
|
||||
seen_vars: Set[str] = set()
|
||||
p_regex = ""
|
||||
for seg in segments:
|
||||
if seg.startswith("?"):
|
||||
var_name = seg[1:]
|
||||
if var_name in seen_vars:
|
||||
if var_name in bindings:
|
||||
# Already bound — require the exact literal value.
|
||||
p_regex += re.escape(bindings[var_name])
|
||||
elif var_name in seen_vars:
|
||||
# Same variable used twice — enforce a backreference.
|
||||
p_regex += f"(?P={var_name})"
|
||||
else:
|
||||
p_regex += f"(?P<{var_name}>.+?)"
|
||||
seen_vars.add(var_name)
|
||||
else:
|
||||
p_regex += re.escape(seg)
|
||||
p_regex = f"^{p_regex}$"
|
||||
return f"^{p_regex}$"
|
||||
|
||||
|
||||
def unify_condition(
|
||||
condition: Any,
|
||||
fact: Fact,
|
||||
initial_bindings: Optional[Dict[str, str]] = None,
|
||||
) -> Optional[Dict[str, str]]:
|
||||
"""Unify a condition pattern against a fact.
|
||||
|
||||
A condition is a pattern string such as ``"Person(?x)"`` or
|
||||
``"knows(?x, ?y)"`` where tokens beginning with ``?`` are variables.
|
||||
The fact is rendered via its ``__str__`` representation
|
||||
(``predicate(arg1, arg2)``) and matched against the pattern.
|
||||
|
||||
This mirrors ``Reasoner._match_pattern`` but is self-contained so the
|
||||
RETE engine does not need a live ``Reasoner`` instance.
|
||||
|
||||
Args:
|
||||
condition: The condition pattern (string). Non-string conditions
|
||||
are stringified before matching.
|
||||
fact: The fact to test.
|
||||
initial_bindings: Bindings already established upstream. Variables
|
||||
already bound must match the corresponding literal in the fact.
|
||||
|
||||
Returns:
|
||||
A dict of variable bindings if the fact unifies with the condition,
|
||||
otherwise ``None``.
|
||||
"""
|
||||
bindings = dict(initial_bindings or {})
|
||||
pattern = condition if isinstance(condition, str) else str(condition)
|
||||
fact_str = str(fact)
|
||||
|
||||
# Build the anchored regex once (variables already bound are inlined as
|
||||
# literals). See ``_build_condition_regex`` for the segment handling.
|
||||
p_regex = _build_condition_regex(pattern, bindings)
|
||||
|
||||
try:
|
||||
match = re.match(p_regex, str(fact))
|
||||
except re.error:
|
||||
return {}
|
||||
match = re.match(p_regex, fact_str)
|
||||
except re.error as e:
|
||||
logger.warning(
|
||||
"unify_condition failed to compile/match condition "
|
||||
"%r (regex: %r) against fact %r: %s",
|
||||
pattern,
|
||||
p_regex,
|
||||
fact_str,
|
||||
e,
|
||||
)
|
||||
return None
|
||||
except Exception as e: # noqa: BLE001 - mirror Reasoner._match_pattern
|
||||
logger.warning(
|
||||
"unify_condition unexpected error matching condition "
|
||||
"%r (regex: %r) against fact %r: %s",
|
||||
pattern,
|
||||
p_regex,
|
||||
fact_str,
|
||||
e,
|
||||
)
|
||||
return None
|
||||
if not match:
|
||||
return {}
|
||||
return {k: v for k, v in match.groupdict().items() if v is not None}
|
||||
return None
|
||||
|
||||
|
||||
def _bindings_for_rule(rule: Rule, facts: List[Fact]) -> Dict[str, Any]:
|
||||
"""Merge ``?var`` bindings from matching a rule's conditions against facts.
|
||||
|
||||
Each fact is matched against every condition of the rule; the first
|
||||
condition that yields bindings for a fact contributes them. Bindings from
|
||||
all facts are merged so multi-condition (joined) rules receive the full
|
||||
variable environment. Later conflicting values do not overwrite earlier
|
||||
ones, preserving the binding that a join already validated.
|
||||
"""
|
||||
bindings: Dict[str, Any] = {}
|
||||
for fact in facts:
|
||||
for condition in rule.conditions:
|
||||
extracted = _extract_bindings(condition, fact)
|
||||
if not extracted:
|
||||
continue
|
||||
for key, value in extracted.items():
|
||||
bindings.setdefault(key, value)
|
||||
break
|
||||
for var, value in match.groupdict().items():
|
||||
if var in bindings and bindings[var] != value:
|
||||
return None # Binding conflict.
|
||||
bindings[var] = value
|
||||
return bindings
|
||||
|
||||
|
||||
@dataclass
|
||||
class Token:
|
||||
"""A partial match flowing through the Rete network.
|
||||
|
||||
A token represents an ordered collection of concrete facts that have
|
||||
been unified so far, together with the consistent variable bindings
|
||||
accumulated across those facts.
|
||||
|
||||
Alpha nodes emit single-fact tokens. Beta nodes merge a left token and
|
||||
a right token into a new token whose ``facts`` are the concatenation of
|
||||
both sides (preserving condition order) and whose ``bindings`` are the
|
||||
consistent union of both sides.
|
||||
"""
|
||||
|
||||
facts: List[Fact] = field(default_factory=list)
|
||||
bindings: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Match:
|
||||
"""Pattern match."""
|
||||
@@ -128,19 +193,68 @@ class AlphaNode(ReteNode):
|
||||
def __init__(self, node_id: str, condition: Any):
|
||||
super().__init__(node_id)
|
||||
self.condition = condition
|
||||
self.matches: List[Fact] = []
|
||||
# Single-fact tokens produced by unifying each matched fact with
|
||||
# this node's condition.
|
||||
self.tokens: List[Token] = []
|
||||
# Pre-compile the condition regex once. Alpha nodes never have
|
||||
# initial bindings, so the pattern is stable for the node's lifetime
|
||||
# and every incoming fact reuses this compiled matcher instead of
|
||||
# rebuilding it (avoids repeated regex construction overhead).
|
||||
pattern = condition if isinstance(condition, str) else str(condition)
|
||||
self._compiled: Optional[re.Pattern] = None
|
||||
try:
|
||||
self._compiled = re.compile(_build_condition_regex(pattern))
|
||||
except re.error as e:
|
||||
logger.warning(
|
||||
"AlphaNode %r failed to compile condition %r: %s; "
|
||||
"node will never match",
|
||||
node_id,
|
||||
pattern,
|
||||
e,
|
||||
)
|
||||
|
||||
def add_fact(self, fact: Fact) -> bool:
|
||||
"""Add fact if it matches condition."""
|
||||
if self._matches(fact):
|
||||
self.matches.append(fact)
|
||||
return True
|
||||
return False
|
||||
def add_fact(self, fact: Fact) -> Optional[Token]:
|
||||
"""Add fact if it matches the condition, returning its token.
|
||||
|
||||
def _matches(self, fact: Fact) -> bool:
|
||||
"""Check if fact matches condition."""
|
||||
# Simple matching - can be enhanced
|
||||
return True
|
||||
Returns the single-fact ``Token`` produced by unification when the
|
||||
fact matches, otherwise ``None``.
|
||||
"""
|
||||
bindings = self._matches(fact)
|
||||
if bindings is not None:
|
||||
token = Token(facts=[fact], bindings=dict(bindings))
|
||||
self.tokens.append(token)
|
||||
return token
|
||||
return None
|
||||
|
||||
def _matches(self, fact: Fact) -> Optional[Dict[str, str]]:
|
||||
"""Check if fact matches the alpha node condition.
|
||||
|
||||
Uses the pre-compiled regex built in ``__init__`` for performance,
|
||||
since RETE evaluates many facts against every alpha node.
|
||||
|
||||
Returns the variable bindings produced by unification if the fact
|
||||
matches, otherwise ``None``. An empty dict signals a match with no
|
||||
variables (still distinct from ``None``).
|
||||
"""
|
||||
if self._compiled is None:
|
||||
# Compilation failed at build time; treat as non-matching.
|
||||
return None
|
||||
fact_str = str(fact)
|
||||
try:
|
||||
match = self._compiled.match(fact_str)
|
||||
except Exception as e: # noqa: BLE001 - mirror unify_condition
|
||||
logger.warning(
|
||||
"AlphaNode %r unexpected error matching condition "
|
||||
"%r against fact %r: %s",
|
||||
self.node_id,
|
||||
self.condition,
|
||||
fact_str,
|
||||
e,
|
||||
)
|
||||
return None
|
||||
if not match:
|
||||
return None
|
||||
return match.groupdict()
|
||||
|
||||
|
||||
class BetaNode(ReteNode):
|
||||
@@ -150,19 +264,28 @@ class BetaNode(ReteNode):
|
||||
super().__init__(node_id)
|
||||
self.left = left
|
||||
self.right = right
|
||||
self.matches: List[Tuple[Fact, Fact]] = []
|
||||
# Token memories for each side. Incoming tokens are stored here so
|
||||
# that later-arriving tokens on the opposite side can be joined
|
||||
# against every token already seen (chained joins).
|
||||
self.left_tokens: List[Token] = []
|
||||
self.right_tokens: List[Token] = []
|
||||
|
||||
def join(self, left_fact: Fact, right_fact: Fact) -> bool:
|
||||
"""Join facts from left and right nodes."""
|
||||
if self._can_join(left_fact, right_fact):
|
||||
self.matches.append((left_fact, right_fact))
|
||||
return True
|
||||
return False
|
||||
def join(self, left_token: Token, right_token: Token) -> Optional[Token]:
|
||||
"""Join a left token with a right token.
|
||||
|
||||
def _can_join(self, left_fact: Fact, right_fact: Fact) -> bool:
|
||||
"""Check if facts can be joined."""
|
||||
# Simple join logic - can be enhanced
|
||||
return True
|
||||
Returns a new merged ``Token`` (facts concatenated in condition
|
||||
order, bindings unified) when the two tokens are consistent,
|
||||
otherwise ``None`` on a binding conflict.
|
||||
"""
|
||||
merged = dict(left_token.bindings)
|
||||
for var, value in right_token.bindings.items():
|
||||
if var in merged and merged[var] != value:
|
||||
return None # Binding conflict — cannot join.
|
||||
merged[var] = value
|
||||
return Token(
|
||||
facts=list(left_token.facts) + list(right_token.facts),
|
||||
bindings=merged,
|
||||
)
|
||||
|
||||
|
||||
class TerminalNode(ReteNode):
|
||||
@@ -248,12 +371,16 @@ class ReteEngine:
|
||||
self._add_rule_to_network(rule)
|
||||
|
||||
self.logger.info(
|
||||
f"Built Rete network with {len(self.network)} nodes for {len(rules)} rules"
|
||||
f"Built Rete network with {len(self.network)} nodes "
|
||||
f"for {len(rules)} rules"
|
||||
)
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
message=f"Built Rete network with {len(self.network)} nodes for {len(rules)} rules",
|
||||
message=(
|
||||
f"Built Rete network with {len(self.network)} nodes "
|
||||
f"for {len(rules)} rules"
|
||||
),
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
@@ -281,6 +408,10 @@ class ReteEngine:
|
||||
self.node_counter += 1
|
||||
beta_node = BetaNode(node_id, current, alpha_nodes[i])
|
||||
self.network[node_id] = beta_node
|
||||
# Wire the beta node as a child of both its inputs so facts
|
||||
# propagating from either side reach the join.
|
||||
current.children.append(beta_node)
|
||||
alpha_nodes[i].children.append(beta_node)
|
||||
current = beta_node
|
||||
final_node = current
|
||||
else:
|
||||
@@ -311,40 +442,58 @@ class ReteEngine:
|
||||
# Find matching alpha nodes
|
||||
for node_id, node in self.network.items():
|
||||
if isinstance(node, AlphaNode):
|
||||
if node.add_fact(fact):
|
||||
# Propagate to children
|
||||
self._propagate_from_alpha(node, fact)
|
||||
token = node.add_fact(fact)
|
||||
if token is not None:
|
||||
# Propagate the single-fact token to children.
|
||||
self._propagate_token(node, token)
|
||||
|
||||
def _propagate_from_alpha(self, alpha_node: AlphaNode, fact: Fact) -> None:
|
||||
"""Propagate from alpha node to children."""
|
||||
for child in alpha_node.children:
|
||||
def _propagate_token(self, source: ReteNode, token: Token) -> None:
|
||||
"""Propagate ``token`` (arriving from ``source``) to its children.
|
||||
|
||||
A ``Token`` carries the ordered facts and consistent bindings of a
|
||||
partial match. Beta children attempt joins and, on success, emit a
|
||||
new merged token downstream; terminal children turn the token into a
|
||||
rule activation using the token's complete facts and bindings.
|
||||
"""
|
||||
for child in source.children:
|
||||
if isinstance(child, BetaNode):
|
||||
# Join with matches from left side
|
||||
for left_fact in alpha_node.matches:
|
||||
if child.join(left_fact, fact):
|
||||
# Propagate to children
|
||||
for grandchild in child.children:
|
||||
if isinstance(grandchild, TerminalNode):
|
||||
facts = [left_fact, fact]
|
||||
match = Match(
|
||||
rule=grandchild.rule,
|
||||
facts=facts,
|
||||
bindings=_bindings_for_rule(
|
||||
grandchild.rule, facts
|
||||
),
|
||||
confidence=1.0,
|
||||
)
|
||||
grandchild.activate(match)
|
||||
self._propagate_to_beta(child, source, token)
|
||||
elif isinstance(child, TerminalNode):
|
||||
# Direct activation
|
||||
match = Match(
|
||||
rule=child.rule,
|
||||
facts=[fact],
|
||||
bindings=_bindings_for_rule(child.rule, [fact]),
|
||||
facts=list(token.facts),
|
||||
bindings=dict(token.bindings),
|
||||
confidence=1.0,
|
||||
)
|
||||
child.activate(match)
|
||||
|
||||
def _propagate_to_beta(
|
||||
self,
|
||||
beta: "BetaNode",
|
||||
source: ReteNode,
|
||||
token: Token,
|
||||
) -> None:
|
||||
"""Attempt joins at ``beta`` for a token arriving from one side.
|
||||
|
||||
The incoming token is stored in the corresponding side's memory,
|
||||
then joined against every token already recorded on the opposite
|
||||
side. Each successful join produces a new merged token that is
|
||||
propagated further downstream, enabling correct chained joins across
|
||||
three or more conditions.
|
||||
"""
|
||||
if source is beta.left:
|
||||
beta.left_tokens.append(token)
|
||||
for right_token in list(beta.right_tokens):
|
||||
merged = beta.join(token, right_token)
|
||||
if merged is not None:
|
||||
self._propagate_token(beta, merged)
|
||||
elif source is beta.right:
|
||||
beta.right_tokens.append(token)
|
||||
for left_token in list(beta.left_tokens):
|
||||
merged = beta.join(left_token, token)
|
||||
if merged is not None:
|
||||
self._propagate_token(beta, merged)
|
||||
|
||||
def match_patterns(self, facts: Optional[List[Fact]] = None) -> List[Match]:
|
||||
"""
|
||||
Match patterns using Rete algorithm.
|
||||
@@ -468,8 +617,11 @@ class ReteEngine:
|
||||
self.facts.clear()
|
||||
self.reset_action_history()
|
||||
for node in self.network.values():
|
||||
if isinstance(node, AlphaNode) or isinstance(node, BetaNode):
|
||||
node.matches.clear()
|
||||
if isinstance(node, AlphaNode):
|
||||
node.tokens.clear()
|
||||
elif isinstance(node, BetaNode):
|
||||
node.left_tokens.clear()
|
||||
node.right_tokens.clear()
|
||||
elif isinstance(node, TerminalNode):
|
||||
node.activations.clear()
|
||||
|
||||
|
||||
@@ -673,6 +673,7 @@ class GeminiProvider(BaseProvider):
|
||||
self.model = model
|
||||
self.client = None
|
||||
self._use_new_genai = False
|
||||
self._legacy_model_cache: Dict[str, Any] = {}
|
||||
self._init_client()
|
||||
|
||||
def _init_client(self):
|
||||
@@ -694,6 +695,38 @@ class GeminiProvider(BaseProvider):
|
||||
self.client = None
|
||||
self.logger.warning("Gemini SDK not installed. Install with: pip install semantica[llm-gemini]")
|
||||
|
||||
def _legacy_client_for(self, requested_model: str):
|
||||
"""Return a legacy-SDK GenerativeModel bound to this instance's own
|
||||
API key, for the given model name.
|
||||
|
||||
The legacy google-generativeai package keeps its API key as
|
||||
module-level state (genai.configure()), so any GenerativeModel built
|
||||
by a different GeminiProvider instance in the same process can leave
|
||||
that state pointing at a different key. Re-asserting configure()
|
||||
with this instance's key right before use, instead of only once at
|
||||
construction, keeps sequential calls across instances from reading
|
||||
each other's credentials. A cache keyed by model name avoids
|
||||
rebuilding a GenerativeModel on every call for the common case of
|
||||
one model being reused.
|
||||
"""
|
||||
try:
|
||||
import google.generativeai as old_genai
|
||||
old_genai.configure(api_key=self.api_key)
|
||||
except Exception:
|
||||
# _init_client() already required this import to reach the
|
||||
# legacy path in the first place, so this only happens when
|
||||
# self.client was injected directly (tests). Fall back to it
|
||||
# without reasserting credentials rather than failing calls
|
||||
# that never needed the real SDK.
|
||||
return self.client
|
||||
if requested_model == self.model:
|
||||
return self.client
|
||||
cached = self._legacy_model_cache.get(requested_model)
|
||||
if cached is None:
|
||||
cached = old_genai.GenerativeModel(requested_model)
|
||||
self._legacy_model_cache[requested_model] = cached
|
||||
return cached
|
||||
|
||||
def is_available(self) -> bool:
|
||||
"""Check if provider is available."""
|
||||
return self.client is not None
|
||||
@@ -724,7 +757,8 @@ class GeminiProvider(BaseProvider):
|
||||
)
|
||||
return self._resp_text(resp)
|
||||
else:
|
||||
response = self.client.generate_content(prompt, generation_config=config or None)
|
||||
legacy_client = self._legacy_client_for(kwargs.get("model", self.model))
|
||||
response = legacy_client.generate_content(prompt, generation_config=config or None)
|
||||
return self._resp_text(response)
|
||||
|
||||
def generate_structured(self, prompt: str, **kwargs) -> dict:
|
||||
@@ -733,15 +767,24 @@ class GeminiProvider(BaseProvider):
|
||||
raise ProcessingError("Gemini client not initialized.")
|
||||
|
||||
json_prompt = f"{prompt}\n\nReturn the response as valid JSON only."
|
||||
|
||||
config = {}
|
||||
self._add_if_set(config, kwargs, "temperature", "top_p", "top_k", "stop_sequences", "candidate_count")
|
||||
if "max_tokens" in kwargs:
|
||||
config["max_output_tokens"] = kwargs["max_tokens"]
|
||||
|
||||
if self._use_new_genai:
|
||||
model = kwargs.get("model", self.model)
|
||||
resp = self.client.models.generate_content(model=model, contents=json_prompt)
|
||||
resp = self.client.models.generate_content(
|
||||
model=model, contents=json_prompt, config=config or None
|
||||
)
|
||||
try:
|
||||
return self._parse_json(self._resp_text(resp))
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to parse JSON from Gemini response: {e}")
|
||||
else:
|
||||
response = self.client.generate_content(json_prompt)
|
||||
legacy_client = self._legacy_client_for(kwargs.get("model", self.model))
|
||||
response = legacy_client.generate_content(json_prompt, generation_config=config or None)
|
||||
try:
|
||||
return self._parse_json(self._resp_text(response))
|
||||
except Exception as e:
|
||||
@@ -967,6 +1010,8 @@ class OllamaProvider(BaseProvider):
|
||||
|
||||
def is_available(self) -> bool:
|
||||
"""Check if provider is available."""
|
||||
if self.client is None:
|
||||
self._init_client()
|
||||
return self.client is not None
|
||||
|
||||
def _build_options(self, kwargs: dict) -> Optional[dict]:
|
||||
@@ -1018,7 +1063,6 @@ class DeepSeekProvider(BaseProvider):
|
||||
self.api_key = api_key or config.get_api_key("deepseek")
|
||||
self.base_url = "https://api.deepseek.com/v1"
|
||||
self.model = model
|
||||
self.base_url = "https://api.deepseek.com/v1"
|
||||
self.client = None
|
||||
self._init_client()
|
||||
|
||||
@@ -1045,7 +1089,7 @@ class DeepSeekProvider(BaseProvider):
|
||||
"model": kwargs.get("model", self.model),
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
}
|
||||
self._add_if_set(create_kwargs, kwargs, "temperature", "max_tokens")
|
||||
self._add_if_set(create_kwargs, kwargs, "temperature", "max_tokens", "top_p", "frequency_penalty", "presence_penalty", "seed", "stop", "user")
|
||||
|
||||
response = self.client.chat.completions.create(**create_kwargs)
|
||||
return response.choices[0].message.content
|
||||
@@ -1058,8 +1102,9 @@ class DeepSeekProvider(BaseProvider):
|
||||
create_kwargs = {
|
||||
"model": kwargs.get("model", self.model),
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"response_format": {"type": "json_object"},
|
||||
}
|
||||
self._add_if_set(create_kwargs, kwargs, "temperature", "max_tokens")
|
||||
self._add_if_set(create_kwargs, kwargs, "temperature", "max_tokens", "top_p", "frequency_penalty", "presence_penalty", "seed", "stop", "user")
|
||||
|
||||
response = self.client.chat.completions.create(**create_kwargs)
|
||||
try:
|
||||
@@ -1103,7 +1148,7 @@ class NovitaProvider(BaseProvider):
|
||||
"model": kwargs.get("model", self.model),
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
}
|
||||
self._add_if_set(create_kwargs, kwargs, "temperature", "max_tokens")
|
||||
self._add_if_set(create_kwargs, kwargs, "temperature", "max_tokens", "top_p", "frequency_penalty", "presence_penalty", "seed", "stop", "user")
|
||||
|
||||
response = self.client.chat.completions.create(**create_kwargs)
|
||||
return response.choices[0].message.content
|
||||
@@ -1118,7 +1163,7 @@ class NovitaProvider(BaseProvider):
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"response_format": {"type": "json_object"},
|
||||
}
|
||||
self._add_if_set(create_kwargs, kwargs, "temperature", "max_tokens")
|
||||
self._add_if_set(create_kwargs, kwargs, "temperature", "max_tokens", "top_p", "frequency_penalty", "presence_penalty", "seed", "stop", "user")
|
||||
|
||||
response = self.client.chat.completions.create(**create_kwargs)
|
||||
try:
|
||||
|
||||
@@ -64,6 +64,8 @@ class SlidingWindowChunker:
|
||||
raise ValidationError("overlap must be non-negative")
|
||||
if self.overlap >= self.chunk_size:
|
||||
raise ValidationError("overlap must be less than chunk_size")
|
||||
if self.stride <= 0:
|
||||
raise ValidationError("stride must be positive")
|
||||
|
||||
def chunk(self, text: str, **options) -> List[Chunk]:
|
||||
"""
|
||||
@@ -215,15 +217,20 @@ class SlidingWindowChunker:
|
||||
Returns:
|
||||
list: List of chunks
|
||||
"""
|
||||
if overlap_size is None:
|
||||
return self.chunk(text)
|
||||
if overlap_size < 0:
|
||||
raise ValidationError("overlap_size must be non-negative")
|
||||
if overlap_size >= self.chunk_size:
|
||||
raise ValidationError("overlap_size must be less than chunk_size")
|
||||
|
||||
original_overlap = self.overlap
|
||||
if overlap_size is not None:
|
||||
original_stride = self.stride
|
||||
|
||||
try:
|
||||
self.overlap = overlap_size
|
||||
self.stride = self.chunk_size - self.overlap
|
||||
|
||||
chunks = self.chunk(text)
|
||||
|
||||
# Restore original overlap
|
||||
self.overlap = original_overlap
|
||||
self.stride = self.chunk_size - self.overlap
|
||||
|
||||
return chunks
|
||||
return self.chunk(text)
|
||||
finally:
|
||||
self.overlap = original_overlap
|
||||
self.stride = original_stride
|
||||
|
||||
@@ -43,7 +43,7 @@ from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
||||
from typing import Any, Callable, Dict, List, Optional, TextIO, Tuple, Union
|
||||
|
||||
from .logging import get_logger
|
||||
|
||||
@@ -144,14 +144,37 @@ class ProgressDisplay(ABC):
|
||||
class ConsoleProgressDisplay(ProgressDisplay):
|
||||
"""Console progress display with real-time updates."""
|
||||
|
||||
def __init__(self, use_emoji: bool = True, update_interval: float = 0.1):
|
||||
def __init__(
|
||||
self,
|
||||
use_emoji: bool = True,
|
||||
update_interval: float = 0.1,
|
||||
stream: Optional[TextIO] = None,
|
||||
):
|
||||
"""Initialize the console display.
|
||||
|
||||
Args:
|
||||
use_emoji: Whether to decorate output with emoji.
|
||||
update_interval: Minimum seconds between redraws.
|
||||
stream: Where progress is written. Defaults to ``sys.stderr``.
|
||||
|
||||
Progress is diagnostic output, so stderr is the correct stream
|
||||
for it, and stdout must stay clean for programs that carry a
|
||||
machine-readable protocol on it — the stdio MCP servers put
|
||||
newline-delimited JSON-RPC there, and a progress bar on stdout
|
||||
corrupts that stream.
|
||||
|
||||
Left as ``None``, the stream is resolved on each write rather
|
||||
than captured here, so a later rebinding of ``sys.stderr``
|
||||
(pytest capture, for instance) is honoured.
|
||||
"""
|
||||
self.use_emoji = use_emoji
|
||||
|
||||
# Check if stdout supports emojis (especially on Windows)
|
||||
self._stream = stream
|
||||
|
||||
# Check if the target stream supports emojis (especially on Windows)
|
||||
if self.use_emoji:
|
||||
try:
|
||||
# Try encoding a test emoji with stdout's encoding
|
||||
encoding = getattr(sys.stdout, "encoding", None)
|
||||
# Try encoding a test emoji with the stream's encoding
|
||||
encoding = getattr(self.stream, "encoding", None)
|
||||
if encoding:
|
||||
"🧠".encode(encoding)
|
||||
except (UnicodeEncodeError, LookupError, AttributeError):
|
||||
@@ -162,6 +185,11 @@ class ConsoleProgressDisplay(ProgressDisplay):
|
||||
self.current_lines: Dict[str, str] = {}
|
||||
self.lock = threading.Lock()
|
||||
|
||||
@property
|
||||
def stream(self) -> TextIO:
|
||||
"""The stream progress is written to; ``sys.stderr`` unless overridden."""
|
||||
return self._stream if self._stream is not None else sys.stderr
|
||||
|
||||
def _should_update(self) -> bool:
|
||||
"""Check if enough time has passed for update."""
|
||||
now = time.time()
|
||||
@@ -259,15 +287,16 @@ class ConsoleProgressDisplay(ProgressDisplay):
|
||||
return f"{base_msg}: {message}"
|
||||
|
||||
def _safe_write(self, text: str) -> None:
|
||||
"""Safely write text to stdout handling encoding errors."""
|
||||
"""Safely write text to the progress stream handling encoding errors."""
|
||||
stream = self.stream
|
||||
try:
|
||||
sys.stdout.write(text)
|
||||
stream.write(text)
|
||||
except UnicodeEncodeError:
|
||||
# Fallback: encode with replacement and write decoded
|
||||
# Use ascii as safe fallback if encoding is unknown or caused error
|
||||
encoding = getattr(sys.stdout, "encoding", None) or "ascii"
|
||||
encoding = getattr(stream, "encoding", None) or "ascii"
|
||||
safe_text = text.encode(encoding, errors="replace").decode(encoding)
|
||||
sys.stdout.write(safe_text)
|
||||
stream.write(safe_text)
|
||||
|
||||
def update(self, item: ProgressItem) -> None:
|
||||
"""Update console progress display."""
|
||||
@@ -302,11 +331,11 @@ class ConsoleProgressDisplay(ProgressDisplay):
|
||||
self._display_item_line(pipeline_item)
|
||||
self._safe_write("\n")
|
||||
|
||||
sys.stdout.flush()
|
||||
self.stream.flush()
|
||||
else:
|
||||
# Original single-item display
|
||||
self._display_item_line(item)
|
||||
sys.stdout.flush()
|
||||
self.stream.flush()
|
||||
|
||||
def _display_item_line(self, item: ProgressItem) -> None:
|
||||
"""Display a single progress item line."""
|
||||
@@ -468,13 +497,13 @@ class ConsoleProgressDisplay(ProgressDisplay):
|
||||
f"Completed: {completed} | Failed: {failed} | Total Time: {total_time:.2f}s\n"
|
||||
)
|
||||
self._safe_write("=" * 80 + "\n")
|
||||
sys.stdout.flush()
|
||||
self.stream.flush()
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Clear console display."""
|
||||
with self.lock:
|
||||
self._safe_write("\r" + " " * 100 + "\r")
|
||||
sys.stdout.flush()
|
||||
self.stream.flush()
|
||||
self.current_lines.clear()
|
||||
|
||||
|
||||
|
||||
@@ -66,12 +66,32 @@ class FAISSIndex:
|
||||
self.metadata: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
def add_vectors(self, vectors: np.ndarray, ids: Optional[List[str]] = None):
|
||||
"""Add vectors to index."""
|
||||
"""
|
||||
Add vectors to index.
|
||||
|
||||
Skips any id already present in vector_ids rather than appending a
|
||||
second physical vector under the same id. FAISS indices here don't
|
||||
support removing or replacing a single vector in place, so an
|
||||
"update" isn't possible; without this check, re-running an add for
|
||||
ids that already exist (e.g. retrying an interrupted migration)
|
||||
would silently duplicate vectors under the same id on every retry.
|
||||
"""
|
||||
if ids is None:
|
||||
ids = [f"vec_{i}" for i in range(len(vectors))]
|
||||
|
||||
self.index.add(vectors.astype(np.float32))
|
||||
self.vector_ids.extend(ids)
|
||||
new_rows = []
|
||||
new_ids = []
|
||||
existing = set(self.vector_ids)
|
||||
for row, vec_id in zip(vectors, ids):
|
||||
if vec_id in existing:
|
||||
continue
|
||||
new_rows.append(row)
|
||||
new_ids.append(vec_id)
|
||||
existing.add(vec_id)
|
||||
|
||||
if new_rows:
|
||||
self.index.add(np.array(new_rows, dtype=np.float32))
|
||||
self.vector_ids.extend(new_ids)
|
||||
|
||||
def search(
|
||||
self, query_vectors: np.ndarray, k: int = 10
|
||||
@@ -305,6 +325,12 @@ class FAISSStore:
|
||||
"""
|
||||
Add vectors to index.
|
||||
|
||||
Any id that already exists in the index is skipped rather than
|
||||
stored as a second physical vector under the same id (see
|
||||
FAISSIndex.add_vectors), so calling this again with ids from a
|
||||
previous call is safe and doesn't accumulate duplicates. Metadata
|
||||
for those ids is still updated.
|
||||
|
||||
Args:
|
||||
vectors: List of vectors or numpy array
|
||||
ids: Vector IDs
|
||||
@@ -312,7 +338,8 @@ class FAISSStore:
|
||||
**options: Additional options
|
||||
|
||||
Returns:
|
||||
List of vector IDs
|
||||
List of vector IDs (including ids that were already present
|
||||
and therefore not re-added as new vectors)
|
||||
"""
|
||||
num_vectors = len(vectors) if isinstance(vectors, (list, np.ndarray)) else 1
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
@@ -526,6 +553,30 @@ class FAISSStore:
|
||||
|
||||
return results
|
||||
|
||||
def scan_vectors(self, offset: int = 0, limit: int = 100) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Page through stored vectors in insertion order.
|
||||
|
||||
Args:
|
||||
offset: Number of vectors to skip
|
||||
limit: Maximum number of vectors to return
|
||||
|
||||
Returns:
|
||||
List of result dicts with 'id', 'metadata', and 'vector'
|
||||
"""
|
||||
if self.index is None or limit <= 0:
|
||||
return []
|
||||
|
||||
ids_page = self.index.vector_ids[offset:offset + limit]
|
||||
return [
|
||||
{
|
||||
"id": vector_id,
|
||||
"metadata": self.get_metadata(vector_id) or {},
|
||||
"vector": self.get_vector(vector_id),
|
||||
}
|
||||
for vector_id in ids_page
|
||||
]
|
||||
|
||||
def get_stats(self) -> Dict[str, Any]:
|
||||
"""Get index statistics."""
|
||||
if self.index is None:
|
||||
|
||||
@@ -656,6 +656,52 @@ class PgVectorStore:
|
||||
self.logger.warning(f"Failed to get metadata for {vector_id}: {e}")
|
||||
return None
|
||||
|
||||
def scan_vectors(self, offset: int = 0, limit: int = 100) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Page through stored vectors ordered by id.
|
||||
|
||||
Args:
|
||||
offset: Number of rows to skip
|
||||
limit: Maximum number of rows to return
|
||||
|
||||
Returns:
|
||||
List of result dicts with 'id', 'metadata', and 'vector'
|
||||
"""
|
||||
if not PSYCOPG3_AVAILABLE and not PSYCOPG2_AVAILABLE:
|
||||
raise ProcessingError(
|
||||
"Neither psycopg3 nor psycopg2 is available. "
|
||||
"Install with: pip install psycopg[binary] or psycopg2-binary"
|
||||
)
|
||||
|
||||
if limit <= 0:
|
||||
return []
|
||||
|
||||
scan_sql = psycopg_sql.SQL("""
|
||||
SELECT id, vector, metadata
|
||||
FROM {}
|
||||
ORDER BY id
|
||||
LIMIT %s OFFSET %s
|
||||
""").format(psycopg_sql.Identifier(self.table_name))
|
||||
|
||||
with self._get_connection() as conn:
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute(scan_sql, (limit, offset))
|
||||
rows = cur.fetchall()
|
||||
cur.close()
|
||||
|
||||
results = []
|
||||
for row in rows:
|
||||
vec_id, vec, meta = row
|
||||
results.append({
|
||||
"id": vec_id,
|
||||
"metadata": meta if isinstance(meta, dict) else json.loads(meta) if meta else {},
|
||||
"vector": np.array(vec) if vec is not None else None,
|
||||
})
|
||||
return results
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to scan vectors: {str(e)}") from e
|
||||
|
||||
def filter_by_metadata(
|
||||
self, filters: Dict[str, Any], limit: int = 10
|
||||
) -> List[Dict[str, Any]]:
|
||||
|
||||
@@ -616,6 +616,49 @@ class SQLiteVecStore:
|
||||
self.logger.warning(f"Failed to get metadata for {vector_id}: {e}")
|
||||
return None
|
||||
|
||||
def scan_vectors(self, offset: int = 0, limit: int = 100) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Page through stored vectors ordered by id.
|
||||
|
||||
Args:
|
||||
offset: Number of rows to skip
|
||||
limit: Maximum number of rows to return
|
||||
|
||||
Returns:
|
||||
List of result dicts with 'id', 'metadata', and 'vector'
|
||||
"""
|
||||
if limit <= 0:
|
||||
return []
|
||||
|
||||
query_sql = f"""
|
||||
SELECT id, embedding, metadata
|
||||
FROM {self.table_name}
|
||||
ORDER BY id
|
||||
LIMIT ? OFFSET ?
|
||||
"""
|
||||
|
||||
with self._lock, self._get_connection() as conn:
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute(query_sql, (limit, offset))
|
||||
rows = cur.fetchall()
|
||||
cur.close()
|
||||
|
||||
results = []
|
||||
for row in rows:
|
||||
vec_id, embedding_blob, meta_json = row
|
||||
vec = None
|
||||
if embedding_blob:
|
||||
vec = np.frombuffer(embedding_blob, dtype=np.float32).copy()
|
||||
results.append({
|
||||
"id": vec_id,
|
||||
"metadata": json.loads(meta_json) if meta_json else {},
|
||||
"vector": vec,
|
||||
})
|
||||
return results
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to scan vectors: {str(e)}") from e
|
||||
|
||||
def filter_by_metadata(
|
||||
self, filters: Dict[str, Any], limit: int = 10
|
||||
) -> List[Dict[str, Any]]:
|
||||
|
||||
@@ -824,6 +824,64 @@ class VectorStore:
|
||||
else:
|
||||
raise NotImplementedError(f"Backend store {type(self._backend_store).__name__} does not implement get_metadata")
|
||||
|
||||
def scan_vectors(self, offset: int = 0, limit: int = 100) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Page through stored vectors, backend-agnostic.
|
||||
|
||||
Follows the get_vector()/get_metadata() precedent (#843): the inmemory
|
||||
backend pages its local dict directly, a persistent backend delegates
|
||||
to a scan_vectors() on the wrapped store when available, and one that
|
||||
cannot enumerate its contents raises NotImplementedError rather than
|
||||
silently returning an empty page.
|
||||
|
||||
Args:
|
||||
offset: Number of vectors to skip
|
||||
limit: Maximum number of vectors to return
|
||||
|
||||
Returns:
|
||||
List of result dicts with 'id', 'metadata', and 'vector'
|
||||
"""
|
||||
if limit <= 0:
|
||||
return []
|
||||
|
||||
if self.backend == "inmemory":
|
||||
ids_page = list(self.vectors.keys())[offset:offset + limit]
|
||||
return [
|
||||
{
|
||||
"id": vec_id,
|
||||
"metadata": self.metadata.get(vec_id, {}),
|
||||
"vector": self.vectors.get(vec_id),
|
||||
}
|
||||
for vec_id in ids_page
|
||||
]
|
||||
elif self._backend_store and hasattr(self._backend_store, "scan_vectors"):
|
||||
return self._backend_store.scan_vectors(offset=offset, limit=limit)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
f"Backend store {type(self._backend_store).__name__} does not "
|
||||
"implement scan_vectors(). Add a scan_vectors() method to the "
|
||||
"backend store adapter to enable enumeration for this backend."
|
||||
)
|
||||
|
||||
def iter_vectors(self, batch_size: int = 500):
|
||||
"""
|
||||
Iterate over every stored vector, one page at a time.
|
||||
|
||||
Args:
|
||||
batch_size: Number of vectors to fetch per underlying scan_vectors() call
|
||||
|
||||
Yields:
|
||||
Result dicts with 'id', 'metadata', and 'vector', in scan order
|
||||
"""
|
||||
offset = 0
|
||||
while True:
|
||||
page = self.scan_vectors(offset=offset, limit=batch_size)
|
||||
if not page:
|
||||
return
|
||||
for item in page:
|
||||
yield item
|
||||
offset += len(page)
|
||||
|
||||
def count(self) -> int:
|
||||
"""Return the number of vectors in the store, backend-agnostic.
|
||||
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
"""AgentMemory.find_by_entity returns all matches by default (#1018).
|
||||
|
||||
The previous default limit of 10 silently truncated results, making erasure
|
||||
workflows incomplete for entities with more than 10 memories: a caller
|
||||
computing "what references this entity" from a truncated page would leave
|
||||
the remainder live. The unbounded default is deliberate — an erasure check
|
||||
cannot paginate — while callers that want a page still pass an explicit
|
||||
limit. (Previously lived in tests/test_seed_manager.py; moved to the
|
||||
AgentMemory area per review.)
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")))
|
||||
|
||||
from semantica.context.agent_memory import AgentMemory
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def memory_with_15():
|
||||
mem = AgentMemory()
|
||||
for i in range(15):
|
||||
mem.store(
|
||||
content=f"fact {i} about entity",
|
||||
entities=[{"id": "e1", "name": "Entity", "type": "thing"}],
|
||||
)
|
||||
return mem
|
||||
|
||||
|
||||
class TestFindByEntityLimit:
|
||||
def test_returns_all_matches_by_default(self, memory_with_15):
|
||||
results = memory_with_15.find_by_entity("e1")
|
||||
assert len(results) == 15, f"expected 15 (all), got {len(results)}"
|
||||
|
||||
def test_explicit_limit_still_works(self, memory_with_15):
|
||||
assert len(memory_with_15.find_by_entity("e1", limit=5)) == 5
|
||||
|
||||
def test_no_matches_returns_empty(self):
|
||||
assert AgentMemory().find_by_entity("nonexistent") == []
|
||||
@@ -291,8 +291,8 @@ class TestDeduplication(unittest.TestCase):
|
||||
class TestProgressTrackerEncoding(unittest.TestCase):
|
||||
"""Regression tests for issue #531 — Unicode crash on cp1252 Windows consoles."""
|
||||
|
||||
def _make_cp1252_stdout(self):
|
||||
"""Return a stdout-like object that raises UnicodeEncodeError for non-cp1252 chars."""
|
||||
def _make_cp1252_stream(self):
|
||||
"""Return a stream that raises UnicodeEncodeError for non-cp1252 chars."""
|
||||
class CP1252Writer:
|
||||
encoding = "cp1252"
|
||||
def write(self, text):
|
||||
@@ -304,39 +304,39 @@ class TestProgressTrackerEncoding(unittest.TestCase):
|
||||
def test_safe_write_does_not_crash_on_cp1252(self):
|
||||
"""_safe_write must not raise UnicodeEncodeError on a cp1252 console."""
|
||||
display = ConsoleProgressDisplay()
|
||||
orig = sys.stdout
|
||||
sys.stdout = self._make_cp1252_stdout()
|
||||
orig = sys.stderr
|
||||
sys.stderr = self._make_cp1252_stream()
|
||||
try:
|
||||
display._safe_write("🧠 Semantica - 📊 Current Progress\n")
|
||||
except UnicodeEncodeError:
|
||||
self.fail("_safe_write raised UnicodeEncodeError on cp1252 stdout")
|
||||
self.fail("_safe_write raised UnicodeEncodeError on a cp1252 stream")
|
||||
finally:
|
||||
sys.stdout = orig
|
||||
sys.stderr = orig
|
||||
|
||||
def test_update_pipeline_header_does_not_crash_on_cp1252(self):
|
||||
"""update() pipeline header write must not crash on a cp1252 console (issue #531)."""
|
||||
from semantica.utils.progress_tracker import ProgressItem
|
||||
display = ConsoleProgressDisplay()
|
||||
display.use_emoji = True # force emoji path to exercise the fixed branch
|
||||
orig = sys.stdout
|
||||
sys.stdout = self._make_cp1252_stdout()
|
||||
orig = sys.stderr
|
||||
sys.stderr = self._make_cp1252_stream()
|
||||
try:
|
||||
display._safe_write("🧠 Semantica - 📊 Current Progress\n")
|
||||
display._safe_write("=" * 150 + "\n")
|
||||
except UnicodeEncodeError:
|
||||
self.fail("Pipeline header write raised UnicodeEncodeError on cp1252 stdout")
|
||||
finally:
|
||||
sys.stdout = orig
|
||||
sys.stderr = orig
|
||||
|
||||
def test_emoji_detection_disables_on_cp1252(self):
|
||||
"""ConsoleProgressDisplay should auto-disable emoji when stdout is cp1252."""
|
||||
orig = sys.stdout
|
||||
sys.stdout = self._make_cp1252_stdout()
|
||||
"""ConsoleProgressDisplay should auto-disable emoji when the progress stream is cp1252."""
|
||||
orig = sys.stderr
|
||||
sys.stderr = self._make_cp1252_stream()
|
||||
try:
|
||||
display = ConsoleProgressDisplay()
|
||||
self.assertFalse(display.use_emoji, "use_emoji should be False on cp1252 stdout")
|
||||
self.assertFalse(display.use_emoji, "use_emoji should be False on a cp1252 progress stream")
|
||||
finally:
|
||||
sys.stdout = orig
|
||||
sys.stderr = orig
|
||||
|
||||
|
||||
class TestResultLimiting(unittest.TestCase):
|
||||
|
||||
@@ -24,14 +24,19 @@ import math
|
||||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from pydantic import ValidationError
|
||||
# fastapi ships in the optional `explorer` extra, not in `dev`, so this module
|
||||
# must skip rather than fail collection when it is absent. The guard has to sit
|
||||
# above the imports below, which need that extra.
|
||||
pytest.importorskip("fastapi")
|
||||
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
from semantica.explorer.app import create_app
|
||||
from semantica.explorer.routes.decisions import _node_to_decision
|
||||
from semantica.explorer.schemas import DecisionResponse
|
||||
from semantica.explorer.session import GraphSession
|
||||
from fastapi.testclient import TestClient # noqa: E402
|
||||
from pydantic import ValidationError # noqa: E402
|
||||
|
||||
from semantica.context.context_graph import ContextGraph # noqa: E402
|
||||
from semantica.explorer.app import create_app # noqa: E402
|
||||
from semantica.explorer.routes.decisions import _node_to_decision # noqa: E402
|
||||
from semantica.explorer.schemas import DecisionResponse # noqa: E402
|
||||
from semantica.explorer.session import GraphSession # noqa: E402
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -9,16 +9,15 @@ import networkx as nx
|
||||
import pytest
|
||||
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
from semantica.explorer.app import create_app
|
||||
from semantica.explorer.session import GraphSession
|
||||
# fastapi ships in the optional `explorer` extra, not in `dev`, so this module
|
||||
# must skip rather than fail collection when it is absent. The guard has to sit
|
||||
# above the import below, which pulls fastapi in transitively.
|
||||
pytest.importorskip("fastapi")
|
||||
|
||||
try:
|
||||
from starlette.testclient import TestClient
|
||||
except ImportError:
|
||||
pytest.skip(
|
||||
"starlette TestClient is required for explorer tests. Install semantica[explorer].",
|
||||
allow_module_level=True,
|
||||
)
|
||||
from semantica.explorer.app import create_app # noqa: E402
|
||||
from semantica.explorer.session import GraphSession # noqa: E402
|
||||
|
||||
from starlette.testclient import TestClient # noqa: E402
|
||||
|
||||
|
||||
|
||||
@@ -1104,7 +1103,7 @@ class TestBidirectionalPathRoute:
|
||||
# _classify_distance unit tests — issue #472
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
from semantica.utils.helpers import classify_path_distance
|
||||
from semantica.utils.helpers import classify_path_distance # noqa: E402
|
||||
|
||||
|
||||
class _FakeSimilarity:
|
||||
|
||||
@@ -13,16 +13,15 @@ browsers can't set custom headers on a WebSocket handshake.
|
||||
import pytest
|
||||
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
from semantica.explorer.app import create_app
|
||||
from semantica.explorer.session import GraphSession
|
||||
# fastapi ships in the optional `explorer` extra, not in `dev`, so this module
|
||||
# must skip rather than fail collection when it is absent. The guard has to sit
|
||||
# above the import below, which pulls fastapi in transitively.
|
||||
pytest.importorskip("fastapi")
|
||||
|
||||
try:
|
||||
from starlette.testclient import TestClient
|
||||
except ImportError:
|
||||
pytest.skip(
|
||||
"starlette TestClient is required for explorer tests. Install semantica[explorer].",
|
||||
allow_module_level=True,
|
||||
)
|
||||
from semantica.explorer.app import create_app # noqa: E402
|
||||
from semantica.explorer.session import GraphSession # noqa: E402
|
||||
|
||||
from starlette.testclient import TestClient # noqa: E402
|
||||
|
||||
|
||||
def _build_sample_graph() -> ContextGraph:
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
"""
|
||||
End-to-end integration test for deterministic Explorer rendering (Issue #1037).
|
||||
|
||||
Validates:
|
||||
1. Building the canonical 4-node, 3-edge graph using ContextGraph.
|
||||
2. Serialization via save_to_file().
|
||||
3. Reloading via load_from_file() and GraphSession.from_file() without mutation.
|
||||
4. Explorer HTTP API serving exact nodes, edges, edge types, and connectivity.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
# fastapi ships in the optional `explorer` extra, not in `dev`, so this module
|
||||
# must skip rather than fail collection when it is absent. The guard has to sit
|
||||
# above the explorer imports below, which pull fastapi in transitively.
|
||||
pytest.importorskip("fastapi")
|
||||
|
||||
from semantica.context.context_graph import ContextGraph # noqa: E402
|
||||
from semantica.explorer.app import create_app # noqa: E402
|
||||
from semantica.explorer.session import GraphSession # noqa: E402
|
||||
|
||||
try:
|
||||
from starlette.testclient import TestClient
|
||||
except ImportError:
|
||||
pytest.skip(
|
||||
"starlette TestClient required. Install semantica[explorer].",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
|
||||
def _build_deterministic_graph() -> ContextGraph:
|
||||
"""Build the exact graph requested in Semantica #1037."""
|
||||
graph = ContextGraph(advanced_analytics=False)
|
||||
|
||||
graph.add_node("alice", "Person", content="Alice")
|
||||
graph.add_node("bob", "Person", content="Bob")
|
||||
graph.add_node("acme", "Organization", content="Acme")
|
||||
graph.add_node("new_york", "Location", content="New York")
|
||||
|
||||
graph.add_edge("alice", "acme", edge_type="WORKS_AT")
|
||||
graph.add_edge("bob", "alice", edge_type="KNOWS")
|
||||
graph.add_edge("acme", "new_york", edge_type="LOCATED_IN")
|
||||
|
||||
return graph
|
||||
|
||||
|
||||
class TestExplorerDeterministicRenderingE2E:
|
||||
"""E2E suite for deterministic graph rendering and session loading."""
|
||||
|
||||
def test_build_and_in_memory_structure(self):
|
||||
graph = _build_deterministic_graph()
|
||||
|
||||
assert len(graph.nodes) == 4
|
||||
assert len(graph.edges) == 3
|
||||
|
||||
node_map = {
|
||||
nid: (node.node_type, node.content) for nid, node in graph.nodes.items()
|
||||
}
|
||||
assert node_map["alice"] == ("Person", "Alice")
|
||||
assert node_map["bob"] == ("Person", "Bob")
|
||||
assert node_map["acme"] == ("Organization", "Acme")
|
||||
assert node_map["new_york"] == ("Location", "New York")
|
||||
|
||||
edge_tuples = {(e.source_id, e.target_id, e.edge_type) for e in graph.edges}
|
||||
assert edge_tuples == {
|
||||
("alice", "acme", "WORKS_AT"),
|
||||
("bob", "alice", "KNOWS"),
|
||||
("acme", "new_york", "LOCATED_IN"),
|
||||
}
|
||||
|
||||
def test_serialization_and_deserialization(self, tmp_path: Path):
|
||||
graph = _build_deterministic_graph()
|
||||
output_file = tmp_path / "explorer_deterministic_graph.json"
|
||||
|
||||
graph.save_to_file(str(output_file))
|
||||
assert output_file.exists()
|
||||
|
||||
# Validate JSON structure
|
||||
with open(output_file, "r", encoding="utf-8") as f:
|
||||
raw_data = json.load(f)
|
||||
|
||||
assert "nodes" in raw_data
|
||||
assert "edges" in raw_data
|
||||
assert len(raw_data["nodes"]) == 4
|
||||
assert len(raw_data["edges"]) == 3
|
||||
|
||||
# Reload with ContextGraph.load_from_file
|
||||
reloaded_graph = ContextGraph(advanced_analytics=False)
|
||||
reloaded_graph.load_from_file(str(output_file))
|
||||
|
||||
assert len(reloaded_graph.nodes) == 4
|
||||
assert len(reloaded_graph.edges) == 3
|
||||
reloaded_nodes = {
|
||||
nid: (node.node_type, node.content)
|
||||
for nid, node in reloaded_graph.nodes.items()
|
||||
}
|
||||
assert reloaded_nodes["alice"] == ("Person", "Alice")
|
||||
assert reloaded_nodes["bob"] == ("Person", "Bob")
|
||||
assert reloaded_nodes["acme"] == ("Organization", "Acme")
|
||||
assert reloaded_nodes["new_york"] == ("Location", "New York")
|
||||
|
||||
reloaded_edges = {
|
||||
(e.source_id, e.target_id, e.edge_type) for e in reloaded_graph.edges
|
||||
}
|
||||
assert reloaded_edges == {
|
||||
("alice", "acme", "WORKS_AT"),
|
||||
("bob", "alice", "KNOWS"),
|
||||
("acme", "new_york", "LOCATED_IN"),
|
||||
}
|
||||
|
||||
def test_session_loading_and_stats(self, tmp_path: Path):
|
||||
graph = _build_deterministic_graph()
|
||||
output_file = tmp_path / "explorer_deterministic_graph.json"
|
||||
graph.save_to_file(str(output_file))
|
||||
|
||||
session = GraphSession.from_file(str(output_file))
|
||||
stats = session.get_stats()
|
||||
|
||||
assert stats["node_count"] == 4
|
||||
assert stats["edge_count"] == 3
|
||||
|
||||
def test_explorer_api_endpoints_with_deterministic_graph(self, tmp_path: Path):
|
||||
graph = _build_deterministic_graph()
|
||||
output_file = tmp_path / "explorer_deterministic_graph.json"
|
||||
graph.save_to_file(str(output_file))
|
||||
|
||||
session = GraphSession.from_file(str(output_file))
|
||||
app = create_app(session=session)
|
||||
|
||||
with TestClient(app) as client:
|
||||
# 1. Health & Info
|
||||
health = client.get("/api/health")
|
||||
assert health.status_code == 200
|
||||
assert health.json() == {"status": "ok"}
|
||||
|
||||
info = client.get("/api/info")
|
||||
assert info.status_code == 200
|
||||
assert info.json()["status"] == "active"
|
||||
|
||||
# 2. Stats
|
||||
stats = client.get("/api/graph/stats")
|
||||
assert stats.status_code == 200
|
||||
stats_data = stats.json()
|
||||
assert stats_data["node_count"] == 4
|
||||
assert stats_data["edge_count"] == 3
|
||||
|
||||
# 3. Nodes endpoint
|
||||
nodes_res = client.get("/api/graph/nodes")
|
||||
assert nodes_res.status_code == 200
|
||||
nodes_data = nodes_res.json()
|
||||
assert nodes_data["total"] == 4
|
||||
assert len(nodes_data["nodes"]) == 4
|
||||
|
||||
returned_nodes = {
|
||||
n["id"]: (n["type"], n["content"]) for n in nodes_data["nodes"]
|
||||
}
|
||||
assert returned_nodes["alice"] == ("Person", "Alice")
|
||||
assert returned_nodes["bob"] == ("Person", "Bob")
|
||||
assert returned_nodes["acme"] == ("Organization", "Acme")
|
||||
assert returned_nodes["new_york"] == ("Location", "New York")
|
||||
|
||||
# 4. Individual node lookups
|
||||
for node_id in ["alice", "bob", "acme", "new_york"]:
|
||||
node_res = client.get(f"/api/graph/node/{node_id}")
|
||||
assert node_res.status_code == 200
|
||||
assert node_res.json()["id"] == node_id
|
||||
|
||||
# 5. Edges endpoint
|
||||
edges_res = client.get("/api/graph/edges")
|
||||
assert edges_res.status_code == 200
|
||||
edges_data = edges_res.json()
|
||||
assert edges_data["total"] == 3
|
||||
assert len(edges_data["edges"]) == 3
|
||||
|
||||
returned_edges = {
|
||||
(e["source"], e["target"], e["type"]) for e in edges_data["edges"]
|
||||
}
|
||||
assert returned_edges == {
|
||||
("alice", "acme", "WORKS_AT"),
|
||||
("bob", "alice", "KNOWS"),
|
||||
("acme", "new_york", "LOCATED_IN"),
|
||||
}
|
||||
|
||||
def test_deterministic_graph_auth_enforcement(self, tmp_path: Path, monkeypatch):
|
||||
graph = _build_deterministic_graph()
|
||||
output_file = tmp_path / "explorer_deterministic_graph.json"
|
||||
graph.save_to_file(str(output_file))
|
||||
|
||||
session = GraphSession.from_file(str(output_file))
|
||||
app = create_app(session=session)
|
||||
|
||||
with TestClient(app) as client:
|
||||
# 1. Unconfigured auth (no SEMANTICA_ALLOW_ANONYMOUS, no SEMANTICA_API_KEY) -> 503
|
||||
monkeypatch.delenv("SEMANTICA_ALLOW_ANONYMOUS", raising=False)
|
||||
monkeypatch.delenv("SEMANTICA_API_KEY", raising=False)
|
||||
unconfigured_res = client.get("/api/graph/stats")
|
||||
assert unconfigured_res.status_code == 503
|
||||
|
||||
# 2. Configured SEMANTICA_API_KEY without header -> 401
|
||||
test_key = "secret-test-key-1037"
|
||||
monkeypatch.setenv("SEMANTICA_API_KEY", test_key)
|
||||
unauthorized_res = client.get("/api/graph/nodes")
|
||||
assert unauthorized_res.status_code == 401
|
||||
|
||||
# 3. Configured SEMANTICA_API_KEY with valid X-API-Key header -> 200
|
||||
authorized_res = client.get(
|
||||
"/api/graph/nodes",
|
||||
headers={"X-API-Key": test_key},
|
||||
)
|
||||
assert authorized_res.status_code == 200
|
||||
assert authorized_res.json()["total"] == 4
|
||||
|
||||
# 4. Explicit local development opt-in: SEMANTICA_ALLOW_ANONYMOUS=true -> 200 without header
|
||||
monkeypatch.setenv("SEMANTICA_ALLOW_ANONYMOUS", "true")
|
||||
monkeypatch.delenv("SEMANTICA_API_KEY", raising=False)
|
||||
anon_res = client.get("/api/graph/edges")
|
||||
assert anon_res.status_code == 200
|
||||
assert anon_res.json()["total"] == 3
|
||||
|
||||
@@ -27,7 +27,12 @@ import threading
|
||||
|
||||
import pytest
|
||||
|
||||
from semantica.explorer.routes import ontology as ontology_mod
|
||||
# fastapi ships in the optional `explorer` extra, not in `dev`, so this module
|
||||
# must skip rather than fail collection when it is absent. The guard has to sit
|
||||
# above the import below, which pulls fastapi in transitively.
|
||||
pytest.importorskip("fastapi")
|
||||
|
||||
from semantica.explorer.routes import ontology as ontology_mod # noqa: E402
|
||||
|
||||
|
||||
def _start_local_server():
|
||||
|
||||
@@ -21,7 +21,12 @@ from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from semantica.explorer.routes import ontology as ontology_mod
|
||||
# fastapi ships in the optional `explorer` extra, not in `dev`, so this module
|
||||
# must skip rather than fail collection when it is absent. The guard has to sit
|
||||
# above the import below, which pulls fastapi in transitively.
|
||||
pytest.importorskip("fastapi")
|
||||
|
||||
from semantica.explorer.routes import ontology as ontology_mod # noqa: E402
|
||||
|
||||
|
||||
def _fake_getaddrinfo(host, *args, **kwargs):
|
||||
|
||||
@@ -6,17 +6,16 @@ from urllib.parse import quote
|
||||
import pytest
|
||||
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
from semantica.explorer.app import create_app
|
||||
from semantica.explorer.routes.ontology import OntologyEntry
|
||||
from semantica.explorer.session import GraphSession
|
||||
# fastapi ships in the optional `explorer` extra, not in `dev`, so this module
|
||||
# must skip rather than fail collection when it is absent. The guard has to sit
|
||||
# above the import below, which pulls fastapi in transitively.
|
||||
pytest.importorskip("fastapi")
|
||||
|
||||
try:
|
||||
from starlette.testclient import TestClient
|
||||
except ImportError:
|
||||
pytest.skip(
|
||||
"starlette TestClient is required for explorer tests. Install semantica[explorer].",
|
||||
allow_module_level=True,
|
||||
)
|
||||
from semantica.explorer.app import create_app # noqa: E402
|
||||
from semantica.explorer.routes.ontology import OntologyEntry # noqa: E402
|
||||
from semantica.explorer.session import GraphSession # noqa: E402
|
||||
|
||||
from starlette.testclient import TestClient # noqa: E402
|
||||
|
||||
|
||||
def _build_ontology_graph() -> ContextGraph:
|
||||
|
||||
@@ -5,13 +5,18 @@ Tests for ProvenanceManager wiring into Explorer routes and application startup.
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from starlette.testclient import TestClient
|
||||
# fastapi ships in the optional `explorer` extra, not in `dev`, so this module
|
||||
# must skip rather than fail collection when it is absent. The guard has to sit
|
||||
# above the starlette/explorer imports below, which need that extra.
|
||||
pytest.importorskip("fastapi")
|
||||
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
from semantica.explorer.app import create_app
|
||||
from semantica.explorer.session import GraphSession
|
||||
from semantica.provenance import ProvenanceManager
|
||||
from semantica.provenance.storage import SQLiteStorage
|
||||
from starlette.testclient import TestClient # noqa: E402
|
||||
|
||||
from semantica.context.context_graph import ContextGraph # noqa: E402
|
||||
from semantica.explorer.app import create_app # noqa: E402
|
||||
from semantica.explorer.session import GraphSession # noqa: E402
|
||||
from semantica.provenance import ProvenanceManager # noqa: E402
|
||||
from semantica.provenance.storage import SQLiteStorage # noqa: E402
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
||||
@@ -2,7 +2,14 @@
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from semantica.explorer.routes.provenance import (
|
||||
import pytest
|
||||
|
||||
# fastapi ships in the optional `explorer` extra, not in `dev`, so this module
|
||||
# must skip rather than fail collection when it is absent. The guard has to sit
|
||||
# above the import below, which pulls fastapi in transitively.
|
||||
pytest.importorskip("fastapi")
|
||||
|
||||
from semantica.explorer.routes.provenance import ( # noqa: E402
|
||||
_add_chain_edges,
|
||||
_build_provenance,
|
||||
_render_markdown,
|
||||
|
||||
@@ -14,18 +14,17 @@ from unittest.mock import patch
|
||||
import pytest
|
||||
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
from semantica.explorer.app import create_app
|
||||
from semantica.explorer.session import GraphSession
|
||||
# fastapi ships in the optional `explorer` extra, not in `dev`, so this module
|
||||
# must skip rather than fail collection when it is absent. The guard has to sit
|
||||
# above the import below, which pulls fastapi in transitively.
|
||||
pytest.importorskip("fastapi")
|
||||
|
||||
try:
|
||||
from starlette.testclient import TestClient
|
||||
except ImportError:
|
||||
pytest.skip(
|
||||
"starlette TestClient is required for explorer tests. Install semantica[explorer].",
|
||||
allow_module_level=True,
|
||||
)
|
||||
from semantica.explorer.app import create_app # noqa: E402
|
||||
from semantica.explorer.session import GraphSession # noqa: E402
|
||||
|
||||
import semantica.explorer.routes.sparql as sparql_mod
|
||||
from starlette.testclient import TestClient # noqa: E402
|
||||
|
||||
import semantica.explorer.routes.sparql as sparql_mod # noqa: E402
|
||||
|
||||
|
||||
def _build_sample_graph() -> ContextGraph:
|
||||
|
||||
@@ -2,12 +2,19 @@
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
import pytest
|
||||
|
||||
from semantica.explorer.dependencies import get_session
|
||||
from semantica.explorer.routes.vocabulary import router
|
||||
from semantica.utils.skos import validate_skos_hierarchy
|
||||
# fastapi ships in the optional `explorer` extra, not in `dev`, so this module
|
||||
# must skip rather than fail collection when it is absent. The guard has to sit
|
||||
# above the import below, which pulls fastapi in transitively.
|
||||
pytest.importorskip("fastapi")
|
||||
|
||||
from fastapi import FastAPI # noqa: E402
|
||||
from fastapi.testclient import TestClient # noqa: E402
|
||||
|
||||
from semantica.explorer.dependencies import get_session # noqa: E402
|
||||
from semantica.explorer.routes.vocabulary import router # noqa: E402
|
||||
from semantica.utils.skos import validate_skos_hierarchy # noqa: E402
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(router)
|
||||
|
||||
@@ -67,7 +67,7 @@ def test_merging_repeated_exports_yields_one_graph_node(tmp_path):
|
||||
assert len(kg_nodes) == 1
|
||||
|
||||
entity_nodes = set(
|
||||
merged.subjects(RDF.type, URIRef("https://semantica.dev/vocab/ORG"))
|
||||
merged.subjects(RDF.type, URIRef("https://semantica.dev/ns#Entity"))
|
||||
)
|
||||
assert len(entity_nodes) == 1
|
||||
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Caller data must never expand into the Semantica namespace (#1146).
|
||||
|
||||
``@vocab`` used to sit in every JSON-LD context pointing at ``ns#``, so every
|
||||
bare term in caller data expanded into it: an extracted type like ``"ORG"``
|
||||
became ``ns#ORG`` (a term the vocabulary does not define), and a metadata key
|
||||
like ``"source"`` collided with the real ``sem:source`` object property,
|
||||
attaching a plain string to a property whose range is a resource. The fix
|
||||
removes ``@vocab`` outright: only explicit ``semantica:``-prefixed terms
|
||||
resolve, caller type labels travel as ``semantica:type`` strings, and caller
|
||||
metadata survives as one ``rdf:JSON`` literal.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from rdflib import RDF, Graph, Literal, URIRef
|
||||
|
||||
from semantica.export.json_exporter import JSONExporter
|
||||
from semantica.export.rdf_exporter import RDFExporter, SEMANTICA_NS
|
||||
|
||||
NS = SEMANTICA_NS
|
||||
E1 = "https://example.org/e1"
|
||||
|
||||
KG = {
|
||||
"entities": [
|
||||
{
|
||||
"id": E1,
|
||||
"text": "Acme",
|
||||
"type": "ORG",
|
||||
"metadata": {"source": "crm_export_2024"},
|
||||
}
|
||||
],
|
||||
"relationships": [
|
||||
{
|
||||
"source_id": E1,
|
||||
"target_id": "https://example.org/e2",
|
||||
"type": "employs",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _jsonld_file(exporter, kind, tmp_path, name):
|
||||
path = tmp_path / name
|
||||
if kind == "knowledge_graph":
|
||||
exporter.export_knowledge_graph(KG, path, format="json-ld")
|
||||
elif kind == "entities":
|
||||
exporter.export_entities(KG["entities"], path, format="json-ld")
|
||||
elif kind == "relationships":
|
||||
exporter.export_relationships(KG["relationships"], path, format="json-ld")
|
||||
elif kind == "generic":
|
||||
exporter.export({"note": "plain payload, no @id"}, path, format="json-ld")
|
||||
else:
|
||||
raise AssertionError(kind)
|
||||
return json.loads(path.read_text())
|
||||
|
||||
|
||||
def test_no_jsonld_context_declares_a_vocab(tmp_path):
|
||||
exporter = JSONExporter()
|
||||
for kind in ("knowledge_graph", "entities", "relationships", "generic"):
|
||||
context = _jsonld_file(exporter, kind, tmp_path, f"{kind}.jsonld")[
|
||||
"@context"
|
||||
]
|
||||
assert "@vocab" not in context, f"{kind}: @vocab expands caller data"
|
||||
|
||||
context = json.loads(RDFExporter().export_to_rdf(KG, format="jsonld"))[
|
||||
"@context"
|
||||
]
|
||||
assert "@vocab" not in context
|
||||
|
||||
|
||||
def test_extracted_type_labels_stay_out_of_the_namespace(tmp_path):
|
||||
path = tmp_path / "kg.jsonld"
|
||||
JSONExporter().export_knowledge_graph(KG, path, format="json-ld")
|
||||
|
||||
graph = Graph()
|
||||
graph.parse(str(path), format="json-ld")
|
||||
|
||||
assert (None, RDF.type, URIRef(NS + "ORG")) not in graph, (
|
||||
"the caller's type label was minted as a class in ns#"
|
||||
)
|
||||
assert (URIRef(E1), RDF.type, URIRef(NS + "Entity")) in graph
|
||||
assert (URIRef(E1), URIRef(NS + "type"), Literal("ORG")) in graph, (
|
||||
"the label itself must survive, as a string"
|
||||
)
|
||||
|
||||
|
||||
def test_metadata_keys_stay_out_of_the_namespace(tmp_path):
|
||||
path = tmp_path / "kg.jsonld"
|
||||
JSONExporter().export_knowledge_graph(KG, path, format="json-ld")
|
||||
|
||||
graph = Graph()
|
||||
graph.parse(str(path), format="json-ld")
|
||||
|
||||
assert (None, URIRef(NS + "source"), Literal("crm_export_2024")) not in (
|
||||
graph
|
||||
), "caller metadata value attached to the real sem:source object property"
|
||||
for _, _, o in graph.triples((None, URIRef(NS + "source"), None)):
|
||||
assert not isinstance(o, Literal), (
|
||||
"sem:source has a resource range but received a plain literal"
|
||||
)
|
||||
|
||||
literals = [
|
||||
o
|
||||
for o in graph.objects(None, URIRef(NS + "metadata"))
|
||||
if isinstance(o, Literal)
|
||||
]
|
||||
assert literals, "the metadata dict was dropped instead of preserved"
|
||||
assert literals[0].datatype == RDF.JSON
|
||||
assert json.loads(str(literals[0])) == {"source": "crm_export_2024"}
|
||||
|
||||
|
||||
def test_rdf_exporter_jsonld_keeps_type_labels_out_of_the_namespace():
|
||||
graph = Graph()
|
||||
graph.parse(
|
||||
data=RDFExporter().export_to_rdf(KG, format="jsonld"), format="json-ld"
|
||||
)
|
||||
|
||||
assert (None, RDF.type, URIRef(NS + "ORG")) not in graph
|
||||
assert (URIRef(E1), RDF.type, URIRef(NS + "Entity")) in graph
|
||||
assert (URIRef(E1), URIRef(NS + "type"), Literal("ORG")) in graph
|
||||
@@ -0,0 +1 @@
|
||||
# tests/integrations/crewai package
|
||||
@@ -0,0 +1 @@
|
||||
# tests/integrations/langchain package
|
||||
@@ -0,0 +1,51 @@
|
||||
import pytest
|
||||
|
||||
from semantica.ontology.class_inferrer import ClassInferrer
|
||||
from semantica.ontology.property_generator import PropertyGenerator
|
||||
from semantica.utils.exceptions import ValidationError
|
||||
|
||||
|
||||
def test_same_kind_normalized_object_properties_are_merged():
|
||||
entities = [
|
||||
{"id": "p1", "type": "Person", "name": "Alice"},
|
||||
{"id": "o1", "type": "Organization", "name": "Acme"},
|
||||
]
|
||||
classes = ClassInferrer(min_occurrences=1).infer_classes(entities)
|
||||
relationships = [
|
||||
{
|
||||
"source_type": "Person",
|
||||
"target_type": "Organization",
|
||||
"type": "works_for",
|
||||
},
|
||||
{
|
||||
"source_type": "Person",
|
||||
"target_type": "Organization",
|
||||
"type": "worksFor",
|
||||
},
|
||||
]
|
||||
|
||||
properties = PropertyGenerator().infer_properties(
|
||||
entities, relationships, classes, min_occurrences=1
|
||||
)
|
||||
|
||||
works_for = [prop for prop in properties if prop["name"] == "worksFor"]
|
||||
assert len(works_for) == 1
|
||||
assert works_for[0]["domain"] == ["Person"]
|
||||
assert works_for[0]["range"] == ["Organization"]
|
||||
|
||||
|
||||
def test_normalized_name_cannot_be_both_object_and_data_property():
|
||||
entities = [
|
||||
{"id": "p1", "type": "Person", "value": "Alice"},
|
||||
{"id": "p2", "type": "Person", "value": "Bob"},
|
||||
]
|
||||
classes = ClassInferrer(min_occurrences=1).infer_classes(entities)
|
||||
relationships = [
|
||||
{"source_type": "Person", "target_type": "Person", "type": "value"},
|
||||
{"source_type": "Person", "target_type": "Person", "type": "value"},
|
||||
]
|
||||
|
||||
with pytest.raises(ValidationError, match="object and data"):
|
||||
PropertyGenerator().infer_properties(
|
||||
entities, relationships, classes, min_occurrences=1
|
||||
)
|
||||
@@ -0,0 +1,366 @@
|
||||
"""Tests for the RETE engine pattern matching (issue #300).
|
||||
|
||||
These tests verify that ``AlphaNode._matches`` and ``BetaNode._can_join`` no
|
||||
longer behave like the old always-``True`` stubs, and that the network as a
|
||||
whole only fires rules whose conditions actually unify with the facts.
|
||||
"""
|
||||
|
||||
import itertools
|
||||
import re
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
from semantica.reasoning import rete_engine
|
||||
from semantica.reasoning.reasoner import Fact, Rule
|
||||
from semantica.reasoning.rete_engine import (
|
||||
AlphaNode,
|
||||
BetaNode,
|
||||
ReteEngine,
|
||||
unify_condition,
|
||||
)
|
||||
|
||||
|
||||
class TestUnifyCondition(unittest.TestCase):
|
||||
def test_single_variable_binds(self):
|
||||
fact = Fact("f1", "Person", ["John"])
|
||||
bindings = unify_condition("Person(?x)", fact)
|
||||
self.assertEqual(bindings, {"x": "John"})
|
||||
|
||||
def test_predicate_mismatch_returns_none(self):
|
||||
fact = Fact("f1", "Company", ["Google"])
|
||||
self.assertIsNone(unify_condition("Person(?x)", fact))
|
||||
|
||||
def test_two_arguments_bind(self):
|
||||
fact = Fact("f2", "Parent", ["John", "Mary"])
|
||||
bindings = unify_condition("Parent(?x, ?y)", fact)
|
||||
self.assertEqual(bindings, {"x": "John", "y": "Mary"})
|
||||
|
||||
def test_literal_argument_must_match(self):
|
||||
fact = Fact("f3", "Parent", ["John", "Mary"])
|
||||
self.assertIsNone(unify_condition("Parent(Bob, ?y)", fact))
|
||||
self.assertEqual(unify_condition("Parent(John, ?y)", fact), {"y": "Mary"})
|
||||
|
||||
def test_repeated_variable_requires_equal_values(self):
|
||||
loves_self = Fact("f4", "Loves", ["John", "John"])
|
||||
loves_other = Fact("f5", "Loves", ["John", "Mary"])
|
||||
self.assertEqual(unify_condition("Loves(?x, ?x)", loves_self), {"x": "John"})
|
||||
self.assertIsNone(unify_condition("Loves(?x, ?x)", loves_other))
|
||||
|
||||
def test_regex_error_logs_warning_and_returns_none(self):
|
||||
"""A regex compilation error is logged with context and yields None."""
|
||||
fact = Fact("f6", "Person", ["John"])
|
||||
with mock.patch.object(
|
||||
rete_engine.re,
|
||||
"match",
|
||||
side_effect=re.error("bad pattern"),
|
||||
), self.assertLogs("semantica.rete_engine", level="WARNING") as captured:
|
||||
result = unify_condition("Person(?x)", fact)
|
||||
self.assertIsNone(result)
|
||||
joined = "\n".join(captured.output)
|
||||
self.assertIn("Person(?x)", joined)
|
||||
self.assertIn("Person(John)", joined)
|
||||
self.assertIn("bad pattern", joined)
|
||||
|
||||
def test_unexpected_error_logs_warning_and_returns_none(self):
|
||||
"""An unexpected error is also logged and swallowed as None."""
|
||||
fact = Fact("f7", "Person", ["John"])
|
||||
with mock.patch.object(
|
||||
rete_engine.re,
|
||||
"match",
|
||||
side_effect=RuntimeError("boom"),
|
||||
), self.assertLogs("semantica.rete_engine", level="WARNING") as captured:
|
||||
result = unify_condition("Person(?x)", fact)
|
||||
self.assertIsNone(result)
|
||||
self.assertIn("boom", "\n".join(captured.output))
|
||||
|
||||
|
||||
class TestAlphaNode(unittest.TestCase):
|
||||
def test_matches_stores_bindings(self):
|
||||
node = AlphaNode("a1", "Person(?x)")
|
||||
fact = Fact("f1", "Person", ["John"])
|
||||
token = node.add_fact(fact)
|
||||
self.assertIsNotNone(token)
|
||||
assert token is not None # narrow type for the checker
|
||||
self.assertEqual(token.facts, [fact])
|
||||
self.assertEqual(token.bindings, {"x": "John"})
|
||||
self.assertIn(token, node.tokens)
|
||||
|
||||
def test_non_matching_fact_rejected(self):
|
||||
node = AlphaNode("a1", "Person(?x)")
|
||||
fact = Fact("f1", "Company", ["Google"])
|
||||
self.assertIsNone(node.add_fact(fact))
|
||||
self.assertEqual(node.tokens, [])
|
||||
|
||||
def test_uses_precompiled_regex(self):
|
||||
"""AlphaNode compiles its condition once and reuses it per fact."""
|
||||
node = AlphaNode("a1", "Person(?x)")
|
||||
self.assertIsNotNone(node._compiled)
|
||||
# Matching goes through the compiled matcher, not unify_condition.
|
||||
with mock.patch.object(rete_engine, "unify_condition") as unify:
|
||||
fact = Fact("f1", "Person", ["John"])
|
||||
token = node.add_fact(fact)
|
||||
unify.assert_not_called()
|
||||
self.assertIsNotNone(token)
|
||||
assert token is not None
|
||||
self.assertEqual(token.bindings, {"x": "John"})
|
||||
|
||||
def test_bad_condition_never_matches_and_logs(self):
|
||||
"""A condition that fails to compile logs a warning and never fires."""
|
||||
with mock.patch.object(
|
||||
rete_engine,
|
||||
"_build_condition_regex",
|
||||
return_value="(unbalanced",
|
||||
), self.assertLogs("semantica.rete_engine", level="WARNING") as captured:
|
||||
node = AlphaNode("bad", "Person(?x)")
|
||||
self.assertIsNone(node._compiled)
|
||||
self.assertIn("failed to compile", "\n".join(captured.output))
|
||||
fact = Fact("f1", "Person", ["John"])
|
||||
self.assertIsNone(node.add_fact(fact))
|
||||
self.assertEqual(node.tokens, [])
|
||||
|
||||
|
||||
class TestBetaNode(unittest.TestCase):
|
||||
def test_join_consistent_bindings(self):
|
||||
left = AlphaNode("a1", "Parent(?x, ?y)")
|
||||
right = AlphaNode("a2", "Person(?x)")
|
||||
beta = BetaNode("b1", left, right)
|
||||
|
||||
parent = Fact("f1", "Parent", ["John", "Mary"])
|
||||
person = Fact("f2", "Person", ["John"])
|
||||
left_token = left.add_fact(parent)
|
||||
right_token = right.add_fact(person)
|
||||
assert left_token is not None and right_token is not None
|
||||
|
||||
merged = beta.join(left_token, right_token)
|
||||
self.assertIsNotNone(merged)
|
||||
assert merged is not None # narrow type for the checker
|
||||
self.assertEqual(merged.bindings, {"x": "John", "y": "Mary"})
|
||||
# Facts are concatenated left-then-right in condition order.
|
||||
self.assertEqual(merged.facts, [parent, person])
|
||||
|
||||
def test_join_conflicting_bindings_rejected(self):
|
||||
left = AlphaNode("a1", "Parent(?x, ?y)")
|
||||
right = AlphaNode("a2", "Person(?x)")
|
||||
beta = BetaNode("b1", left, right)
|
||||
|
||||
parent = Fact("f1", "Parent", ["John", "Mary"])
|
||||
# ?x conflicts: John vs Alice
|
||||
person = Fact("f2", "Person", ["Alice"])
|
||||
left_token = left.add_fact(parent)
|
||||
right_token = right.add_fact(person)
|
||||
assert left_token is not None and right_token is not None
|
||||
|
||||
self.assertIsNone(beta.join(left_token, right_token))
|
||||
|
||||
|
||||
class TestReteEngineEndToEnd(unittest.TestCase):
|
||||
def test_only_matching_rule_fires(self):
|
||||
engine = ReteEngine()
|
||||
rule = Rule(
|
||||
rule_id="r1",
|
||||
name="person rule",
|
||||
conditions=["Person(?x)"],
|
||||
conclusion="Mortal(?x)",
|
||||
)
|
||||
engine.build_network([rule])
|
||||
|
||||
engine.add_fact(Fact("f1", "Person", ["John"]))
|
||||
engine.add_fact(Fact("f2", "Company", ["Google"])) # should NOT fire
|
||||
|
||||
matches = engine.match_patterns()
|
||||
self.assertEqual(len(matches), 1)
|
||||
self.assertEqual(matches[0].bindings, {"x": "John"})
|
||||
|
||||
def test_multi_condition_join(self):
|
||||
engine = ReteEngine()
|
||||
rule = Rule(
|
||||
rule_id="r1",
|
||||
name="child rule",
|
||||
conditions=["Person(?x)", "Parent(?x, ?y)"],
|
||||
conclusion="Child(?y, ?x)",
|
||||
)
|
||||
engine.build_network([rule])
|
||||
|
||||
engine.add_fact(Fact("f1", "Person", ["John"]))
|
||||
engine.add_fact(Fact("f2", "Parent", ["John", "Mary"]))
|
||||
# Unrelated parent whose ?x does not match any Person -> no activation.
|
||||
engine.add_fact(Fact("f3", "Parent", ["Bob", "Sue"]))
|
||||
|
||||
matches = engine.match_patterns()
|
||||
self.assertEqual(len(matches), 1)
|
||||
self.assertEqual(matches[0].bindings, {"x": "John", "y": "Mary"})
|
||||
|
||||
def test_no_activation_when_join_inconsistent(self):
|
||||
engine = ReteEngine()
|
||||
rule = Rule(
|
||||
rule_id="r1",
|
||||
name="child rule",
|
||||
conditions=["Person(?x)", "Parent(?x, ?y)"],
|
||||
conclusion="Child(?y, ?x)",
|
||||
)
|
||||
engine.build_network([rule])
|
||||
|
||||
engine.add_fact(Fact("f1", "Person", ["John"]))
|
||||
engine.add_fact(Fact("f2", "Parent", ["Alice", "Mary"])) # ?x mismatch
|
||||
|
||||
matches = engine.match_patterns()
|
||||
self.assertEqual(matches, [])
|
||||
|
||||
|
||||
class TestThreeConditionChain(unittest.TestCase):
|
||||
"""Chained beta joins across three or more conditions (issue #300).
|
||||
|
||||
These exercise the Token model: a token must accumulate the ordered
|
||||
facts and the consistent bindings of every condition, so that deep
|
||||
chains neither drop bindings nor duplicate facts, and a conflict on the
|
||||
third condition correctly suppresses activation.
|
||||
"""
|
||||
|
||||
def _three_condition_rule(self):
|
||||
return Rule(
|
||||
rule_id="r1",
|
||||
name="location chain",
|
||||
conditions=[
|
||||
"Person(?x)",
|
||||
"Parent(?x, ?y)",
|
||||
"Located(?y, ?z)",
|
||||
],
|
||||
conclusion="LivesNear(?x, ?z)",
|
||||
)
|
||||
|
||||
def test_three_condition_valid_match(self):
|
||||
engine = ReteEngine()
|
||||
engine.build_network([self._three_condition_rule()])
|
||||
|
||||
engine.add_fact(Fact("f1", "Person", ["John"]))
|
||||
engine.add_fact(Fact("f2", "Parent", ["John", "Mary"]))
|
||||
engine.add_fact(Fact("f3", "Located", ["Mary", "Paris"]))
|
||||
|
||||
matches = engine.match_patterns()
|
||||
self.assertEqual(len(matches), 1)
|
||||
self.assertEqual(
|
||||
matches[0].bindings,
|
||||
{"x": "John", "y": "Mary", "z": "Paris"},
|
||||
)
|
||||
|
||||
def test_three_condition_third_level_conflict(self):
|
||||
engine = ReteEngine()
|
||||
engine.build_network([self._three_condition_rule()])
|
||||
|
||||
engine.add_fact(Fact("f1", "Person", ["John"]))
|
||||
engine.add_fact(Fact("f2", "Parent", ["John", "Mary"]))
|
||||
# ?y is bound to Mary, so a Located fact about Bob must not join.
|
||||
engine.add_fact(Fact("f3", "Located", ["Bob", "Paris"]))
|
||||
|
||||
matches = engine.match_patterns()
|
||||
self.assertEqual(matches, [])
|
||||
|
||||
def test_fact_insertion_order_independent(self):
|
||||
# Whatever order facts arrive, the same single match must result.
|
||||
base_facts = [
|
||||
Fact("f1", "Person", ["John"]),
|
||||
Fact("f2", "Parent", ["John", "Mary"]),
|
||||
Fact("f3", "Located", ["Mary", "Paris"]),
|
||||
]
|
||||
expected = {"x": "John", "y": "Mary", "z": "Paris"}
|
||||
|
||||
for order in itertools.permutations(base_facts):
|
||||
engine = ReteEngine()
|
||||
engine.build_network([self._three_condition_rule()])
|
||||
for fact in order:
|
||||
engine.add_fact(fact)
|
||||
matches = engine.match_patterns()
|
||||
self.assertEqual(len(matches), 1, f"order={order}")
|
||||
self.assertEqual(matches[0].bindings, expected)
|
||||
|
||||
def test_match_facts_complete_in_condition_order(self):
|
||||
engine = ReteEngine()
|
||||
engine.build_network([self._three_condition_rule()])
|
||||
|
||||
person = Fact("f1", "Person", ["John"])
|
||||
parent = Fact("f2", "Parent", ["John", "Mary"])
|
||||
located = Fact("f3", "Located", ["Mary", "Paris"])
|
||||
engine.add_fact(person)
|
||||
engine.add_fact(parent)
|
||||
engine.add_fact(located)
|
||||
|
||||
matches = engine.match_patterns()
|
||||
self.assertEqual(len(matches), 1)
|
||||
# All three facts preserved, in condition order, no duplicates.
|
||||
self.assertEqual(matches[0].facts, [person, parent, located])
|
||||
|
||||
def test_multiple_left_tokens_join_one_right_fact(self):
|
||||
# Two Person/Parent chains sharing the same Located(?y, ?z) fact.
|
||||
engine = ReteEngine()
|
||||
engine.build_network([self._three_condition_rule()])
|
||||
|
||||
engine.add_fact(Fact("f1", "Person", ["John"]))
|
||||
engine.add_fact(Fact("f2", "Parent", ["John", "Mary"]))
|
||||
engine.add_fact(Fact("f3", "Person", ["Alice"]))
|
||||
engine.add_fact(Fact("f4", "Parent", ["Alice", "Mary"]))
|
||||
# One right fact should join with both accumulated left tokens.
|
||||
engine.add_fact(Fact("f5", "Located", ["Mary", "Paris"]))
|
||||
|
||||
matches = engine.match_patterns()
|
||||
self.assertEqual(len(matches), 2)
|
||||
result = {m.bindings["x"]: m.bindings["z"] for m in matches}
|
||||
self.assertEqual(result, {"John": "Paris", "Alice": "Paris"})
|
||||
|
||||
def test_matches_reasoner_match_rule(self):
|
||||
from semantica.reasoning.reasoner import Reasoner
|
||||
|
||||
rule = self._three_condition_rule()
|
||||
facts = [
|
||||
Fact("f1", "Person", ["John"]),
|
||||
Fact("f2", "Parent", ["John", "Mary"]),
|
||||
Fact("f3", "Located", ["Mary", "Paris"]),
|
||||
]
|
||||
|
||||
# Reasoner works over stringified facts and returns
|
||||
# (conclusion, matched_facts, bindings) tuples from self.facts.
|
||||
reasoner = Reasoner()
|
||||
for fact in facts:
|
||||
reasoner.add_fact(str(fact))
|
||||
reasoner_matches = reasoner._match_rule(rule)
|
||||
|
||||
engine = ReteEngine()
|
||||
engine.build_network([rule])
|
||||
for fact in facts:
|
||||
engine.add_fact(fact)
|
||||
rete_matches = engine.match_patterns()
|
||||
|
||||
# Both engines must agree on the number of activations.
|
||||
self.assertEqual(len(rete_matches), len(reasoner_matches))
|
||||
self.assertEqual(len(rete_matches), 1)
|
||||
self.assertEqual(
|
||||
rete_matches[0].bindings,
|
||||
{"x": "John", "y": "Mary", "z": "Paris"},
|
||||
)
|
||||
# The RETE match must carry the instantiated conclusion facts too.
|
||||
conclusion, _, _ = reasoner_matches[0]
|
||||
self.assertEqual(conclusion, "LivesNear(John, Paris)")
|
||||
|
||||
def test_reset_clears_all_token_memory(self):
|
||||
engine = ReteEngine()
|
||||
engine.build_network([self._three_condition_rule()])
|
||||
|
||||
engine.add_fact(Fact("f1", "Person", ["John"]))
|
||||
engine.add_fact(Fact("f2", "Parent", ["John", "Mary"]))
|
||||
engine.add_fact(Fact("f3", "Located", ["Mary", "Paris"]))
|
||||
self.assertEqual(len(engine.match_patterns()), 1)
|
||||
|
||||
engine.reset()
|
||||
|
||||
# No stale facts, tokens or activations remain anywhere.
|
||||
self.assertEqual(engine.facts, [])
|
||||
for node in engine.network.values():
|
||||
if isinstance(node, AlphaNode):
|
||||
self.assertEqual(node.tokens, [])
|
||||
elif isinstance(node, BetaNode):
|
||||
self.assertEqual(node.left_tokens, [])
|
||||
self.assertEqual(node.right_tokens, [])
|
||||
self.assertEqual(engine.match_patterns(), [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -60,6 +60,11 @@ class TestSlidingWindowChunker:
|
||||
with pytest.raises(ValidationError):
|
||||
SlidingWindowChunker(chunk_size=100, overlap=100)
|
||||
|
||||
@pytest.mark.parametrize("stride", [0, -1])
|
||||
def test_init_rejects_non_positive_stride(self, stride):
|
||||
with pytest.raises(ValidationError, match="stride must be positive"):
|
||||
SlidingWindowChunker(chunk_size=100, stride=stride)
|
||||
|
||||
def test_empty_text_returns_empty(self):
|
||||
chunker = SlidingWindowChunker(chunk_size=50, overlap=10)
|
||||
assert chunker.chunk("") == []
|
||||
@@ -97,6 +102,49 @@ class TestSlidingWindowChunker:
|
||||
chunks = chunker.chunk_with_overlap(text, overlap_size=15)
|
||||
assert len(chunks) >= 2
|
||||
assert chunker.overlap == 0
|
||||
assert chunker.stride == 50
|
||||
|
||||
@pytest.mark.parametrize("overlap_size", [-1, 50, 51])
|
||||
def test_chunk_with_overlap_rejects_invalid_override(self, overlap_size):
|
||||
chunker = SlidingWindowChunker(chunk_size=50)
|
||||
|
||||
with pytest.raises(ValidationError):
|
||||
chunker.chunk_with_overlap(
|
||||
"non-empty input", overlap_size=overlap_size
|
||||
)
|
||||
|
||||
def test_chunk_with_overlap_accepts_largest_valid_override(self):
|
||||
chunker = SlidingWindowChunker(chunk_size=5)
|
||||
|
||||
chunks = chunker.chunk_with_overlap("abcdefghij", overlap_size=4)
|
||||
|
||||
assert [chunk.start_index for chunk in chunks] == list(range(10))
|
||||
|
||||
def test_chunk_with_overlap_restores_custom_stride(self):
|
||||
chunker = SlidingWindowChunker(chunk_size=10, overlap=2, stride=3)
|
||||
|
||||
chunker.chunk_with_overlap(
|
||||
"abcdefghijklmnopqrstuvwxyz", overlap_size=4
|
||||
)
|
||||
|
||||
assert chunker.overlap == 2
|
||||
assert chunker.stride == 3
|
||||
|
||||
def test_chunk_with_overlap_restores_state_when_chunk_raises(
|
||||
self, monkeypatch
|
||||
):
|
||||
chunker = SlidingWindowChunker(chunk_size=10, overlap=2, stride=3)
|
||||
|
||||
def raise_error(text):
|
||||
raise RuntimeError("chunk failed")
|
||||
|
||||
monkeypatch.setattr(chunker, "chunk", raise_error)
|
||||
|
||||
with pytest.raises(RuntimeError, match="chunk failed"):
|
||||
chunker.chunk_with_overlap("non-empty input", overlap_size=4)
|
||||
|
||||
assert chunker.overlap == 2
|
||||
assert chunker.stride == 3
|
||||
|
||||
def test_boundary_preservation_avoids_mid_word_when_possible(self):
|
||||
text = (
|
||||
|
||||
@@ -1359,6 +1359,101 @@ class TestStore:
|
||||
result = runner.invoke(cli_module.main, ["store", "migrate", "--from", "faiss"])
|
||||
assert result.exit_code != 0
|
||||
|
||||
def test_migrate_refuses_unsupported_backend_pair(self, runner):
|
||||
result = runner.invoke(cli_module.main, ["store", "migrate",
|
||||
"--from", "faiss", "--to", "qdrant"])
|
||||
assert result.exit_code != 0
|
||||
assert "faiss, pgvector, sqlite" in result.output
|
||||
|
||||
def _fake_migrate_store_module(self, source_items, stored, dest_configs=None):
|
||||
class _FakeBackendStore:
|
||||
def __init__(self, dimension=None):
|
||||
self.dimension = dimension
|
||||
|
||||
class _FakeStore:
|
||||
def __init__(self, backend, config=None, **kw):
|
||||
self.backend = backend
|
||||
self._config = config or {}
|
||||
dim = self._config.get("dimension")
|
||||
self._backend_store = _FakeBackendStore(dimension=dim)
|
||||
if dest_configs is not None:
|
||||
dest_configs[backend] = dict(self._config)
|
||||
|
||||
def iter_vectors(self, batch_size=500):
|
||||
if self.backend == "sqlite":
|
||||
yield from source_items
|
||||
return
|
||||
return
|
||||
yield # pragma: no cover - makes this a generator for other backends
|
||||
|
||||
def store_vectors(self, vectors, metadata, ids=None):
|
||||
for vec_id, meta in zip(ids, metadata):
|
||||
stored[vec_id] = meta
|
||||
|
||||
return _fake_module(VectorStore=_FakeStore)
|
||||
|
||||
def test_migrate_runs_between_supported_backends(self, runner, monkeypatch):
|
||||
source_items = [
|
||||
{"id": "a", "vector": [0.1, 0.2], "metadata": {"tag": "x"}},
|
||||
{"id": "b", "vector": [0.3, 0.4], "metadata": {}},
|
||||
]
|
||||
stored = {}
|
||||
fake_vs = self._fake_migrate_store_module(source_items, stored)
|
||||
monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs)
|
||||
|
||||
result = runner.invoke(cli_module.main, ["store", "migrate",
|
||||
"--from", "sqlite", "--to", "pgvector",
|
||||
"--namespace", "prod", "--json"])
|
||||
_ok(result)
|
||||
data = _json_output(result)
|
||||
assert data == {"from": "sqlite", "to": "pgvector", "migrated": 2}
|
||||
assert stored == {"a": {"tag": "x", "namespace": "prod"}, "b": {"namespace": "prod"}}
|
||||
|
||||
def test_migrate_reports_zero_for_empty_source(self, runner, monkeypatch):
|
||||
stored = {}
|
||||
fake_vs = self._fake_migrate_store_module([], stored)
|
||||
monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs)
|
||||
|
||||
result = runner.invoke(cli_module.main, ["store", "migrate",
|
||||
"--from", "sqlite", "--to", "pgvector", "--json"])
|
||||
_ok(result)
|
||||
assert _json_output(result)["migrated"] == 0
|
||||
assert stored == {}
|
||||
|
||||
def test_migrate_inherits_source_dimension_into_dest(self, runner, monkeypatch):
|
||||
source_items = [{"id": "a", "vector": [0.1, 0.2, 0.3], "metadata": {}}]
|
||||
stored = {}
|
||||
dest_configs: dict = {}
|
||||
fake_vs = self._fake_migrate_store_module(source_items, stored, dest_configs)
|
||||
monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs)
|
||||
monkeypatch.setattr(
|
||||
cli_module.Config, "to_dict",
|
||||
lambda self: {"vector_store": {"sqlite": {"dimension": 3}, "pgvector": {}}},
|
||||
)
|
||||
|
||||
result = runner.invoke(cli_module.main, ["store", "migrate",
|
||||
"--from", "sqlite", "--to", "pgvector", "--json"])
|
||||
_ok(result)
|
||||
assert dest_configs["pgvector"].get("dimension") == 3
|
||||
|
||||
def test_migrate_faiss_source_requires_index_path(self, runner, monkeypatch):
|
||||
fake_vs = _fake_module(VectorStore=lambda **kw: MagicMock())
|
||||
monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs)
|
||||
|
||||
result = runner.invoke(cli_module.main, ["store", "migrate",
|
||||
"--from", "faiss", "--to", "sqlite"])
|
||||
assert result.exit_code != 0
|
||||
assert "index_path" in result.output
|
||||
|
||||
def test_migrate_faiss_dest_requires_index_path(self, runner, monkeypatch):
|
||||
fake_vs = _fake_module(VectorStore=lambda **kw: MagicMock())
|
||||
monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs)
|
||||
|
||||
result = runner.invoke(cli_module.main, ["store", "migrate",
|
||||
"--from", "sqlite", "--to", "faiss"])
|
||||
assert result.exit_code != 0
|
||||
assert "index_path" in result.output
|
||||
|
||||
def test_flush_requires_confirm(self, runner):
|
||||
result = runner.invoke(cli_module.main, ["store", "flush"])
|
||||
assert result.exit_code != 0
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Tests for the Anthropic LLM provider wrapper (semantica.llms.Anthropic)."""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from semantica.llms import Anthropic
|
||||
from semantica.utils.exceptions import ProcessingError
|
||||
|
||||
|
||||
def test_construction_stores_model_and_api_key():
|
||||
"""Anthropic(...) should not crash and should remember what it was given."""
|
||||
claude = Anthropic(model="claude-sonnet-4-6", api_key="fake-key")
|
||||
assert claude.model == "claude-sonnet-4-6"
|
||||
assert claude.api_key == "fake-key"
|
||||
|
||||
|
||||
def test_is_available_false_with_no_key(monkeypatch):
|
||||
"""Without a real key, is_available() must be a real False, not truthy junk.
|
||||
|
||||
api_key=None alone isn't enough to prove this: AnthropicProvider falls
|
||||
back to the ANTHROPIC_API_KEY environment variable, so this test has to
|
||||
clear it too or it would pass/fail depending on whoever's machine or CI
|
||||
runner happens to run it.
|
||||
"""
|
||||
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
|
||||
claude = Anthropic(api_key=None)
|
||||
assert claude.is_available() is False
|
||||
|
||||
|
||||
def test_generate_raises_clear_error_when_unavailable(monkeypatch):
|
||||
"""generate() must fail loudly.
|
||||
|
||||
Clears ANTHROPIC_API_KEY for the same reason as test_is_available_false_with_no_key:
|
||||
otherwise this test flakes depending on whether the runner's environment has a key set.
|
||||
"""
|
||||
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
|
||||
claude = Anthropic(api_key=None)
|
||||
with pytest.raises(ProcessingError, match="Anthropic provider not available"):
|
||||
claude.generate("hello")
|
||||
|
||||
|
||||
def test_generate_forwards_to_the_real_provider_when_available():
|
||||
"""When available, generate() must actually call through to the real provider."""
|
||||
claude = Anthropic(api_key="fake-key")
|
||||
|
||||
claude.provider = MagicMock()
|
||||
claude.provider.is_available.return_value = True
|
||||
claude.provider.generate.return_value = "a fake response"
|
||||
|
||||
result = claude.generate("hello", temperature=0.5)
|
||||
|
||||
assert result == "a fake response"
|
||||
claude.provider.generate.assert_called_once_with("hello", temperature=0.5)
|
||||
|
||||
|
||||
def test_generate_structured_forwards_to_the_real_provider():
|
||||
claude = Anthropic(api_key="fake-key")
|
||||
claude.provider = MagicMock()
|
||||
claude.provider.is_available.return_value = True
|
||||
claude.provider.generate_structured.return_value = {"key": "value"}
|
||||
|
||||
result = claude.generate_structured("hello")
|
||||
|
||||
assert result == {"key": "value"}
|
||||
claude.provider.generate_structured.assert_called_once_with("hello")
|
||||
|
||||
|
||||
def test_generate_typed_forwards_schema_and_max_retries():
|
||||
claude = Anthropic(api_key="fake-key")
|
||||
claude.provider = MagicMock()
|
||||
claude.provider.is_available.return_value = True
|
||||
fake_schema = object()
|
||||
claude.provider.generate_typed.return_value = "typed result"
|
||||
|
||||
result = claude.generate_typed("hello", fake_schema, max_retries=5)
|
||||
|
||||
assert result == "typed result"
|
||||
claude.provider.generate_typed.assert_called_once_with(
|
||||
"hello", fake_schema, max_retries=5
|
||||
)
|
||||
|
||||
|
||||
def test_generate_structured_raises_clear_error_when_unavailable(monkeypatch):
|
||||
"""generate_structured() must fail loudly, same as generate()."""
|
||||
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
|
||||
claude = Anthropic(api_key=None)
|
||||
with pytest.raises(ProcessingError, match="Anthropic provider not available"):
|
||||
claude.generate_structured("hello")
|
||||
|
||||
|
||||
def test_generate_typed_raises_clear_error_when_unavailable(monkeypatch):
|
||||
"""generate_typed() must fail loudly, same as generate() and generate_structured()."""
|
||||
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
|
||||
claude = Anthropic(api_key=None)
|
||||
with pytest.raises(ProcessingError, match="Anthropic provider not available"):
|
||||
claude.generate_typed("hello", object())
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Tests for the DeepSeek LLM provider wrapper (semantica.llms.DeepSeek)."""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from semantica.llms import DeepSeek
|
||||
from semantica.utils.exceptions import ProcessingError
|
||||
|
||||
|
||||
def test_construction_stores_model_and_api_key():
|
||||
llm = DeepSeek(model="deepseek-chat", api_key="fake-key")
|
||||
assert llm.model == "deepseek-chat"
|
||||
assert llm.api_key == "fake-key"
|
||||
|
||||
|
||||
def test_is_available_false_with_no_key(monkeypatch):
|
||||
monkeypatch.delenv("DEEPSEEK_API_KEY", raising=False)
|
||||
llm = DeepSeek(api_key=None)
|
||||
assert llm.is_available() is False
|
||||
|
||||
|
||||
def test_generate_raises_clear_error_when_unavailable(monkeypatch):
|
||||
monkeypatch.delenv("DEEPSEEK_API_KEY", raising=False)
|
||||
llm = DeepSeek(api_key=None)
|
||||
with pytest.raises(ProcessingError, match="DeepSeek provider not available"):
|
||||
llm.generate("hello")
|
||||
|
||||
|
||||
def test_generate_forwards_to_the_real_provider_when_available():
|
||||
llm = DeepSeek(api_key="fake-key")
|
||||
llm.provider = MagicMock()
|
||||
llm.provider.is_available.return_value = True
|
||||
llm.provider.generate.return_value = "a fake response"
|
||||
|
||||
result = llm.generate("hello", temperature=0.5)
|
||||
|
||||
assert result == "a fake response"
|
||||
llm.provider.generate.assert_called_once_with("hello", temperature=0.5)
|
||||
|
||||
|
||||
def test_generate_structured_forwards_to_the_real_provider():
|
||||
llm = DeepSeek(api_key="fake-key")
|
||||
llm.provider = MagicMock()
|
||||
llm.provider.is_available.return_value = True
|
||||
llm.provider.generate_structured.return_value = {"key": "value"}
|
||||
|
||||
result = llm.generate_structured("hello")
|
||||
|
||||
assert result == {"key": "value"}
|
||||
llm.provider.generate_structured.assert_called_once_with("hello")
|
||||
|
||||
|
||||
def test_generate_typed_forwards_schema_and_max_retries():
|
||||
llm = DeepSeek(api_key="fake-key")
|
||||
llm.provider = MagicMock()
|
||||
llm.provider.is_available.return_value = True
|
||||
fake_schema = object()
|
||||
llm.provider.generate_typed.return_value = "typed result"
|
||||
|
||||
result = llm.generate_typed("hello", fake_schema, max_retries=5)
|
||||
|
||||
assert result == "typed result"
|
||||
llm.provider.generate_typed.assert_called_once_with(
|
||||
"hello", fake_schema, max_retries=5
|
||||
)
|
||||
|
||||
|
||||
def test_generate_structured_raises_clear_error_when_unavailable(monkeypatch):
|
||||
monkeypatch.delenv("DEEPSEEK_API_KEY", raising=False)
|
||||
llm = DeepSeek(api_key=None)
|
||||
with pytest.raises(ProcessingError, match="DeepSeek provider not available"):
|
||||
llm.generate_structured("hello")
|
||||
|
||||
|
||||
def test_generate_typed_raises_clear_error_when_unavailable(monkeypatch):
|
||||
monkeypatch.delenv("DEEPSEEK_API_KEY", raising=False)
|
||||
llm = DeepSeek(api_key=None)
|
||||
with pytest.raises(ProcessingError, match="DeepSeek provider not available"):
|
||||
llm.generate_typed("hello", object())
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Tests for the Gemini LLM provider wrapper (semantica.llms.Gemini)."""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from semantica.llms import Gemini
|
||||
from semantica.utils.exceptions import ProcessingError
|
||||
|
||||
|
||||
def test_construction_stores_model_and_api_key():
|
||||
gemini = Gemini(model="gemini-pro", api_key="fake-key")
|
||||
assert gemini.model == "gemini-pro"
|
||||
assert gemini.api_key == "fake-key"
|
||||
|
||||
|
||||
def test_is_available_false_with_no_key(monkeypatch):
|
||||
monkeypatch.delenv("GEMINI_API_KEY", raising=False)
|
||||
gemini = Gemini(api_key=None)
|
||||
assert gemini.is_available() is False
|
||||
|
||||
|
||||
def test_generate_raises_clear_error_when_unavailable(monkeypatch):
|
||||
monkeypatch.delenv("GEMINI_API_KEY", raising=False)
|
||||
gemini = Gemini(api_key=None)
|
||||
with pytest.raises(ProcessingError, match="Gemini provider not available"):
|
||||
gemini.generate("hello")
|
||||
|
||||
|
||||
def test_generate_forwards_to_the_real_provider_when_available():
|
||||
gemini = Gemini(api_key="fake-key")
|
||||
gemini.provider = MagicMock()
|
||||
gemini.provider.is_available.return_value = True
|
||||
gemini.provider.generate.return_value = "a fake response"
|
||||
|
||||
result = gemini.generate("hello", temperature=0.5)
|
||||
|
||||
assert result == "a fake response"
|
||||
gemini.provider.generate.assert_called_once_with("hello", temperature=0.5)
|
||||
|
||||
|
||||
def test_generate_structured_forwards_to_the_real_provider():
|
||||
gemini = Gemini(api_key="fake-key")
|
||||
gemini.provider = MagicMock()
|
||||
gemini.provider.is_available.return_value = True
|
||||
gemini.provider.generate_structured.return_value = {"key": "value"}
|
||||
|
||||
result = gemini.generate_structured("hello")
|
||||
|
||||
assert result == {"key": "value"}
|
||||
gemini.provider.generate_structured.assert_called_once_with("hello")
|
||||
|
||||
|
||||
def test_generate_typed_forwards_schema_and_max_retries():
|
||||
gemini = Gemini(api_key="fake-key")
|
||||
gemini.provider = MagicMock()
|
||||
gemini.provider.is_available.return_value = True
|
||||
fake_schema = object()
|
||||
gemini.provider.generate_typed.return_value = "typed result"
|
||||
|
||||
result = gemini.generate_typed("hello", fake_schema, max_retries=5)
|
||||
|
||||
assert result == "typed result"
|
||||
gemini.provider.generate_typed.assert_called_once_with(
|
||||
"hello", fake_schema, max_retries=5
|
||||
)
|
||||
|
||||
|
||||
def test_generate_structured_raises_clear_error_when_unavailable(monkeypatch):
|
||||
monkeypatch.delenv("GEMINI_API_KEY", raising=False)
|
||||
gemini = Gemini(api_key=None)
|
||||
with pytest.raises(ProcessingError, match="Gemini provider not available"):
|
||||
gemini.generate_structured("hello")
|
||||
|
||||
|
||||
def test_generate_typed_raises_clear_error_when_unavailable(monkeypatch):
|
||||
monkeypatch.delenv("GEMINI_API_KEY", raising=False)
|
||||
gemini = Gemini(api_key=None)
|
||||
with pytest.raises(ProcessingError, match="Gemini provider not available"):
|
||||
gemini.generate_typed("hello", object())
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Tests for the Novita LLM provider wrapper (semantica.llms.Novita)."""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from semantica.llms import Novita
|
||||
from semantica.utils.exceptions import ProcessingError
|
||||
|
||||
|
||||
def test_construction_stores_model_and_api_key():
|
||||
llm = Novita(model="deepseek/deepseek-v3.2", api_key="fake-key")
|
||||
assert llm.model == "deepseek/deepseek-v3.2"
|
||||
assert llm.api_key == "fake-key"
|
||||
|
||||
|
||||
def test_is_available_false_with_no_key(monkeypatch):
|
||||
monkeypatch.delenv("NOVITA_API_KEY", raising=False)
|
||||
llm = Novita(api_key=None)
|
||||
assert llm.is_available() is False
|
||||
|
||||
|
||||
def test_generate_raises_clear_error_when_unavailable(monkeypatch):
|
||||
monkeypatch.delenv("NOVITA_API_KEY", raising=False)
|
||||
llm = Novita(api_key=None)
|
||||
with pytest.raises(ProcessingError, match="Novita provider not available"):
|
||||
llm.generate("hello")
|
||||
|
||||
|
||||
def test_generate_forwards_to_the_real_provider_when_available():
|
||||
llm = Novita(api_key="fake-key")
|
||||
llm.provider = MagicMock()
|
||||
llm.provider.is_available.return_value = True
|
||||
llm.provider.generate.return_value = "a fake response"
|
||||
|
||||
result = llm.generate("hello", temperature=0.5)
|
||||
|
||||
assert result == "a fake response"
|
||||
llm.provider.generate.assert_called_once_with("hello", temperature=0.5)
|
||||
|
||||
|
||||
def test_generate_structured_forwards_to_the_real_provider():
|
||||
llm = Novita(api_key="fake-key")
|
||||
llm.provider = MagicMock()
|
||||
llm.provider.is_available.return_value = True
|
||||
llm.provider.generate_structured.return_value = {"key": "value"}
|
||||
|
||||
result = llm.generate_structured("hello")
|
||||
|
||||
assert result == {"key": "value"}
|
||||
llm.provider.generate_structured.assert_called_once_with("hello")
|
||||
|
||||
|
||||
def test_generate_typed_forwards_schema_and_max_retries():
|
||||
llm = Novita(api_key="fake-key")
|
||||
llm.provider = MagicMock()
|
||||
llm.provider.is_available.return_value = True
|
||||
fake_schema = object()
|
||||
llm.provider.generate_typed.return_value = "typed result"
|
||||
|
||||
result = llm.generate_typed("hello", fake_schema, max_retries=5)
|
||||
|
||||
assert result == "typed result"
|
||||
llm.provider.generate_typed.assert_called_once_with(
|
||||
"hello", fake_schema, max_retries=5
|
||||
)
|
||||
|
||||
|
||||
def test_generate_structured_raises_clear_error_when_unavailable(monkeypatch):
|
||||
monkeypatch.delenv("NOVITA_API_KEY", raising=False)
|
||||
llm = Novita(api_key=None)
|
||||
with pytest.raises(ProcessingError, match="Novita provider not available"):
|
||||
llm.generate_structured("hello")
|
||||
|
||||
|
||||
def test_generate_typed_raises_clear_error_when_unavailable(monkeypatch):
|
||||
monkeypatch.delenv("NOVITA_API_KEY", raising=False)
|
||||
llm = Novita(api_key=None)
|
||||
with pytest.raises(ProcessingError, match="Novita provider not available"):
|
||||
llm.generate_typed("hello", object())
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Tests for the Ollama LLM provider wrapper (semantica.llms.Ollama)."""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from semantica.llms import Ollama
|
||||
from semantica.utils.exceptions import ProcessingError
|
||||
|
||||
|
||||
def test_construction_stores_model_and_base_url():
|
||||
llm = Ollama(model="llama2", base_url="http://localhost:11434")
|
||||
assert llm.model == "llama2"
|
||||
assert llm.base_url == "http://localhost:11434"
|
||||
|
||||
|
||||
def test_is_available_false_without_a_running_server():
|
||||
"""No api_key here, Ollama has none. Without a real server (or the ollama
|
||||
package) reachable at base_url, this must be a real False."""
|
||||
llm = Ollama(base_url="http://localhost:1")
|
||||
assert llm.is_available() is False
|
||||
|
||||
|
||||
def test_generate_raises_clear_error_when_unavailable():
|
||||
llm = Ollama(base_url="http://localhost:1")
|
||||
with pytest.raises(ProcessingError, match="Ollama provider not available"):
|
||||
llm.generate("hello")
|
||||
|
||||
|
||||
def test_generate_forwards_to_the_real_provider_when_available():
|
||||
llm = Ollama()
|
||||
llm.provider = MagicMock()
|
||||
llm.provider.is_available.return_value = True
|
||||
llm.provider.generate.return_value = "a fake response"
|
||||
|
||||
result = llm.generate("hello", temperature=0.5)
|
||||
|
||||
assert result == "a fake response"
|
||||
llm.provider.generate.assert_called_once_with("hello", temperature=0.5)
|
||||
|
||||
|
||||
def test_generate_structured_forwards_to_the_real_provider():
|
||||
llm = Ollama()
|
||||
llm.provider = MagicMock()
|
||||
llm.provider.is_available.return_value = True
|
||||
llm.provider.generate_structured.return_value = {"key": "value"}
|
||||
|
||||
result = llm.generate_structured("hello")
|
||||
|
||||
assert result == {"key": "value"}
|
||||
llm.provider.generate_structured.assert_called_once_with("hello")
|
||||
|
||||
|
||||
def test_generate_typed_forwards_schema_and_max_retries():
|
||||
llm = Ollama()
|
||||
llm.provider = MagicMock()
|
||||
llm.provider.is_available.return_value = True
|
||||
fake_schema = object()
|
||||
llm.provider.generate_typed.return_value = "typed result"
|
||||
|
||||
result = llm.generate_typed("hello", fake_schema, max_retries=5)
|
||||
|
||||
assert result == "typed result"
|
||||
llm.provider.generate_typed.assert_called_once_with(
|
||||
"hello", fake_schema, max_retries=5
|
||||
)
|
||||
|
||||
|
||||
def test_generate_structured_raises_clear_error_when_unavailable():
|
||||
llm = Ollama(base_url="http://localhost:1")
|
||||
with pytest.raises(ProcessingError, match="Ollama provider not available"):
|
||||
llm.generate_structured("hello")
|
||||
|
||||
|
||||
def test_generate_typed_raises_clear_error_when_unavailable():
|
||||
llm = Ollama(base_url="http://localhost:1")
|
||||
with pytest.raises(ProcessingError, match="Ollama provider not available"):
|
||||
llm.generate_typed("hello", object())
|
||||
File diff suppressed because it is too large
Load Diff
@@ -24,9 +24,20 @@ import pytest
|
||||
# rdf:/rdfs: namespaces) and neither the code nor this test caught it,
|
||||
# since both had the same bug. Importing the real function makes that class
|
||||
# of drift impossible.
|
||||
from semantica.explorer.routes.sparql import _is_read_only_query
|
||||
# fastapi ships in the optional `explorer` extra, not in `dev`, so this import
|
||||
# fails on a plain dev install. Only the SPARQL class below needs it; the Cypher,
|
||||
# XXE, vector-serialization and SSRF classes in this module are independent, so
|
||||
# the skip is scoped to the one class rather than the whole file.
|
||||
try:
|
||||
from semantica.explorer.routes.sparql import _is_read_only_query
|
||||
except ImportError: # pragma: no cover - depends on the installed extras
|
||||
_is_read_only_query = None
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
_is_read_only_query is None,
|
||||
reason="requires semantica[explorer] (fastapi)",
|
||||
)
|
||||
class TestSparqlReadOnlyValidation:
|
||||
"""Regression tests for SPARQL injection prevention."""
|
||||
|
||||
|
||||
@@ -443,4 +443,3 @@ def test_export_seed_data(seed_manager, temp_data_dir):
|
||||
rows = list(reader)
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["id"] == "1"
|
||||
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
"""Console progress must be written to stderr, never to stdout.
|
||||
|
||||
Progress is diagnostic output. Writing it to stdout corrupts any program that
|
||||
carries a machine-readable protocol there — the stdio MCP servers put
|
||||
newline-delimited JSON-RPC on stdout, and a progress bar interleaved with a
|
||||
response body makes that response unparseable (#1134).
|
||||
|
||||
Both servers currently defend against this by setting
|
||||
SEMANTICA_DISABLE_PROGRESS, and a console display is only attached when the
|
||||
console is interactive. Those are containments, not the fix: they give up
|
||||
progress output entirely, and every future entry point has to remember them.
|
||||
Writing to the correct stream in the first place is what these tests pin.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import io
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def progress_module():
|
||||
"""Import the real progress_tracker, bypassing a mocked sys.modules entry.
|
||||
|
||||
tests/test_extractors_dispatch.py assigns a MagicMock over
|
||||
'semantica.utils.progress_tracker' at import time and never restores it, so
|
||||
a plain module-level import here returns mocks when that file has already
|
||||
run. Dropping the cached entry re-imports the real module.
|
||||
|
||||
Both bindings are restored afterwards: importing a submodule also rebinds it
|
||||
as an attribute of its parent package, so restoring only the sys.modules
|
||||
entry would leave `semantica.utils.progress_tracker` and
|
||||
`sys.modules["semantica.utils.progress_tracker"]` pointing at different
|
||||
objects for every test that follows.
|
||||
"""
|
||||
name = "semantica.utils.progress_tracker"
|
||||
attr = name.rsplit(".", 1)[1]
|
||||
parent = importlib.import_module("semantica.utils")
|
||||
|
||||
missing = object()
|
||||
saved_entry = sys.modules.get(name, missing)
|
||||
saved_attr = getattr(parent, attr, missing)
|
||||
|
||||
sys.modules.pop(name, None)
|
||||
try:
|
||||
module = importlib.import_module(name)
|
||||
assert hasattr(module, "__file__"), "expected the real module, got a stand-in"
|
||||
yield module
|
||||
finally:
|
||||
if saved_entry is missing:
|
||||
sys.modules.pop(name, None)
|
||||
else:
|
||||
sys.modules[name] = saved_entry
|
||||
|
||||
if saved_attr is missing:
|
||||
if hasattr(parent, attr):
|
||||
delattr(parent, attr)
|
||||
else:
|
||||
setattr(parent, attr, saved_attr)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def display_cls(progress_module):
|
||||
return progress_module.ConsoleProgressDisplay
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def make_item(progress_module):
|
||||
def _make(**overrides):
|
||||
defaults = dict(
|
||||
module="kg",
|
||||
submodule="Reasoner",
|
||||
message="Inferring facts",
|
||||
status="running",
|
||||
total_items=10,
|
||||
processed_items=3,
|
||||
)
|
||||
defaults.update(overrides)
|
||||
return progress_module.ProgressItem(**defaults)
|
||||
|
||||
return _make
|
||||
|
||||
|
||||
class TestProgressStreamDefaults:
|
||||
def test_defaults_to_stderr(self, display_cls):
|
||||
assert display_cls().stream is sys.stderr
|
||||
|
||||
def test_default_is_not_stdout(self, display_cls):
|
||||
"""The whole point: stdout stays clean for protocol traffic."""
|
||||
assert display_cls().stream is not sys.stdout
|
||||
|
||||
def test_stream_follows_rebinding(self, display_cls, monkeypatch):
|
||||
"""Resolved per write, so pytest capture and later rebinds are honoured."""
|
||||
display = display_cls()
|
||||
replacement = io.StringIO()
|
||||
monkeypatch.setattr(sys, "stderr", replacement)
|
||||
assert display.stream is replacement
|
||||
|
||||
def test_explicit_stream_overrides_the_default(self, display_cls):
|
||||
buffer = io.StringIO()
|
||||
assert display_cls(stream=buffer).stream is buffer
|
||||
|
||||
|
||||
class TestProgressWritesGoToTheStream:
|
||||
def test_update_writes_to_the_configured_stream(self, display_cls, make_item):
|
||||
buffer = io.StringIO()
|
||||
display = display_cls(stream=buffer, use_emoji=False, update_interval=0.0)
|
||||
|
||||
display.update(make_item())
|
||||
|
||||
assert buffer.getvalue(), "progress should have been rendered"
|
||||
|
||||
def test_update_writes_nothing_to_stdout(self, display_cls, make_item, monkeypatch):
|
||||
"""Regression guard for #1134: stdout must stay untouched."""
|
||||
fake_stdout = io.StringIO()
|
||||
monkeypatch.setattr(sys, "stdout", fake_stdout)
|
||||
buffer = io.StringIO()
|
||||
|
||||
display = display_cls(stream=buffer, use_emoji=False, update_interval=0.0)
|
||||
display.update(make_item())
|
||||
display.clear()
|
||||
|
||||
assert fake_stdout.getvalue() == "", (
|
||||
f"console progress leaked to stdout: {fake_stdout.getvalue()!r}"
|
||||
)
|
||||
|
||||
def test_default_display_writes_nothing_to_stdout(
|
||||
self, display_cls, make_item, monkeypatch
|
||||
):
|
||||
"""Same guard without an explicit stream, i.e. the real default path."""
|
||||
fake_stdout = io.StringIO()
|
||||
fake_stderr = io.StringIO()
|
||||
monkeypatch.setattr(sys, "stdout", fake_stdout)
|
||||
monkeypatch.setattr(sys, "stderr", fake_stderr)
|
||||
|
||||
display = display_cls(use_emoji=False, update_interval=0.0)
|
||||
display.update(make_item())
|
||||
|
||||
assert fake_stdout.getvalue() == ""
|
||||
assert fake_stderr.getvalue(), "progress should have gone to stderr"
|
||||
|
||||
def test_clear_flushes_the_stream_not_stdout(self, display_cls, monkeypatch):
|
||||
flushed = []
|
||||
|
||||
class RecordingStream(io.StringIO):
|
||||
def flush(self):
|
||||
flushed.append("stream")
|
||||
|
||||
class ExplodingStdout(io.StringIO):
|
||||
def flush(self): # pragma: no cover - fails the test if reached
|
||||
raise AssertionError("progress must not flush stdout")
|
||||
|
||||
monkeypatch.setattr(sys, "stdout", ExplodingStdout())
|
||||
display = display_cls(
|
||||
stream=RecordingStream(), use_emoji=False, update_interval=0.0
|
||||
)
|
||||
|
||||
display.clear()
|
||||
|
||||
assert flushed == ["stream"]
|
||||
|
||||
|
||||
class TestEncodingFallback:
|
||||
def test_falls_back_when_the_stream_cannot_encode(self, display_cls):
|
||||
"""A cp1252-style console must not raise on emoji; it degrades instead."""
|
||||
|
||||
class AsciiOnly(io.StringIO):
|
||||
encoding = "ascii"
|
||||
|
||||
def write(self, text):
|
||||
text.encode("ascii") # raises UnicodeEncodeError on emoji
|
||||
return super().write(text)
|
||||
|
||||
buffer = AsciiOnly()
|
||||
display = display_cls(stream=buffer, use_emoji=True, update_interval=0.0)
|
||||
|
||||
display._safe_write("progress \U0001f504 bar\n")
|
||||
|
||||
assert "progress" in buffer.getvalue()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__])
|
||||
@@ -3,7 +3,7 @@ from unittest.mock import MagicMock
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from semantica.vector_store.faiss_store import FAISSIndex
|
||||
from semantica.vector_store.faiss_store import FAISSIndex, FAISSStore
|
||||
|
||||
|
||||
def test_get_vector_reconstructs_from_flat_l2_index():
|
||||
@@ -120,3 +120,90 @@ def test_get_vector_reconstructs_from_real_ivfflat_index_without_prior_direct_ma
|
||||
result = index.get_vector("vec_target")
|
||||
|
||||
np.testing.assert_allclose(result, vectors[3], atol=1e-6)
|
||||
|
||||
|
||||
def _store_with_fake_index(ids, metadata_by_id=None):
|
||||
backend_index = MagicMock()
|
||||
backend_index.reconstruct.side_effect = lambda idx: [float(idx)] * 3
|
||||
index = FAISSIndex(backend_index, dimension=3)
|
||||
index.vector_ids = list(ids)
|
||||
index.metadata = dict(metadata_by_id or {})
|
||||
|
||||
store = FAISSStore(dimension=3)
|
||||
store.index = index
|
||||
return store
|
||||
|
||||
|
||||
def test_scan_vectors_returns_all_across_pages():
|
||||
store = _store_with_fake_index(["a", "b", "c", "d", "e"])
|
||||
|
||||
seen_ids = []
|
||||
offset = 0
|
||||
while True:
|
||||
page = store.scan_vectors(offset=offset, limit=2)
|
||||
if not page:
|
||||
break
|
||||
seen_ids.extend(p["id"] for p in page)
|
||||
offset += len(page)
|
||||
|
||||
assert seen_ids == ["a", "b", "c", "d", "e"]
|
||||
|
||||
|
||||
def test_scan_vectors_includes_vector_and_metadata():
|
||||
store = _store_with_fake_index(["a"], {"a": {"tag": "only"}})
|
||||
|
||||
page = store.scan_vectors(offset=0, limit=10)
|
||||
|
||||
assert len(page) == 1
|
||||
assert page[0]["id"] == "a"
|
||||
assert page[0]["metadata"] == {"tag": "only"}
|
||||
np.testing.assert_array_equal(page[0]["vector"], np.array([0.0, 0.0, 0.0], dtype=np.float32))
|
||||
|
||||
|
||||
def test_scan_vectors_no_index_returns_empty_list():
|
||||
store = FAISSStore(dimension=3)
|
||||
assert store.scan_vectors(offset=0, limit=10) == []
|
||||
|
||||
|
||||
def test_scan_vectors_zero_limit_returns_empty_list():
|
||||
store = _store_with_fake_index(["a"])
|
||||
assert store.scan_vectors(offset=0, limit=0) == []
|
||||
|
||||
|
||||
def test_scan_vectors_offset_past_end_returns_empty_list():
|
||||
store = _store_with_fake_index(["a"])
|
||||
assert store.scan_vectors(offset=100, limit=10) == []
|
||||
|
||||
|
||||
def test_add_vectors_retry_with_same_ids_does_not_duplicate():
|
||||
"""Re-running add_vectors with ids already in the index (e.g. retrying
|
||||
an interrupted migration) must not create a second physical vector
|
||||
under the same id."""
|
||||
backend_index = MagicMock()
|
||||
store = FAISSStore(dimension=3)
|
||||
store.index = FAISSIndex(backend_index, dimension=3)
|
||||
|
||||
vectors = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]], dtype=np.float32)
|
||||
ids = ["a", "b", "c", "d"]
|
||||
|
||||
store.add_vectors(vectors, ids=ids, metadata=[{"i": i} for i in range(4)])
|
||||
assert store.count() == 4
|
||||
|
||||
store.add_vectors(vectors, ids=ids, metadata=[{"i": i} for i in range(4)])
|
||||
|
||||
assert store.count() == 4
|
||||
assert store.index.vector_ids == ids
|
||||
|
||||
|
||||
def test_add_vectors_retry_with_partial_overlap_only_adds_new_ids():
|
||||
backend_index = MagicMock()
|
||||
store = FAISSStore(dimension=3)
|
||||
store.index = FAISSIndex(backend_index, dimension=3)
|
||||
|
||||
store.add_vectors(np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float32), ids=["a", "b"])
|
||||
store.add_vectors(np.array([[1, 2, 3], [7, 8, 9]], dtype=np.float32), ids=["a", "c"])
|
||||
|
||||
assert store.index.vector_ids == ["a", "b", "c"]
|
||||
second_call_vectors = backend_index.add.call_args[0][0]
|
||||
assert second_call_vectors.shape[0] == 1
|
||||
np.testing.assert_array_equal(second_call_vectors[0], np.array([7, 8, 9], dtype=np.float32))
|
||||
|
||||
@@ -467,6 +467,48 @@ class TestPgVectorStoreDelete:
|
||||
assert success is True
|
||||
|
||||
|
||||
class TestPgVectorStoreScan:
|
||||
"""Test scan_vectors pagination."""
|
||||
|
||||
def test_scan_returns_all_vectors_across_pages(self, store):
|
||||
vectors = [np.random.rand(128).astype(np.float32) for _ in range(5)]
|
||||
ids = store.add(vectors, [{"index": i} for i in range(5)])
|
||||
|
||||
seen_ids = []
|
||||
offset = 0
|
||||
while True:
|
||||
page = store.scan_vectors(offset=offset, limit=2)
|
||||
if not page:
|
||||
break
|
||||
seen_ids.extend(p["id"] for p in page)
|
||||
offset += len(page)
|
||||
|
||||
assert set(seen_ids) == set(ids)
|
||||
assert len(seen_ids) == 5
|
||||
|
||||
def test_scan_page_includes_vector_and_metadata(self, store):
|
||||
vectors = [np.random.rand(128).astype(np.float32)]
|
||||
ids = store.add(vectors, [{"tag": "only"}])
|
||||
|
||||
page = store.scan_vectors(offset=0, limit=10)
|
||||
|
||||
assert len(page) == 1
|
||||
assert page[0]["id"] == ids[0]
|
||||
assert page[0]["metadata"] == {"tag": "only"}
|
||||
assert page[0]["vector"] is not None
|
||||
|
||||
def test_scan_empty_store_returns_empty_list(self, store):
|
||||
assert store.scan_vectors(offset=0, limit=10) == []
|
||||
|
||||
def test_scan_zero_limit_returns_empty_list(self, store):
|
||||
store.add([np.random.rand(128).astype(np.float32)])
|
||||
assert store.scan_vectors(offset=0, limit=0) == []
|
||||
|
||||
def test_scan_offset_past_end_returns_empty_list(self, store):
|
||||
store.add([np.random.rand(128).astype(np.float32)])
|
||||
assert store.scan_vectors(offset=100, limit=10) == []
|
||||
|
||||
|
||||
class TestPgVectorStoreIndex:
|
||||
"""Test index creation operations."""
|
||||
|
||||
|
||||
@@ -415,6 +415,48 @@ class TestSQLiteVecStoreStats:
|
||||
assert stats["vector_count"] == 4
|
||||
|
||||
|
||||
class TestSQLiteVecStoreScan:
|
||||
"""Test scan_vectors pagination."""
|
||||
|
||||
def test_scan_returns_all_vectors_across_pages(self, store):
|
||||
vectors = [np.random.rand(128).astype(np.float32) for _ in range(5)]
|
||||
ids = store.add(vectors, [{"index": i} for i in range(5)])
|
||||
|
||||
seen_ids = []
|
||||
offset = 0
|
||||
while True:
|
||||
page = store.scan_vectors(offset=offset, limit=2)
|
||||
if not page:
|
||||
break
|
||||
seen_ids.extend(p["id"] for p in page)
|
||||
offset += len(page)
|
||||
|
||||
assert set(seen_ids) == set(ids)
|
||||
assert len(seen_ids) == 5
|
||||
|
||||
def test_scan_page_includes_vector_and_metadata(self, store):
|
||||
vectors = [np.random.rand(128).astype(np.float32)]
|
||||
ids = store.add(vectors, [{"tag": "only"}])
|
||||
|
||||
page = store.scan_vectors(offset=0, limit=10)
|
||||
|
||||
assert len(page) == 1
|
||||
assert page[0]["id"] == ids[0]
|
||||
assert page[0]["metadata"] == {"tag": "only"}
|
||||
assert page[0]["vector"] is not None
|
||||
|
||||
def test_scan_empty_store_returns_empty_list(self, store):
|
||||
assert store.scan_vectors(offset=0, limit=10) == []
|
||||
|
||||
def test_scan_zero_limit_returns_empty_list(self, store):
|
||||
store.add([np.random.rand(128).astype(np.float32)])
|
||||
assert store.scan_vectors(offset=0, limit=0) == []
|
||||
|
||||
def test_scan_offset_past_end_returns_empty_list(self, store):
|
||||
store.add([np.random.rand(128).astype(np.float32)])
|
||||
assert store.scan_vectors(offset=100, limit=10) == []
|
||||
|
||||
|
||||
class TestSQLiteVecStoreFilterByMetadata:
|
||||
"""Test filter_by_metadata, including list-valued metadata handling."""
|
||||
|
||||
|
||||
@@ -120,6 +120,78 @@ class VectorStoreCountTests(unittest.TestCase):
|
||||
self.assertIn("count()", msg)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# VectorStore.scan_vectors() / iter_vectors() dispatch tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class _ScanningBackendStore:
|
||||
"""Fake persistent backend store that supports scan_vectors()."""
|
||||
|
||||
def __init__(self, items):
|
||||
self._items = items
|
||||
|
||||
def scan_vectors(self, offset=0, limit=100):
|
||||
return self._items[offset:offset + limit]
|
||||
|
||||
|
||||
class _NonScanningBackendStore:
|
||||
"""Fake persistent backend store without any scan capability."""
|
||||
|
||||
|
||||
class VectorStoreScanVectorsTests(unittest.TestCase):
|
||||
"""VectorStore.scan_vectors() / iter_vectors() backend-agnostic accessors."""
|
||||
|
||||
def setUp(self):
|
||||
self.vectors = [np.array([1.0, 0.0]), np.array([0.0, 1.0]), np.array([1.0, 1.0])]
|
||||
self.metadata = [{"type": "a"}, {"type": "b"}, {"type": "c"}]
|
||||
|
||||
def test_scan_inmemory_pages_through_all_vectors(self):
|
||||
store = VectorStore(backend="inmemory", dimension=2)
|
||||
ids = store.store_vectors(self.vectors, self.metadata)
|
||||
|
||||
page1 = store.scan_vectors(offset=0, limit=2)
|
||||
page2 = store.scan_vectors(offset=2, limit=2)
|
||||
|
||||
self.assertEqual([p["id"] for p in page1], ids[:2])
|
||||
self.assertEqual([p["id"] for p in page2], ids[2:])
|
||||
self.assertEqual(page2[0]["metadata"], {"type": "c"})
|
||||
|
||||
def test_scan_inmemory_empty_store(self):
|
||||
store = VectorStore(backend="inmemory", dimension=2)
|
||||
self.assertEqual(store.scan_vectors(offset=0, limit=10), [])
|
||||
|
||||
def test_scan_zero_limit_returns_empty_list(self):
|
||||
store = VectorStore(backend="inmemory", dimension=2)
|
||||
store.store_vectors(self.vectors, self.metadata)
|
||||
self.assertEqual(store.scan_vectors(offset=0, limit=0), [])
|
||||
|
||||
def test_scan_delegates_to_backend_store(self):
|
||||
items = [{"id": "a", "metadata": {}, "vector": None}]
|
||||
store = VectorStore(backend="inmemory", dimension=2)
|
||||
store.backend = "faiss"
|
||||
store._backend_store = _ScanningBackendStore(items)
|
||||
self.assertEqual(store.scan_vectors(offset=0, limit=10), items)
|
||||
|
||||
def test_scan_raises_not_implemented_without_backend_support(self):
|
||||
store = VectorStore(backend="inmemory", dimension=2)
|
||||
store.backend = "faiss"
|
||||
store._backend_store = _NonScanningBackendStore()
|
||||
with self.assertRaises(NotImplementedError):
|
||||
store.scan_vectors(offset=0, limit=10)
|
||||
|
||||
def test_iter_vectors_walks_every_page(self):
|
||||
store = VectorStore(backend="inmemory", dimension=2)
|
||||
ids = store.store_vectors(self.vectors, self.metadata)
|
||||
|
||||
collected = list(store.iter_vectors(batch_size=2))
|
||||
|
||||
self.assertEqual([item["id"] for item in collected], ids)
|
||||
|
||||
def test_iter_vectors_empty_store_yields_nothing(self):
|
||||
store = VectorStore(backend="inmemory", dimension=2)
|
||||
self.assertEqual(list(store.iter_vectors(batch_size=2)), [])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# VectorManager tests — inmemory backend
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user